topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 21:17:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project] supersonic 0.7.2 version backend update (#28)

Browse Source 
Co-authored-by: jipengli <jipengli@tencent.com>
This commit is contained in:
jipeli
2023-08-15 08:56:18 +08:00
committed by GitHub
parent 27283001a8
commit b1952d64ab
461 changed files with 18548 additions and 11939 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/executor/QueryExecutor.java
View File

@@ -8,5 +8,5 @@ public interface QueryExecutor {
boolean accept(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(Catalog catalog,QueryStatement queryStatement);
QueryResultWithSchemaResp execute(Catalog catalog, QueryStatement queryStatement);
}

1
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/QueryOptimizer.java
View File

@@ -4,5 +4,6 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
public interface QueryOptimizer {
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
}

1
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/SqlParser.java
View File

@@ -5,5 +5,6 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
public interface SqlParser {
QueryStatement explain(MetricReq metricReq, boolean isAgg, Catalog catalog) throws Exception;
}

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java
View File

@@ -3,14 +3,15 @@ package com.tencent.supersonic.semantic.query.parser.calcite;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SqlParser;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import org.springframework.stereotype.Component;
@Component("CalciteSqlParser")
public class CalciteSqlParser implements SqlParser {
private final SemanticSchemaManager semanticSchemaManager;
public CalciteSqlParser(

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/Configuration.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.parser.calcite;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl;
import java.util.Properties;
import org.apache.calcite.avatica.util.Casing;
import org.apache.calcite.avatica.util.Quoting;

14
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java
View File

@@ -54,18 +54,18 @@ public class SemanticSchemaManager {
public SemanticModel reload(String rootPath) {
SemanticModel semanticModel = new SemanticModel();
semanticModel.setRootPath(rootPath);
Map<Long, String> domainFullPathMap = catalog.getDomainFullPath();
log.info("domainFullPathMap {}", domainFullPathMap);
Set<Long> domainIds = domainFullPathMap.entrySet().stream().filter(e -> e.getValue().startsWith(rootPath))
.map(e -> e.getKey()).collect(Collectors.toSet());
if (domainIds.isEmpty()) {
log.error("get domainId empty {}", rootPath);
Map<Long, String> modelFullPathMap = catalog.getModelFullPath();
log.info("modelFullPathMap {}", modelFullPathMap);
Set<Long> modelIds = modelFullPathMap.entrySet().stream().filter(e -> e.getValue().startsWith(rootPath))
.map(Entry::getKey).collect(Collectors.toSet());
if (modelIds.isEmpty()) {
log.error("get modelIds empty {}", rootPath);
return semanticModel;
}
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DatasourceYamlTpl> datasourceYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
catalog.getModelYamlTplByDomainIds(domainIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls);
catalog.getModelYamlTplByMoldelIds(modelIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls);
if (!datasourceYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = datasourceYamlTpls.stream().map(d -> getDatasource(d))
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java
View File

@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.FilterRender;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.OutputRender;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Renderer.java
View File

@@ -2,15 +2,15 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;

21
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
@Slf4j
public class DataSourceNode extends SemanticNode {
@@ -61,16 +60,20 @@ public class DataSourceNode extends SemanticNode {
metricCommand.getMetrics().stream().filter(m -> !schemaMetricName.contains(m)).forEach(m -> measures.add(m));
}
public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricReq metricCommand,
Set<String> queryDimension, List<String> measures,SqlValidatorScope scope) throws Exception {
if(Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
Set<String> queryDimension, List<String> measures, SqlValidatorScope scope) throws Exception {
if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
Set<String> filterConditions = new HashSet<>();
FilterNode.getFilterField(parse(metricCommand.getWhere(), scope), filterConditions);
Set<String> queryMeasures = new HashSet<>(measures);
Set<String> schemaMetricName = schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet());
for(String filterCondition : filterConditions) {
if(schemaMetricName.contains(filterCondition)) {
schema.getMetrics().stream().filter(m->m.getName().equalsIgnoreCase(filterCondition)).forEach(m -> m.getMetricTypeParams().getMeasures().stream().forEach(mm -> queryMeasures.add(mm.getName())));
Set<String> queryMeasures = new HashSet<>(measures);
Set<String> schemaMetricName = schema.getMetrics().stream().map(m -> m.getName())
.collect(Collectors.toSet());
for (String filterCondition : filterConditions) {
if (schemaMetricName.contains(filterCondition)) {
schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(filterCondition)).forEach(
m -> m.getMetricTypeParams().getMeasures().stream()
.forEach(mm -> queryMeasures.add(mm.getName())));
continue;
}
queryDimension.add(filterCondition);
@@ -118,7 +121,7 @@ public class DataSourceNode extends SemanticNode {
}
filterMeasure.addAll(sourceMeasure);
filterMeasure.addAll(dimension);
mergeQueryFilterDimensionMeasure(schema,metricCommand,queryDimension,measures,scope);
mergeQueryFilterDimensionMeasure(schema, metricCommand, queryDimension, measures, scope);
boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, metricCommand, scope);
if (isAllMatch) {
log.info("baseDataSource match all ");

11
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/SemanticNode.java
View File

@@ -2,8 +2,8 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Optimization;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Optimization;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -16,9 +16,11 @@ import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.SqlWriterConfig;
import org.apache.calcite.sql.advise.SqlSimpleParser;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.pretty.SqlPrettyWriter;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.commons.lang3.StringUtils;
@@ -40,7 +42,12 @@ public abstract class SemanticNode {
public static String getSql(SqlNode sqlNode) {
SqlSimpleParser sqlSimpleParser = new SqlSimpleParser("", Configuration.getParserConfig());
return sqlSimpleParser.simplifySql(sqlNode.toSqlString(SemanticSqlDialect.DEFAULT).getSql());
SqlWriterConfig config = SqlPrettyWriter.config().withDialect(SemanticSqlDialect.DEFAULT)
.withKeywordsLowerCase(true).withClauseEndsLine(true).withAlwaysUseParentheses(false)
.withSelectListItemsOnSeparateLines(false).withUpdateSetListNewline(false).withIndentation(0);
return sqlSimpleParser.simplifySql(sqlNode.toSqlString((c) -> {
return config;
}).getSql());
}
public static boolean isNumeric(String expr) {

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/FilterRender.java
View File

@@ -2,14 +2,14 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.HashSet;

13
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java
View File

@@ -1,18 +1,18 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.AggFunctionNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.AggFunctionNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.Arrays;
@@ -28,7 +28,6 @@ import org.apache.calcite.sql.JoinConditionType;
import org.apache.calcite.sql.JoinType;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlJoin;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/OutputRender.java
View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.List;

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java
View File

@@ -70,7 +70,7 @@ public class CalculateAggConverter implements SemanticConverter {
return generateRatioSqlCommand(queryStructCmd, engineTypeEnum, version);
}
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getDomainFullPath(queryStructCmd.getDomainId()));
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelId()));
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
@@ -107,7 +107,7 @@ public class CalculateAggConverter implements SemanticConverter {
return false;
}
//todo ck类型暂不拼with语句
if (queryStructCmd.getDomainId().equals(34L)) {
if (queryStructCmd.getModelId().equals(34L)) {
return false;
}
int nonSumFunction = 0;
@@ -130,7 +130,7 @@ public class CalculateAggConverter implements SemanticConverter {
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
DatabaseResp databaseResp = catalog.getDatabaseByDomainId(queryStructCmd.getDomainId());
DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelId());
ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd,
EngineTypeEnum.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
sqlCommend.setSql(parseSqlReq.getSql());
@@ -160,7 +160,7 @@ public class CalculateAggConverter implements SemanticConverter {
throws Exception {
check(queryStructCmd);
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getDomainFullPath(queryStructCmd.getDomainId()));
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelId()));
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
@@ -180,7 +180,7 @@ public class CalculateAggConverter implements SemanticConverter {
case MYSQL:
if (Objects.nonNull(version) && version.startsWith(mysqlLowVersion)) {
sqlCommand.setSupportWith(false);
sql = new MysqlEngineSql().sql(queryStructCmd, isOver, metricTableName);
sql = new MysqlEngineSql().sql(queryStructCmd, isOver, metricTableName);
break;
}
case DORIS:
@@ -199,7 +199,7 @@ public class CalculateAggConverter implements SemanticConverter {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getFunc().getOperator(),f.getColumn());
f.getFunc().getOperator(), f.getColumn());
} else {
return f.getColumn();
}
@@ -336,8 +336,8 @@ public class CalculateAggConverter implements SemanticConverter {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format(
"if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getColumn(),f.getFunc().getOperator(),f.getColumn());
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getColumn(), f.getFunc().getOperator(), f.getColumn());
} else {
return f.getColumn();
}

11
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java
View File

@@ -8,11 +8,11 @@ import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
@Component("DefaultDimValueConverter")
@@ -24,13 +24,14 @@ public class DefaultDimValueConverter implements SemanticConverter {
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend, MetricReq metricCommand) throws Exception {
List<DimensionResp> dimensionResps = catalog.getDimensions(queryStructCmd.getDomainId());
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
List<DimensionResp> dimensionResps = catalog.getDimensions(queryStructCmd.getModelId());
//dimension which has default values
dimensionResps = dimensionResps.stream()
.filter(dimensionResp -> !CollectionUtils.isEmpty(dimensionResp.getDefaultValues()))
.collect(Collectors.toList());
if(CollectionUtils.isEmpty(dimensionResps)){
if (CollectionUtils.isEmpty(dimensionResps)) {
return;
}
log.info("dimension with default values:{}, queryStruct:{}", dimensionResps, queryStructCmd);

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java
View File

@@ -48,15 +48,15 @@ public class MultiSourceJoin implements SemanticConverter {
if (CollectionUtils.isEmpty(groups) || CollectionUtils.isEmpty(aggregators)) {
return;
}
Long domainId = queryStructCmd.getDomainId();
Long modelId = queryStructCmd.getModelId();
List<String> aggs = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList());
Map<String, DimensionResp> dimensionMap = catalog.getDimensions(domainId).stream()
Map<String, DimensionResp> dimensionMap = catalog.getDimensions(modelId).stream()
.filter(dimensionDesc -> fields.contains(dimensionDesc.getBizName()))
.collect(Collectors.toMap(DimensionResp::getBizName, dimensionDesc -> dimensionDesc));
List<MetricResp> metricDescList = catalog.getMetrics(domainId).stream()
List<MetricResp> metricDescList = catalog.getMetrics(modelId).stream()
.filter(metricDesc -> aggs.contains(metricDesc.getBizName()))
.collect(Collectors.toList());
Map<Long, DatasourceResp> datasourceMap = catalog.getDatasourceList(domainId)
Map<Long, DatasourceResp> datasourceMap = catalog.getDatasourceList(modelId)
.stream().collect(Collectors.toMap(DatasourceResp::getId, datasource -> datasource));
//check groups filters and aggs is in same datasource
if (!isInSameDatasource(new ArrayList<>(dimensionMap.values()), metricDescList)) {
@@ -133,5 +133,5 @@ public class MultiSourceJoin implements SemanticConverter {
buildJoinPrefix(queryStructCmd);
}
}

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java
View File

@@ -63,7 +63,7 @@ public class ParserDefaultConverter implements SemanticConverter {
sqlCommend.setVariables(queryStructCmd.getParams().stream()
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
sqlCommend.setLimit(queryStructCmd.getLimit());
String rootPath = catalog.getDomainFullPath(queryStructCmd.getDomainId());
String rootPath = catalog.getModelFullPath(queryStructCmd.getModelId());
sqlCommend.setRootPath(rootPath);
// todo tmp delete
@@ -84,7 +84,7 @@ public class ParserDefaultConverter implements SemanticConverter {
} else {
String group = queryStructCmd.getGroups().get(0).equalsIgnoreCase("sys_imp_date")
? queryStructCmd.getGroups().get(1) : queryStructCmd.getGroups().get(0);
DimensionResp dimension = catalog.getDimension(group, queryStructCmd.getDomainId());
DimensionResp dimension = catalog.getDimension(group, queryStructCmd.getModelId());
String datasourceBizName = dimension.getDatasourceBizName();
if (Strings.isNotEmpty(datasourceBizName)) {
internalMetricNamePrefix = datasourceBizName + UNDERLINE;

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/pojo/QueryStatement.java
View File

@@ -4,7 +4,8 @@ import lombok.Data;
@Data
public class QueryStatement {
private Long domainId = 0L;
private Long modelId = 0L;
private String sql = "";
private String sourceId = "";
private String errMsg = "";

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepositoryImpl.java
View File

@@ -4,10 +4,10 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.query.persistence.mapper.StatMapper;
import java.util.ArrayList;
import java.util.Comparator;
@@ -44,8 +44,8 @@ public class StatRepositoryImpl implements StatRepository {
statInfos.stream().forEach(stat -> {
String dimensions = stat.getDimensions();
String metrics = stat.getMetrics();
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.getName(), stat.getDomainId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.getName(), stat.getDomainId());
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.getName(), stat.getModelId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.getName(), stat.getModelId());
});
map.forEach((k, v) -> {
@@ -65,13 +65,13 @@ public class StatRepositoryImpl implements StatRepository {
return statMapper.getStatInfo(itemUseCommend);
}
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type, Long domainId) {
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type, Long modelId) {
if (Strings.isNotEmpty(dimensions)) {
try {
List<String> dimensionList = mapper.readValue(dimensions, new TypeReference<List<String>>() {
});
dimensionList.stream().forEach(dimension -> {
String key = domainId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension;
String key = modelId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension;
if (map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {
@@ -84,9 +84,9 @@ public class StatRepositoryImpl implements StatRepository {
}
}
private void updateStatMapInfo(Map<String, Long> map, Long domainId, String type) {
if (Objects.nonNull(domainId)) {
String key = type + AT_SYMBOL + AT_SYMBOL + domainId;
private void updateStatMapInfo(Map<String, Long> map, Long modelId, String type) {
if (Objects.nonNull(modelId)) {
String key = type + AT_SYMBOL + AT_SYMBOL + modelId;
if (map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -3,11 +3,15 @@ package com.tencent.supersonic.semantic.query.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.*;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

30
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/SchemaController.java
View File

@@ -3,24 +3,25 @@ package com.tencent.supersonic.semantic.query.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@RestController
@@ -31,30 +32,27 @@ public class SchemaController {
private SchemaService schemaService;
@PostMapping
public List<DomainSchemaResp> fetchDomainSchema(@RequestBody DomainSchemaFilterReq filter,
public List<ModelSchemaResp> fetchModelSchema(@RequestBody ModelSchemaFilterReq filter,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.fetchDomainSchema(filter, user);
return schemaService.fetchModelSchema(filter, user);
}
/**
* get domain list
*
* @param
*/
@GetMapping("/domain/list")
public List<DomainResp> getDomainList(HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.getDomainListForAdmin(user);
return schemaService.getDomainList(user);
}
@GetMapping("/domain/view/list")
public List<DomainResp> getDomainListForViewer(HttpServletRequest request,
HttpServletResponse response) {
@GetMapping("/model/list")
public List<ModelResp> getModelList(@RequestParam("domainId") Long domainId,
@RequestParam("authType") String authType,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.getDomainListForViewer(user);
return schemaService.getModelList(user, AuthType.valueOf(authType), domainId);
}
@PostMapping("/dimension/page")

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
View File

@@ -4,8 +4,8 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import java.util.List;

22
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -2,11 +2,11 @@ package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
@@ -59,16 +59,16 @@ public class QueryServiceImpl implements QueryService {
@Override
public Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception {
DomainSchemaFilterReq filter = new DomainSchemaFilterReq();
List<Long> domainIds = new ArrayList<>();
domainIds.add(querySqlCmd.getDomainId());
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = new ArrayList<>();
modelIds.add(querySqlCmd.getModelId());
filter.setDomainIds(domainIds);
filter.setModelIds(modelIds);
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<DomainSchemaResp> domainSchemas = schemaService.fetchDomainSchema(filter, user);
List<ModelSchemaResp> domainSchemas = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchemas);
queryStatement.setDomainId(querySqlCmd.getDomainId());
queryStatement.setModelId(querySqlCmd.getModelId());
return semanticQueryEngine.execute(queryStatement);
}
@@ -78,7 +78,7 @@ public class QueryServiceImpl implements QueryService {
log.info("[queryStructCmd:{}]", queryStructCmd);
try {
statUtils.initStatInfo(queryStructCmd, user);
String cacheKey = cacheUtils.generateCacheKey(queryStructCmd.getDomainId().toString(),
String cacheKey = cacheUtils.generateCacheKey(queryStructCmd.getModelId().toString(),
queryStructCmd.generateCommandMd5());
handleGlobalCacheDisable(queryStructCmd);
boolean isCache = isCache(queryStructCmd);
@@ -121,7 +121,7 @@ public class QueryServiceImpl implements QueryService {
throws Exception {
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
String cacheKey = cacheUtils.generateCacheKey(
queryMultiStructReq.getQueryStructReqs().get(0).getDomainId().toString(),
queryMultiStructReq.getQueryStructReqs().get(0).getModelId().toString(),
queryMultiStructReq.generateCommandMd5());
boolean isCache = isCache(queryMultiStructReq);
QueryResultWithSchemaResp queryResultWithColumns;

17
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaService.java
View File

@@ -2,25 +2,26 @@ package com.tencent.supersonic.semantic.query.service;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import java.util.List;
public interface SchemaService {
List<DomainSchemaResp> fetchDomainSchema(DomainSchemaFilterReq filter, User user);
List<DomainResp> getDomainListForAdmin(User user);
List<DomainResp> getDomainListForViewer(User user);
List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq filter, User user);
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq, User user);
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
List<DomainResp> getDomainList(User user);
List<ModelResp> getModelList(User user, AuthType authType, Long domainId);
}

54
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java
View File

@@ -4,21 +4,24 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -31,33 +34,35 @@ import org.springframework.util.CollectionUtils;
public class SchemaServiceImpl implements SchemaService {
private final QueryService queryService;
private final DomainService domainService;
private final ModelService modelService;
private final DimensionService dimensionService;
private final MetricService metricService;
private final DomainService domainService;
public SchemaServiceImpl(QueryService queryService,
DomainService domainService,
ModelService modelService,
DimensionService dimensionService,
MetricService metricService) {
MetricService metricService,
DomainService domainService) {
this.queryService = queryService;
this.domainService = domainService;
this.modelService = modelService;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.domainService = domainService;
}
@Override
public List<DomainSchemaResp> fetchDomainSchema(DomainSchemaFilterReq filter, User user) {
List<DomainSchemaResp> domainSchemaDescList = domainService.fetchDomainSchema(filter, user);
public List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq filter, User user) {
List<ModelSchemaResp> domainSchemaDescList = modelService.fetchModelSchema(filter);
List<ItemUseResp> statInfos = queryService.getStatInfo(new ItemUseReq());
log.debug("statInfos:{}", statInfos);
fillCnt(domainSchemaDescList, statInfos);
return domainSchemaDescList;
}
private void fillCnt(List<DomainSchemaResp> domainSchemaDescList, List<ItemUseResp> statInfos) {
private void fillCnt(List<ModelSchemaResp> domainSchemaDescList, List<ItemUseResp> statInfos) {
Map<String, ItemUseResp> typeIdAndStatPair = statInfos.stream()
.collect(Collectors.toMap(
@@ -65,13 +70,13 @@ public class SchemaServiceImpl implements SchemaService {
itemUseInfo -> itemUseInfo,
(item1, item2) -> item1));
log.debug("typeIdAndStatPair:{}", typeIdAndStatPair);
for (DomainSchemaResp domainSchemaDesc : domainSchemaDescList) {
for (ModelSchemaResp domainSchemaDesc : domainSchemaDescList) {
fillDimCnt(domainSchemaDesc, typeIdAndStatPair);
fillMetricCnt(domainSchemaDesc, typeIdAndStatPair);
}
}
private void fillMetricCnt(DomainSchemaResp domainSchemaDesc, Map<String, ItemUseResp> typeIdAndStatPair) {
private void fillMetricCnt(ModelSchemaResp domainSchemaDesc, Map<String, ItemUseResp> typeIdAndStatPair) {
List<MetricSchemaResp> metrics = domainSchemaDesc.getMetrics();
if (CollectionUtils.isEmpty(domainSchemaDesc.getMetrics())) {
return;
@@ -88,7 +93,7 @@ public class SchemaServiceImpl implements SchemaService {
domainSchemaDesc.setMetrics(metrics);
}
private void fillDimCnt(DomainSchemaResp domainSchemaDesc, Map<String, ItemUseResp> typeIdAndStatPair) {
private void fillDimCnt(ModelSchemaResp domainSchemaDesc, Map<String, ItemUseResp> typeIdAndStatPair) {
List<DimSchemaResp> dimensions = domainSchemaDesc.getDimensions();
if (CollectionUtils.isEmpty(domainSchemaDesc.getDimensions())) {
return;
@@ -104,16 +109,6 @@ public class SchemaServiceImpl implements SchemaService {
domainSchemaDesc.setDimensions(dimensions);
}
@Override
public List<DomainResp> getDomainListForAdmin(User user) {
return domainService.getDomainListForAdmin(user.getName());
}
@Override
public List<DomainResp> getDomainListForViewer(User user) {
return domainService.getDomainListForViewer(user.getName());
}
@Override
public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionCmd, User user) {
return dimensionService.queryDimension(pageDimensionCmd);
@@ -123,4 +118,15 @@ public class SchemaServiceImpl implements SchemaService {
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricCmd, User user) {
return metricService.queryMetric(pageMetricCmd);
}
@Override
public List<DomainResp> getDomainList(User user) {
return domainService.getDomainListWithAdminAuth(user);
}
@Override
public List<ModelResp> getModelList(User user, AuthType authTypeEnum, Long domainId) {
return modelService.getModelListWithAuth(user.getName(), domainId, authTypeEnum);
}
}

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SemanticQueryEngineImpl.java
View File

@@ -36,8 +36,8 @@ public class SemanticQueryEngineImpl implements SemanticQueryEngine {
if (queryExecutor != null) {
queryResultWithColumns = queryExecutor.execute(catalog, queryStatement);
queryResultWithColumns.setSql(queryStatement.getSql());
if (queryStatement.getDomainId() > 0) {
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getDomainId());
if (queryStatement.getModelId() > 0) {
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelId());
}
}
return queryResultWithColumns;
@@ -46,7 +46,7 @@ public class SemanticQueryEngineImpl implements SemanticQueryEngine {
public QueryStatement plan(QueryStructReq queryStructCmd) throws Exception {
QueryStatement queryStatement = queryParser.logicSql(queryStructCmd);
queryUtils.checkSqlParse(queryStatement);
queryStatement.setDomainId(queryStructCmd.getDomainId());
queryStatement.setModelId(queryStructCmd.getModelId());
log.info("queryStatement:{}", queryStatement);
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStructCmd, queryStatement);

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java
View File

@@ -43,7 +43,7 @@ public class ComponentFactory {
}
public static List<QueryOptimizer> getQueryOptimizers() {
if(queryOptimizers.isEmpty()){
if (queryOptimizers.isEmpty()) {
initQueryOptimizer();
}
return queryOptimizers;
@@ -59,9 +59,11 @@ public class ComponentFactory {
public static void setSqlParser(SqlParser parser) {
sqlParser = parser;
}
private static void initQueryOptimizer() {
queryOptimizers.add(getBean("DetailQuery", DetailQuery.class));
}
private static void initSemanticConverter() {
semanticConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
semanticConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));

106
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -12,18 +12,23 @@ import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -35,9 +40,6 @@ import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
@@ -66,7 +68,7 @@ public class DataPermissionAOP {
@Autowired
private MetricService metricService;
@Autowired
private DomainService domainService;
private ModelService modelService;
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@@ -77,7 +79,7 @@ public class DataPermissionAOP {
@Around(value = "dataPermissionAOP()")
public Object around(ProceedingJoinPoint point) throws Throwable {
Object[] args = point.getArgs();
QueryStructReq queryStructCmd = (QueryStructReq) args[0];
QueryStructReq queryStructReq = (QueryStructReq) args[0];
User user = (User) args[1];
if (!permissionDataEnable) {
@@ -90,29 +92,29 @@ public class DataPermissionAOP {
}
// 1. determine whether the subject field is visible
doDomainVisible(user, queryStructCmd);
doDomainVisible(user, queryStructReq);
// 2. fetch data permission meta information
Long domainId = queryStructCmd.getDomainId();
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructCmd);
log.info("classId:{}, res4Privilege:{}", domainId, res4Privilege);
Long modelId = queryStructReq.getModelId();
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
Set<String> sensitiveResByDomain = getHighSensitiveColsByDomainId(domainId);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelId);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(res -> sensitiveResByDomain.contains(res)).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", domainId, sensitiveResReq);
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelId, sensitiveResReq);
// query user privilege info
HttpServletRequest request = (HttpServletRequest) args[2];
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, request, domainId, sensitiveResReq);
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, request, modelId, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, queryStructCmd.getDomainId());
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, queryStructReq.getModelId());
// 3.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructCmd, resAuthSet, sensitiveResReq);
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
// 4.row permission pre-filter
doRowPermission(queryStructCmd, authorizedResource);
doRowPermission(queryStructReq, authorizedResource);
// 5.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) point.proceed();
@@ -120,7 +122,7 @@ public class DataPermissionAOP {
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, domainId, authorizedResource);
return getQueryResultWithColumns(queryResultWithColumns, modelId, authorizedResource);
}
// 6.if the column has no permission, hit *
@@ -128,7 +130,7 @@ public class DataPermissionAOP {
.collect(Collectors.toSet());
QueryResultWithSchemaResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns,
need2Apply);
addPromptInfoInfo(domainId, queryResultAfterDesensitization, authorizedResource);
addPromptInfoInfo(modelId, queryResultAfterDesensitization, authorizedResource);
return queryResultAfterDesensitization;
@@ -136,27 +138,27 @@ public class DataPermissionAOP {
private void doDomainVisible(User user, QueryStructReq queryStructCmd) {
Boolean visible = true;
Long domainId = queryStructCmd.getDomainId();
List<DomainResp> classListForViewer = domainService.getDomainListForViewer(user.getName());
if (CollectionUtils.isEmpty(classListForViewer)) {
Long domainId = queryStructCmd.getModelId();
List<ModelResp> modelListVisible = modelService.getModelListWithAuth(user.getName(), null, AuthType.VISIBLE);
if (CollectionUtils.isEmpty(modelListVisible)) {
visible = false;
} else {
Map<Long, List<DomainResp>> id2domainDesc = classListForViewer.stream()
.collect(Collectors.groupingBy(classInfo -> classInfo.getId()));
Map<Long, List<ModelResp>> id2domainDesc = modelListVisible.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
if (!CollectionUtils.isEmpty(id2domainDesc) && !id2domainDesc.containsKey(domainId)) {
visible = false;
}
}
if (!visible) {
List<Long> domainIds = new ArrayList<>();
domainIds.add(domainId);
List<DomainResp> classInfos = domainService.getDomainList(domainIds);
if (CollectionUtils.isEmpty(classInfos)) {
List<Long> modelIds = new ArrayList<>();
modelIds.add(domainId);
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
if (CollectionUtils.isEmpty(modelInfos)) {
throw new InvalidArgumentException(
"invalid domainId:" + domainId + ", please contact admin for details");
}
String domainName = classInfos.get(0).getName();
String domainName = modelInfos.get(0).getName();
throw new InvalidPermissionException(
"You do not have domain:" + domainName + " permission, please contact admin for details");
@@ -176,12 +178,12 @@ public class DataPermissionAOP {
if (!CollectionUtils.isEmpty(filters)) {
log.debug("dimensionFilters:{}", filters);
List<Long> domainIds = new ArrayList<>();
domainIds.add(domainId);
List<DomainResp> classInfos = domainService.getDomainList(domainIds);
String classNameCn = "";
if (!CollectionUtils.isEmpty(classInfos)) {
classNameCn = classInfos.get(0).getName();
List<Long> modelIds = new ArrayList<>();
modelIds.add(domainId);
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = "";
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
List<String> exprList = new ArrayList<>();
@@ -196,7 +198,7 @@ public class DataPermissionAOP {
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(classNameCn, exprList, descList, message));
new QueryAuthorization(modelNameCn, exprList, descList, message));
log.info("queryResultWithColumns:{}", queryResultWithColumns);
}
}
@@ -240,7 +242,7 @@ public class DataPermissionAOP {
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getDomainId())))
cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getModelId())))
.forEach(col -> resAuthName.add(col.getName()));
}
@@ -256,17 +258,17 @@ public class DataPermissionAOP {
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setUser(user.getName());
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setDomainId(domainId + "");
queryAuthResReq.setModelId(domainId + "");
AuthorizedResourceResp authorizedResource = fetchAuthRes(request, queryAuthResReq);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId,
authorizedResource);
return authorizedResource;
}
private Set<String> getHighSensitiveColsByDomainId(Long domainId) {
private Set<String> getHighSensitiveColsByModelId(Long modelId) {
Set<String> highSensitiveCols = new HashSet<>();
List<DimensionResp> highSensitiveDimensions = dimensionService.getHighSensitiveDimension(domainId);
List<MetricResp> highSensitiveMetrics = metricService.getHighSensitiveMetric(domainId);
List<DimensionResp> highSensitiveDimensions = dimensionService.getHighSensitiveDimension(modelId);
List<MetricResp> highSensitiveMetrics = metricService.getHighSensitiveMetric(modelId);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
@@ -373,16 +375,16 @@ public class DataPermissionAOP {
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> domainIds = new ArrayList<>();
domainIds.add(queryStructCmd.getDomainId());
List<DomainResp> classInfos = domainService.getDomainList(domainIds);
String classNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(classInfos)) {
classNameCn = classInfos.get(0).getName();
List<Long> modelIds = new ArrayList<>();
modelIds.add(queryStructCmd.getModelId());
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryStructCmd.getDomainId());
String finalDomainNameCn = classNameCn;
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryStructCmd.getModelId());
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DateUtils.java
View File

@@ -11,7 +11,6 @@ import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
@@ -20,7 +19,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.StringJoiner;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

24
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -7,6 +7,12 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.aspectj.lang.ProceedingJoinPoint;
@@ -17,9 +23,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@Aspect
@Component
@Slf4j
@@ -43,7 +46,7 @@ public class DimValueAspect {
Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0];
Long domainId = queryStructReq.getDomainId();
Long domainId = queryStructReq.getModelId();
List<DimensionResp> dimensions = dimensionService.getDimensions(domainId);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = new ConcurrentHashMap<>();
@@ -60,7 +63,8 @@ public class DimValueAspect {
return queryResultWithColumns;
}
private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns, Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (!selectDimValueMap(queryResultWithColumns.getColumns(), dimAndTechNameAndBizNamePair)) {
return;
}
@@ -81,8 +85,10 @@ public class DimValueAspect {
}
}
private boolean selectDimValueMap(List<QueryColumn> columns, Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair) || CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)) {
private boolean selectDimValueMap(List<QueryColumn> columns,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair) || CollectionUtils.isEmpty(
dimAndTechNameAndBizNamePair)) {
return false;
}
@@ -111,7 +117,9 @@ public class DimValueAspect {
List<String> values = (List) value;
List<String> valuesNew = new ArrayList<>();
for (String valueSingle : values) {
boolean f = aliasPair.containsKey(valueSingle) ? valuesNew.add(aliasPair.get(valueSingle)) : valuesNew.add(valueSingle);
boolean f =
aliasPair.containsKey(valueSingle) ? valuesNew.add(aliasPair.get(valueSingle))
: valuesNew.add(valueSingle);
}
filter.setValue(valuesNew);
}

28
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryReqConverter.java
View File

@@ -1,19 +1,19 @@
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.util.calcite.SqlParseUtils;
import com.tencent.supersonic.common.util.calcite.SqlParserInfo;
import com.tencent.supersonic.common.util.jsqlparser.CCJSqlParserUtils;
import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -23,21 +23,21 @@ import org.springframework.util.CollectionUtils;
public class QueryReqConverter {
@Autowired
private DomainService domainService;
private ModelService domainService;
@Autowired
private SemanticQueryEngine parserService;
public QueryStatement convert(QueryDslReq databaseReq, List<DomainSchemaResp> domainSchemas) throws Exception {
public QueryStatement convert(QueryDslReq databaseReq, List<ModelSchemaResp> domainSchemas) throws Exception {
List<MetricTable> tables = new ArrayList<>();
MetricTable metricTable = new MetricTable();
String sql = databaseReq.getSql();
SqlParserInfo sqlParseInfo = SqlParseUtils.getSqlParseInfo(sql);
List<String> allFields = sqlParseInfo.getAllFields();
List<String> allFields = CCJSqlParserUtils.getAllFields(sql);
String tableName = CCJSqlParserUtils.getTableName(sql);
if (CollectionUtils.isEmpty(domainSchemas)) {
if (CollectionUtils.isEmpty(domainSchemas) || StringUtils.isEmpty(tableName)) {
return new QueryStatement();
}
@@ -50,21 +50,21 @@ public class QueryReqConverter {
.collect(Collectors.toSet());
metricTable.setMetrics(allFields.stream().filter(entry -> metrics.contains(entry.toLowerCase()))
.map(entry -> entry.toLowerCase()).collect(Collectors.toList()));
.map(String::toLowerCase).collect(Collectors.toList()));
Set<String> collect = allFields.stream().filter(entry -> dimensions.contains(entry.toLowerCase()))
.map(entry -> entry.toLowerCase()).collect(Collectors.toSet());
.map(String::toLowerCase).collect(Collectors.toSet());
for (String internalCol : QueryStructUtils.internalCols) {
if (sql.contains(internalCol)) {
collect.add(internalCol);
}
}
metricTable.setDimensions(new ArrayList<>(collect));
metricTable.setAlias(sqlParseInfo.getTableName().toLowerCase());
metricTable.setAlias(tableName.toLowerCase());
tables.add(metricTable);
ParseSqlReq result = new ParseSqlReq();
BeanUtils.copyProperties(databaseReq, result);
result.setRootPath(domainService.getDomainFullPath(databaseReq.getDomainId()));
result.setRootPath(domainService.getModelFullPathMap().get(databaseReq.getModelId()));
result.setTables(tables);
QueryStatement queryStatement = parserService.physicalSql(result);

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -1,17 +1,16 @@
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -20,7 +19,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
@@ -50,7 +48,7 @@ public class QueryStructUtils {
private List<Long> getDimensionIds(QueryStructReq queryStructCmd) {
List<Long> dimensionIds = new ArrayList<>();
List<DimensionResp> dimensions = catalog.getDimensions(queryStructCmd.getDomainId());
List<DimensionResp> dimensions = catalog.getDimensions(queryStructCmd.getModelId());
Map<String, List<DimensionResp>> pair = dimensions.stream()
.collect(Collectors.groupingBy(DimensionResp::getBizName));
for (String group : queryStructCmd.getGroups()) {
@@ -71,7 +69,7 @@ public class QueryStructUtils {
private List<Long> getMetricIds(QueryStructReq queryStructCmd) {
List<Long> metricIds = new ArrayList<>();
List<MetricResp> metrics = catalog.getMetrics(queryStructCmd.getDomainId());
List<MetricResp> metrics = catalog.getMetrics(queryStructCmd.getModelId());
Map<String, List<MetricResp>> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName));
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (pair.containsKey(agg.getColumn())) {

20
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java
View File

@@ -3,11 +3,11 @@ package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
@@ -64,11 +64,11 @@ public class QueryUtils {
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, Long domainId) {
List<MetricResp> metricDescList = catalog.getMetrics(domainId);
List<DimensionResp> dimensionDescList = catalog.getDimensions(domainId);
Map<String,MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a,(k1, k2)->k1));
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, Long modelId) {
List<MetricResp> metricDescList = catalog.getMetrics(modelId);
List<DimensionResp> dimensionDescList = catalog.getDimensions(modelId);
Map<String, MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, String> namePair = new HashMap<>();
Map<String, String> nameTypePair = new HashMap<>();
addSysTimeDimension(namePair, nameTypePair);
@@ -82,7 +82,7 @@ public class QueryUtils {
});
List<QueryColumn> columns = queryResultWithColumns.getColumns();
columns.forEach(column -> {
String nameEn = column.getNameEn();
String nameEn = column.getNameEn().toLowerCase();
if (nameEn.contains(JOIN_UNDERLINE)) {
nameEn = nameEn.split(JOIN_UNDERLINE)[1];
}
@@ -92,7 +92,7 @@ public class QueryUtils {
if (nameTypePair.containsKey(nameEn)) {
column.setShowType(nameTypePair.get(nameEn));
}
if(metricRespMap.containsKey(nameEn)){
if (metricRespMap.containsKey(nameEn)) {
column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType());
column.setDataFormat(metricRespMap.get(nameEn).getDataFormat());
}
@@ -113,7 +113,7 @@ public class QueryUtils {
namePair.putAll(metricNameFromAgg);
List<QueryColumn> columns = queryResultWithColumns.getColumns();
columns.forEach(column -> {
String nameEn = column.getNameEn();
String nameEn = column.getNameEn().toLowerCase();
if (nameEn.contains(JOIN_UNDERLINE)) {
nameEn = nameEn.split(JOIN_UNDERLINE)[1];
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java
View File

@@ -5,10 +5,10 @@ import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Criterion;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

5
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlGenerateUtils.java
View File

@@ -1,14 +1,15 @@
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Aggregator;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
@Component
@Slf4j

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java
View File

@@ -4,13 +4,13 @@ import com.alibaba.ttl.TransmittableThreadLocal;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeBackEnum;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.semantic.query.persistence.repository.StatRepository;
import java.util.ArrayList;
import java.util.List;
@@ -81,7 +81,7 @@ public class StatUtils {
try {
queryStatInfo.setTraceId(traceId)
.setClassId(queryStructCmd.getDomainId())
.setClassId(queryStructCmd.getModelId())
.setUser(user)
.setQueryType(QueryTypeEnum.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())

31
semantic/query/src/main/resources/mapper/StatMapper.xml
View File

@@ -5,10 +5,10 @@
<mapper namespace="com.tencent.supersonic.semantic.query.persistence.mapper.StatMapper">
<resultMap id="QueryStatDO"
type="com.tencent.supersonic.semantic.api.model.pojo.QueryStat">
type="com.tencent.supersonic.semantic.api.model.pojo.QueryStat">
<id column="id" property="id"/>
<result column="trace_id" property="traceId"/>
<result column="domain_id" property="domainId"/>
<result column="model_id" property="modelId"/>
<result column="user" property="user"/>
<result column="created_at" property="createdAt"/>
<result column="query_type" property="queryType"/>
@@ -40,17 +40,18 @@
<insert id="createRecord">
insert into s2_query_stat_info
(
trace_id, domain_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics, select_cols, agg_cols, filter_cols, group_by_cols,
order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key
)
values
(
#{traceId}, #{domainId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
#{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate}, #{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols},
#{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey}, #{resultCacheKey}
)
(trace_id, model_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5,
query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics,
select_cols, agg_cols, filter_cols, group_by_cols,
order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key)
values (#{traceId}, #{modelId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd},
#{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5},
#{queryEngine},
#{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate},
#{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols},
#{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey},
#{resultCacheKey})
</insert>
<select id="getStatInfo"
@@ -61,8 +62,8 @@
<if test="startTime != null">
and start_time >= #{startTime}
</if>
<if test="domainId != null">
and domain_id = #{domainId}
<if test="modelId != null">
and model_id = #{modelId}
</if>
<if test="metric != null">
and metrics like concat('%',#{metric},'%')

67
semantic/query/src/main/resources/sql.ddl/s2_query_stat_info.sql
View File

@@ -1,34 +1,35 @@
CREATE TABLE `s2_query_stat_info` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`trace_id` varchar(200) DEFAULT NULL COMMENT '查询标识',
`domain_id` bigint(20) DEFAULT NULL COMMENT '主题域ID',
`user` varchar(200) DEFAULT NULL COMMENT '执行sql的用户',
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`query_type` varchar(200) DEFAULT NULL COMMENT '查询对应的场景',
`query_type_back` int(10) DEFAULT '0' COMMENT '查询类型, 0-正常查询, 1-预刷类型',
`query_sql_cmd` mediumtext COMMENT '对应查询的struct',
`sql_cmd_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
`query_struct_cmd` mediumtext COMMENT '对应查询的struct',
`struct_cmd_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
`sql` mediumtext COMMENT '对应查询的sql',
`sql_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
`query_engine` varchar(20) DEFAULT NULL COMMENT '查询引擎',
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT '查询耗时',
`query_state` varchar(20) DEFAULT NULL COMMENT '查询最终状态',
`native_query` int(10) DEFAULT NULL COMMENT '1-明细查询,0-聚合查询',
`start_date` varchar(50) DEFAULT NULL COMMENT 'sql开始日期',
`end_date` varchar(50) DEFAULT NULL COMMENT 'sql结束日期',
`dimensions` mediumtext COMMENT 'sql 涉及的维度',
`metrics` mediumtext COMMENT 'sql 涉及的指标',
`select_cols` mediumtext COMMENT 'sql select部分涉及的标签',
`agg_cols` mediumtext COMMENT 'sql agg部分涉及的标签',
`filter_cols` mediumtext COMMENT 'sql where部分涉及的标签',
`group_by_cols` mediumtext COMMENT 'sql grouy by部分涉及的标签',
`order_by_cols` mediumtext COMMENT 'sql order by部分涉及的标签',
`use_result_cache` tinyint(1) DEFAULT '-1' COMMENT '是否命中sql缓存',
`use_sql_cache` tinyint(1) DEFAULT '-1' COMMENT '是否命中sql缓存',
`sql_cache_key` mediumtext COMMENT '缓存的key',
`result_cache_key` mediumtext COMMENT '缓存的key',
PRIMARY KEY (`id`),
KEY `domain_index` (`domain_id`)
CREATE TABLE `s2_query_stat_info`
(
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`trace_id` varchar(200) DEFAULT NULL COMMENT '查询标识',
`domain_id` bigint(20) DEFAULT NULL COMMENT '主题域ID',
`user` varchar(200) DEFAULT NULL COMMENT '执行sql的用户',
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`query_type` varchar(200) DEFAULT NULL COMMENT '查询对应的场景',
`query_type_back` int(10) DEFAULT '0' COMMENT '查询类型, 0-正常查询, 1-预刷类型',
`query_sql_cmd` mediumtext COMMENT '对应查询的struct',
`sql_cmd_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
`query_struct_cmd` mediumtext COMMENT '对应查询的struct',
`struct_cmd_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
`sql` mediumtext COMMENT '对应查询的sql',
`sql_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
`query_engine` varchar(20) DEFAULT NULL COMMENT '查询引擎',
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT '查询耗时',
`query_state` varchar(20) DEFAULT NULL COMMENT '查询最终状态',
`native_query` int(10) DEFAULT NULL COMMENT '1-明细查询,0-聚合查询',
`start_date` varchar(50) DEFAULT NULL COMMENT 'sql开始日期',
`end_date` varchar(50) DEFAULT NULL COMMENT 'sql结束日期',
`dimensions` mediumtext COMMENT 'sql 涉及的维度',
`metrics` mediumtext COMMENT 'sql 涉及的指标',
`select_cols` mediumtext COMMENT 'sql select部分涉及的标签',
`agg_cols` mediumtext COMMENT 'sql agg部分涉及的标签',
`filter_cols` mediumtext COMMENT 'sql where部分涉及的标签',
`group_by_cols` mediumtext COMMENT 'sql grouy by部分涉及的标签',
`order_by_cols` mediumtext COMMENT 'sql order by部分涉及的标签',
`use_result_cache` tinyint(1) DEFAULT '-1' COMMENT '是否命中sql缓存',
`use_sql_cache` tinyint(1) DEFAULT '-1' COMMENT '是否命中sql缓存',
`sql_cache_key` mediumtext COMMENT '缓存的key',
`result_cache_key` mediumtext COMMENT '缓存的key',
PRIMARY KEY (`id`),
KEY `domain_index` (`domain_id`)
) COMMENT='查询统计信息表'

52
semantic/query/src/main/resources/sql.ddl/s2_semantic_pasre_info.sql
View File

@@ -1,18 +1,46 @@
CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`trace_id` varchar(200) NOT NULL COMMENT '查询标识' ,
`domain_id` bigint(20) NOT NULL COMMENT '主题域ID',
CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info`
(
`id` bigint
(
20
) unsigned NOT NULL AUTO_INCREMENT,
`trace_id` varchar
(
200
) NOT NULL COMMENT '查询标识' ,
`domain_id` bigint
(
20
) NOT NULL COMMENT '主题域ID',
`dimensions` mediumtext COMMENT '查询相关的维度信息',
`metrics` mediumtext COMMENT '查询相关的指标信息',
`orders` mediumtext COMMENT '查询相关的排序信息',
`filters` mediumtext COMMENT '查询相关的过滤信息',
`date_info` mediumtext COMMENT '查询相关的日期信息',
`limit` bigint(20) NOT NULL COMMENT'查询相关的limit信息',
`native_query` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1-明细查询,0-聚合查询',
`limit` bigint
(
20
) NOT NULL COMMENT '查询相关的limit信息',
`native_query` tinyint
(
1
) NOT NULL DEFAULT '0' COMMENT '1-明细查询,0-聚合查询',
`sql` mediumtext COMMENT '解析后的sql',
`created_at` datetime NOT NULL COMMENT '创建时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
`status` int(10) NOT NULL COMMENT '运行状态',
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT 'sql解析耗时',
PRIMARY KEY (`id`)
)COMMENT='语义层sql解析信息表'
`created_at` datetime NOT NULL COMMENT '创建时间',
`created_by` varchar
(
100
) NOT NULL COMMENT '创建人',
`status` int
(
10
) NOT NULL COMMENT '运行状态',
`elapsed_ms` bigint
(
10
) DEFAULT NULL COMMENT 'sql解析耗时',
PRIMARY KEY
(
`id`
)
) COMMENT='语义层sql解析信息表'

4
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java
View File

@@ -1,5 +1,7 @@
package com.tencent.supersonic.semantic.query.domain.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
@@ -7,9 +9,7 @@ import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;