topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-24 10:27:28 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement) Move out the datasource and merge the datasource with the model, and adapt the chat module (#423)

Browse Source 
Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
jipeli
2023-11-27 11:05:24 +08:00
committed by GitHub
parent 0534053ff9
commit 27bb1b322e
190 changed files with 3900 additions and 10561 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java
View File

@@ -85,8 +85,9 @@ public class MaterializationQuery implements QueryOptimizer {
return;
}
try {
if (Objects.isNull(queryStructCmd) || Objects.isNull(queryStatement) || Objects.isNull(
queryStructCmd.getModelId()) || Objects.isNull(
if (Objects.isNull(queryStructCmd) || Objects.isNull(queryStatement)
|| CollectionUtils.isEmpty(queryStructCmd.getModelIds())
|| Objects.isNull(
queryStructCmd.getDateInfo())) {
return;
}
@@ -172,10 +173,11 @@ public class MaterializationQuery implements QueryOptimizer {
ImmutablePair<String, String> timeRange = queryStructUtils.getBeginEndTime(queryStructReq);
String start = timeRange.left;
String end = timeRange.right;
Long modelId = queryStructReq.getModelId();
//todo
Long modelId = 1L;
List<MaterializationResp> materializationResps = materializationConfService.getMaterializationByModel(modelId);
List<DimensionResp> dimensionResps = catalog.getDimensions(modelId);
List<MetricResp> metrics = catalog.getMetrics(modelId);
List<DimensionResp> dimensionResps = catalog.getDimensions(queryStructReq.getModelIds());
List<MetricResp> metrics = catalog.getMetrics(queryStructReq.getModelIds());
Set<String> fields = new HashSet<>();
if (Objects.nonNull(metricReq.getWhere()) && !metricReq.getWhere().isEmpty()) {

1
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java
View File

@@ -41,6 +41,7 @@ public class CalciteSqlParser implements SqlParser {
semanticSchema.setDatasource(semanticModel.getDatasourceMap());
semanticSchema.setDimension(semanticModel.getDimensionMap());
semanticSchema.setMetric(semanticModel.getMetrics());
semanticSchema.setJoinRelations(semanticModel.getJoinRelations());
return semanticSchema;
}
}

51
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java
View File

@@ -3,7 +3,9 @@ package com.tencent.supersonic.semantic.query.parser.calcite;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl;
@@ -17,12 +19,14 @@ import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -34,6 +38,7 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.EnableCaching;
@@ -57,20 +62,27 @@ public class SemanticSchemaManager {
public SemanticModel reload(String rootPath) {
SemanticModel semanticModel = new SemanticModel();
semanticModel.setRootPath(rootPath);
Map<Long, String> modelFullPathMap = catalog.getModelFullPath();
log.info("modelFullPathMap {}", modelFullPathMap);
Set<Long> modelIds = modelFullPathMap.entrySet().stream().filter(e -> e.getValue().startsWith(rootPath))
.map(Entry::getKey).collect(Collectors.toSet());
//Map<Long, String> modelFullPathMap = catalog.getModelFullPath();
//log.info("modelFullPathMap {}", modelFullPathMap);
//Set<Long> modelIds = modelFullPathMap.entrySet().stream().filter(e -> e.getValue().startsWith(rootPath))
// .map(Entry::getKey).collect(Collectors.toSet());
Set<Long> modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim()))
.collect(Collectors.toSet());
if (modelIds.isEmpty()) {
log.error("get modelIds empty {}", rootPath);
return semanticModel;
}
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DatasourceYamlTpl> datasourceYamlTpls = new ArrayList<>();
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls);
if (!datasourceYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = datasourceYamlTpls.stream().map(d -> getDatasource(d))
Map<Long, String> modelIdName = new HashMap<>();
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
if (!CollectionUtils.isEmpty(modelRelas)) {
semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(d -> getDatasource(d))
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
semanticModel.setDatasourceMap(dataSourceMap);
}
@@ -107,7 +119,7 @@ public class SemanticSchemaManager {
}
public static DataSource getDatasource(final DatasourceYamlTpl d) {
public static DataSource getDatasource(final DataModelYamlTpl d) {
DataSource datasource = DataSource.builder().sourceId(d.getSourceId()).sqlQuery(d.getSqlQuery())
.name(d.getName()).tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasures(d.getMeasures())).dimensions(getDimensions(d.getDimensions())).build();
@@ -200,6 +212,25 @@ public class SemanticSchemaManager {
return identifies;
}
private static List<JoinRelation> getJoinRelation(List<ModelRela> modelRelas, Map<Long, String> modelIdName) {
List<JoinRelation> joinRelations = new ArrayList<>();
modelRelas.stream().forEach(r -> {
if (modelIdName.containsKey(r.getFromModelId()) && modelIdName.containsKey(r.getToModelId())) {
JoinRelation joinRelation = JoinRelation.builder().left(modelIdName.get(r.getFromModelId()))
.right(modelIdName.get(r.getToModelId())).joinType(r.getJoinType()).build();
List<Triple<String, String, String>> conditions = new ArrayList<>();
r.getJoinConditions().stream().forEach(rr -> {
if (FilterOperatorEnum.isValueCompare(rr.getOperator())) {
conditions.add(Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField()));
}
});
joinRelation.setJoinCondition(conditions);
joinRelations.add(joinRelation);
}
});
return joinRelations;
}
public static void update(SemanticSchema schema, List<Metric> metric) throws Exception {
if (schema != null) {

29
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java
View File

@@ -15,18 +15,19 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.FilterRen
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.OutputRender;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.SourceRender;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Objects;
import java.util.Stack;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;
public class AggPlanner implements Planner {
private MetricReq metricCommand;
private MetricReq metricReq;
private SemanticSchema schema;
private SqlValidatorScope scope;
private Stack<TableView> dataSets = new Stack<>();
@@ -61,13 +62,13 @@ public class AggPlanner implements Planner {
while (it.hasNext()) {
Renderer renderer = it.next();
if (previous != null) {
previous.render(metricCommand, datasource, scope, schema, !isAgg);
previous.render(metricReq, datasource, scope, schema, !isAgg);
renderer.setTable(previous.builderAs(DataSourceNode.getNames(datasource) + "_" + String.valueOf(i)));
i++;
}
previous = renderer;
}
builders.getLast().render(metricCommand, datasource, scope, schema, !isAgg);
builders.getLast().render(metricReq, datasource, scope, schema, !isAgg);
parserNode = builders.getLast().builder();
@@ -75,7 +76,7 @@ public class AggPlanner implements Planner {
private List<DataSource> getMatchDataSource(SqlValidatorScope scope) throws Exception {
return DataSourceNode.getMatchDataSources(scope, schema, metricCommand);
return DataSourceNode.getMatchDataSources(scope, schema, metricReq);
}
private boolean getAgg(DataSource dataSource) {
@@ -85,7 +86,7 @@ public class AggPlanner implements Planner {
// default by dataSource time aggregation
if (Objects.nonNull(dataSource.getAggTime()) && !dataSource.getAggTime().equalsIgnoreCase(
Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) {
if (!metricCommand.isNativeQuery()) {
if (!metricReq.isNativeQuery()) {
return true;
}
}
@@ -95,15 +96,15 @@ public class AggPlanner implements Planner {
@Override
public void explain(QueryStatement queryStatement, AggOption aggOption) throws Exception {
this.metricCommand = queryStatement.getMetricReq();
if (metricCommand.getMetrics() == null) {
metricCommand.setMetrics(new ArrayList<>());
this.metricReq = queryStatement.getMetricReq();
if (metricReq.getMetrics() == null) {
metricReq.setMetrics(new ArrayList<>());
}
if (metricCommand.getDimensions() == null) {
metricCommand.setDimensions(new ArrayList<>());
if (metricReq.getDimensions() == null) {
metricReq.setDimensions(new ArrayList<>());
}
if (metricCommand.getLimit() == null) {
metricCommand.setLimit(0L);
if (metricReq.getLimit() == null) {
metricReq.setLimit(0L);
}
this.aggOption = aggOption;
// build a parse Node

17
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/JoinRelation.java Normal file
View File

@@ -0,0 +1,17 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.List;
import lombok.Builder;
import lombok.Data;
import org.apache.commons.lang3.tuple.Triple;
@Data
@Builder
public class JoinRelation {
private String left;
private String right;
private String joinType;
private List<Triple<String, String, String>> joinCondition;
}

1
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/SemanticModel.java
View File

@@ -14,4 +14,5 @@ public class SemanticModel {
private Map<String, DataSource> datasourceMap = new HashMap<>();
private Map<String, List<Dimension>> dimensionMap = new HashMap<>();
private List<Materialization> materializationList = new ArrayList<>();
private List<JoinRelation> joinRelations;
}

18
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java
View File

@@ -3,17 +3,19 @@ package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaVersion;
import org.apache.calcite.schema.Table;
import org.apache.calcite.schema.impl.AbstractSchema;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class SemanticSchema extends AbstractSchema {
private final String rootPath;
@@ -21,6 +23,8 @@ public class SemanticSchema extends AbstractSchema {
private SemanticModel semanticModel = new SemanticModel();
private List<JoinRelation> joinRelations;
private SemanticSchema(String rootPath, Map<String, Table> tableMap) {
this.rootPath = rootPath;
@@ -84,6 +88,14 @@ public class SemanticSchema extends AbstractSchema {
return semanticModel.getMaterializationList();
}
public void setJoinRelations(List<JoinRelation> joinRelations) {
this.joinRelations = joinRelations;
}
public List<JoinRelation> getJoinRelations() {
return joinRelations;
}
public static final class Builder {

123
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -1,13 +1,26 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.google.common.collect.Lists;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.extend.LateralViewExplodeNode;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -19,16 +32,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
@Slf4j
public class DataSourceNode extends SemanticNode {
@@ -159,21 +162,26 @@ public class DataSourceNode extends SemanticNode {
return dataSources;
}
// find all dataSource has the same identifiers
Set<String> baseIdentifiers = baseDataSource.getIdentifiers().stream().map(i -> i.getName())
.collect(Collectors.toSet());
if (baseIdentifiers.isEmpty()) {
throw new Exception("datasource error : " + baseDataSource.getName() + " miss identifier");
}
List<DataSource> linkDataSources = getLinkDataSources(baseIdentifiers, queryDimension, measures,
List<DataSource> linkDataSources = getLinkDataSourcesByJoinRelation(queryDimension, measures,
baseDataSource, schema);
if (linkDataSources.isEmpty()) {
throw new Exception(
String.format("not find the match datasource : dimension[%s],measure[%s]", queryDimension,
measures));
if (CollectionUtils.isEmpty(linkDataSources)) {
log.info("baseDataSource get by identifiers ");
Set<String> baseIdentifiers = baseDataSource.getIdentifiers().stream().map(i -> i.getName())
.collect(Collectors.toSet());
if (baseIdentifiers.isEmpty()) {
throw new Exception("datasource error : " + baseDataSource.getName() + " miss identifier");
}
linkDataSources = getLinkDataSources(baseIdentifiers, queryDimension, measures,
baseDataSource, schema);
if (linkDataSources.isEmpty()) {
throw new Exception(
String.format("not find the match datasource : dimension[%s],measure[%s]", queryDimension,
measures));
}
}
log.debug("linkDataSources {}", linkDataSources);
dataSources.addAll(linkDataSources);
return linkDataSources;
//dataSources.addAll(linkDataSources);
}
return dataSources;
@@ -208,6 +216,69 @@ public class DataSourceNode extends SemanticNode {
return isAllMatch;
}
private static List<DataSource> getLinkDataSourcesByJoinRelation(Set<String> queryDimension, List<String> measures,
DataSource baseDataSource, SemanticSchema schema) {
Set<String> linkDataSourceName = new HashSet<>();
List<DataSource> linkDataSources = new ArrayList<>();
Set<String> before = new HashSet<>();
before.add(baseDataSource.getName());
if (!CollectionUtils.isEmpty(schema.getJoinRelations())) {
for (JoinRelation joinRelation : schema.getJoinRelations()) {
if (!before.contains(joinRelation.getLeft()) && !before.contains(joinRelation.getRight())) {
continue;
}
boolean isMatch = false;
boolean isRight = before.contains(joinRelation.getLeft());
DataSource other = isRight ? schema.getDatasource().get(joinRelation.getRight())
: schema.getDatasource().get(joinRelation.getLeft());
if (!queryDimension.isEmpty()) {
Set<String> linkDimension = other.getDimensions().stream().map(dd -> dd.getName())
.collect(Collectors.toSet());
other.getIdentifiers().stream().forEach(i -> linkDimension.add(i.getName()));
linkDimension.retainAll(queryDimension);
if (!linkDimension.isEmpty()) {
isMatch = true;
}
}
Set<String> linkMeasure = other.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet());
linkMeasure.retainAll(measures);
if (!linkMeasure.isEmpty()) {
isMatch = true;
}
if (!isMatch && schema.getDimension().containsKey(other.getName())) {
Set<String> linkDimension = schema.getDimension().get(other.getName()).stream()
.map(dd -> dd.getName())
.collect(Collectors.toSet());
linkDimension.retainAll(queryDimension);
if (!linkDimension.isEmpty()) {
isMatch = true;
}
}
if (isMatch) {
linkDataSourceName.add(other.getName());
before.add(other.getName());
}
}
}
if (!CollectionUtils.isEmpty(linkDataSourceName)) {
Map<String, Long> orders = new HashMap<>();
linkDataSourceName.add(baseDataSource.getName());
orders.put(baseDataSource.getName(), 0L);
for (JoinRelation joinRelation : schema.getJoinRelations()) {
if (linkDataSourceName.contains(joinRelation.getLeft()) && linkDataSourceName.contains(
joinRelation.getRight())) {
orders.put(joinRelation.getLeft(), 0L);
orders.put(joinRelation.getRight(), 1L);
}
}
orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> {
linkDataSources.add(schema.getDatasource().get(d.getKey()));
});
}
return linkDataSources;
}
private static List<DataSource> getLinkDataSources(Set<String> baseIdentifiers,
Set<String> queryDimension,
List<String> measures,
@@ -258,6 +329,12 @@ public class DataSourceNode extends SemanticNode {
for (String linkName : linkDataSourceName) {
linkDataSources.add(schema.getDatasource().get(linkName));
}
return linkDataSources;
if (!CollectionUtils.isEmpty(linkDataSources)) {
List<DataSource> all = new ArrayList<>();
all.add(baseDataSource);
all.addAll(linkDataSources);
return all;
}
return Lists.newArrayList();
}
}

62
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/SemanticNode.java
View File

@@ -2,24 +2,21 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Optimization;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Optimization;
import org.apache.calcite.jdbc.CalciteSchema;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.sql.JoinType;
import org.apache.calcite.sql.SqlAsOperator;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlBinaryOperator;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.SqlWriterConfig;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
@@ -29,6 +26,15 @@ import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
public abstract class SemanticNode {
public static SqlNode parse(String expression, SqlValidatorScope scope) throws Exception {
@@ -130,6 +136,46 @@ public abstract class SemanticNode {
sqlValidator.validate(SqlParser.create(sql, SqlParser.Config.DEFAULT).parseStmt()), false, true).rel;
}
public static SqlBinaryOperator getBinaryOperator(String val) {
if (val.equals("=")) {
return SqlStdOperatorTable.EQUALS;
}
if (val.equals(">")) {
return SqlStdOperatorTable.GREATER_THAN;
}
if (val.equals(">=")) {
return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL;
}
if (val.equals("<")) {
return SqlStdOperatorTable.LESS_THAN;
}
if (val.equals("<=")) {
return SqlStdOperatorTable.LESS_THAN_OR_EQUAL;
}
if (val.equals("!=")) {
return SqlStdOperatorTable.NOT_EQUALS;
}
return SqlStdOperatorTable.EQUALS;
}
public static SqlLiteral getJoinSqlLiteral(String joinType) {
if (Objects.nonNull(joinType) && !joinType.isEmpty()) {
if (joinType.toLowerCase().contains(JoinType.INNER.lowerName)) {
return SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO);
}
if (joinType.toLowerCase().contains(JoinType.LEFT.lowerName)) {
return SqlLiteral.createSymbol(JoinType.LEFT, SqlParserPos.ZERO);
}
if (joinType.toLowerCase().contains(JoinType.RIGHT.lowerName)) {
return SqlLiteral.createSymbol(JoinType.RIGHT, SqlParserPos.ZERO);
}
if (joinType.toLowerCase().contains(JoinType.FULL.lowerName)) {
return SqlLiteral.createSymbol(JoinType.FULL, SqlParserPos.ZERO);
}
}
return SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO);
}
public void accept(Optimization optimization) {
optimization.visit(this);
}

116
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java
View File

@@ -5,10 +5,9 @@ import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
@@ -19,6 +18,18 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.IdentifyNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.JoinConditionType;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlJoin;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.util.CollectionUtils;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
@@ -28,21 +39,11 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.JoinConditionType;
import org.apache.calcite.sql.JoinType;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlJoin;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
@Slf4j
public class JoinRender extends Renderer {
@@ -51,7 +52,7 @@ public class JoinRender extends Renderer {
public void render(MetricReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere();
dataSources = getOrderSource(dataSources);
//dataSources = getOrderSource(dataSources);
Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>();
if (queryWhere != null && !queryWhere.isEmpty()) {
@@ -68,6 +69,7 @@ public class JoinRender extends Renderer {
TableView filterView = new TableView();
Map<String, SqlNode> innerSelect = new HashMap<>();
Set<String> filterDimension = new HashSet<>();
Map<String, String> beforeSources = new HashMap<>();
for (int i = 0; i < dataSources.size(); i++) {
final DataSource dataSource = dataSources.get(i);
@@ -112,18 +114,12 @@ public class JoinRender extends Renderer {
if (left == null) {
leftTable = tableView;
left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope));
beforeSources.put(dataSource.getName(), leftTable.getAlias());
continue;
}
left = new SqlJoin(
SqlParserPos.ZERO,
left,
SqlLiteral.createBoolean(false, SqlParserPos.ZERO),
SqlLiteral.createSymbol(JoinType.INNER, SqlParserPos.ZERO),
SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)),
SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO),
getCondition(leftTable, tableView, dataSource, schema, scope));
left = buildJoin(left, leftTable, tableView, beforeSources, dataSource, schema, scope);
leftTable = tableView;
beforeSources.put(dataSource.getName(), tableView.getAlias());
}
for (Map.Entry<String, SqlNode> entry : innerSelect.entrySet()) {
@@ -258,6 +254,76 @@ public class JoinRender extends Renderer {
return SemanticNode.getTable(tableView.getTable());
}
private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView, Map<String, String> before,
DataSource dataSource,
SemanticSchema schema, SqlValidatorScope scope)
throws Exception {
SqlNode condition = getCondition(leftTable, tableView, dataSource, schema, scope);
SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral("");
if (!TimePartType.ZIPPER.equals(leftTable.getDataSource().getTimePartType()) && !TimePartType.ZIPPER.equals(
tableView.getDataSource().getTimePartType())) {
JoinRelation matchJoinRelation = getMatchJoinRelation(before, tableView, schema);
if (!CollectionUtils.isEmpty(matchJoinRelation.getJoinCondition())) {
sqlLiteral = SemanticNode.getJoinSqlLiteral(matchJoinRelation.getJoinType());
condition = getCondition(matchJoinRelation, scope);
}
}
return new SqlJoin(
SqlParserPos.ZERO,
left,
SqlLiteral.createBoolean(false, SqlParserPos.ZERO),
sqlLiteral,
SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)),
SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO),
condition
);
}
private JoinRelation getMatchJoinRelation(Map<String, String> before, TableView tableView, SemanticSchema schema) {
JoinRelation matchJoinRelation = JoinRelation.builder().build();
if (!CollectionUtils.isEmpty(schema.getJoinRelations())) {
for (JoinRelation joinRelation : schema.getJoinRelations()) {
if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataSource().getName())
&& before.containsKey(joinRelation.getLeft())) {
matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream()
.map(r -> Triple.of(before.get(joinRelation.getLeft()) + "." + r.getLeft(),
r.getMiddle(), tableView.getAlias() + "." + r.getRight())).collect(
Collectors.toList()));
matchJoinRelation.setJoinType(joinRelation.getJoinType());
}
}
}
return matchJoinRelation;
}
private SqlNode getCondition(JoinRelation joinRelation,
SqlValidatorScope scope) throws Exception {
SqlNode condition = null;
for (Triple<String, String, String> con : joinRelation.getJoinCondition()) {
List<SqlNode> ons = new ArrayList<>();
ons.add(SemanticNode.parse(con.getLeft(), scope));
ons.add(SemanticNode.parse(con.getRight(), scope));
if (Objects.isNull(condition)) {
condition = new SqlBasicCall(
SemanticNode.getBinaryOperator(con.getMiddle()),
ons,
SqlParserPos.ZERO, null);
continue;
}
SqlNode addCondition = new SqlBasicCall(
SemanticNode.getBinaryOperator(con.getMiddle()),
ons,
SqlParserPos.ZERO, null);
condition = new SqlBasicCall(
SqlStdOperatorTable.AND,
new ArrayList<>(Arrays.asList(condition, addCondition)),
SqlParserPos.ZERO, null);
}
return condition;
}
private SqlNode getCondition(TableView left, TableView right, DataSource dataSource, SemanticSchema schema,
SqlValidatorScope scope) throws Exception {
if (TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType()) || TimePartType.ZIPPER.equals(
@@ -318,8 +384,7 @@ public class JoinRender extends Renderer {
});
int cnt = dataSources.size();
List<Map.Entry<String, List<Identify>>> dataSourceIdentifyList = dataSourceIdentifies.entrySet().stream()
.collect(
Collectors.toList());
.collect(Collectors.toList());
for (int i = 0; i < cnt; i++) {
for (int j = i + 1; j < cnt; j++) {
Set<String> primaries = IdentifyNode.getIdentifyNames(dataSourceIdentifyList.get(i).getValue(),
@@ -384,6 +449,7 @@ public class JoinRender extends Renderer {
orders.poll();
visited.put(id, false);
}
private void addZipperField(DataSource dataSource, List<String> fields) {
if (TimePartType.ZIPPER.equals(dataSource.getTimePartType())) {
dataSource.getDimensions().stream()

19
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java
View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
@@ -15,20 +16,20 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import com.tencent.supersonic.semantic.query.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Component("CalculateAggConverter")
@Slf4j
@@ -66,7 +67,7 @@ public class CalculateAggConverter implements SemanticConverter {
return generateRatioSqlCommand(queryStructCmd, engineTypeEnum, version);
}
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelId()));
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds()));
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
@@ -120,7 +121,7 @@ public class CalculateAggConverter implements SemanticConverter {
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelId());
DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelIds().get(0));
ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd,
EngineTypeEnum.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
sqlCommend.setSql(parseSqlReq.getSql());
@@ -150,7 +151,7 @@ public class CalculateAggConverter implements SemanticConverter {
throws Exception {
check(queryStructCmd);
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelId()));
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds()));
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java
View File

@@ -26,7 +26,7 @@ public class DefaultDimValueConverter implements SemanticConverter {
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd,
ParseSqlReq sqlCommend, MetricReq metricCommand) throws Exception {
List<DimensionResp> dimensionResps = catalog.getDimensions(queryStructCmd.getModelId());
List<DimensionResp> dimensionResps = catalog.getDimensions(queryStructCmd.getModelIds());
//dimension which has default values
dimensionResps = dimensionResps.stream()
.filter(dimensionResp -> !CollectionUtils.isEmpty(dimensionResp.getDefaultValues()))

5
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java
View File

@@ -33,9 +33,8 @@ public class MetricCheckConverter implements SemanticConverter {
@Override
public void converter(Catalog catalog, QueryStructReq queryStructReq, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
Long modelId = queryStructReq.getModelId();
List<MetricResp> metricResps = catalog.getMetrics(modelId);
List<DimensionResp> dimensionResps = catalog.getDimensions(modelId);
List<MetricResp> metricResps = catalog.getMetrics(queryStructReq.getModelIds());
List<DimensionResp> dimensionResps = catalog.getDimensions(queryStructReq.getModelIds());
Map<Long, DimensionResp> dimensionMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getId, d -> d));
List<String> metricBizNames = queryStructReq.getMetrics();

135
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java
View File

@@ -1,135 +0,0 @@
package com.tencent.supersonic.semantic.query.parser.convert;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.api.model.pojo.Identify;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@Component("MultiSourceJoin")
public class MultiSourceJoin implements SemanticConverter {
private final Catalog catalog;
public MultiSourceJoin(Catalog catalog) {
this.catalog = catalog;
}
public void buildJoinPrefix(QueryStructReq queryStructCmd) {
List<String> groups = queryStructCmd.getGroups();
List<Aggregator> aggregators = queryStructCmd.getAggregators();
List<Filter> filters = queryStructCmd.getOriginalFilter();
List<String> fields = Lists.newArrayList();
fields.addAll(groups);
fields.addAll(filters.stream().map(Filter::getBizName).collect(Collectors.toList()));
if (CollectionUtils.isEmpty(groups) || CollectionUtils.isEmpty(aggregators)) {
return;
}
Long modelId = queryStructCmd.getModelId();
List<String> aggs = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList());
Map<String, DimensionResp> dimensionMap = catalog.getDimensions(modelId).stream()
.filter(dimensionDesc -> fields.contains(dimensionDesc.getBizName()))
.collect(Collectors.toMap(DimensionResp::getBizName, dimensionDesc -> dimensionDesc));
List<MetricResp> metricDescList = catalog.getMetrics(modelId).stream()
.filter(metricDesc -> aggs.contains(metricDesc.getBizName()))
.collect(Collectors.toList());
Map<Long, DatasourceResp> datasourceMap = catalog.getDatasourceList(modelId)
.stream().collect(Collectors.toMap(DatasourceResp::getId, datasource -> datasource));
//check groups filters and aggs is in same datasource
if (!isInSameDatasource(new ArrayList<>(dimensionMap.values()), metricDescList)) {
List<String> groupsWithPrefix = Lists.newArrayList();
for (String group : groups) {
DimensionResp dimensionDesc = dimensionMap.get(group);
if (dimensionDesc == null) {
groupsWithPrefix.add(group);
continue;
}
String joinKeyName = getJoinKey(datasourceMap, dimensionDesc.getDatasourceId());
if (joinKeyName.equalsIgnoreCase(group)) {
groupsWithPrefix.add(group);
} else {
String groupWithPrefix = String.format("%s__%s", joinKeyName, group);
groupsWithPrefix.add(groupWithPrefix);
}
}
List<Filter> filtersWithPrefix = Lists.newArrayList();
for (Filter filter : filters) {
DimensionResp dimensionDesc = dimensionMap.get(filter.getBizName());
if (dimensionDesc == null) {
filtersWithPrefix.add(filter);
continue;
}
String joinKeyName = getJoinKey(datasourceMap, dimensionDesc.getDatasourceId());
if (joinKeyName.equalsIgnoreCase(filter.getBizName())) {
filtersWithPrefix.add(filter);
} else {
String filterWithPrefix = String.format("%s__%s", joinKeyName, filter.getBizName());
filter.setBizName(filterWithPrefix);
filtersWithPrefix.add(filter);
}
}
queryStructCmd.setGroups(groupsWithPrefix);
queryStructCmd.setDimensionFilters(filtersWithPrefix);
}
}
private String getJoinKey(Map<Long, DatasourceResp> datasourceMap, Long datasourceId) {
DatasourceResp datasourceDesc = datasourceMap.get(datasourceId);
List<Identify> identifies = datasourceDesc.getDatasourceDetail().getIdentifiers();
Optional<Identify> identifyOptional = identifies.stream()
.filter(identify -> identify.getType().equalsIgnoreCase("primary")).findFirst();
if (identifyOptional.isPresent()) {
return identifyOptional.get().getBizName();
}
return "";
}
private boolean isInSameDatasource(List<DimensionResp> dimensionDescs, List<MetricResp> metricDescs) {
Set<Long> datasourceIdSet = Sets.newHashSet();
datasourceIdSet.addAll(dimensionDescs.stream().map(DimensionResp::getDatasourceId).filter(Objects::nonNull)
.collect(Collectors.toSet()));
datasourceIdSet.addAll(
metricDescs.stream().flatMap(metricDesc -> metricDesc.getTypeParams().getMeasures().stream())
.map(Measure::getDatasourceId).filter(Objects::nonNull).collect(Collectors.toList()));
log.info("[multi source join] datasource id:{}", datasourceIdSet);
return datasourceIdSet.size() <= 1;
}
@Override
public boolean accept(QueryStructReq queryStructCmd) {
return true;
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
buildJoinPrefix(queryStructCmd);
}
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java
View File

@@ -19,7 +19,6 @@ import org.springframework.util.CollectionUtils;
public class ParserDefaultConverter implements SemanticConverter {
private final CalculateAggConverter calculateCoverterAgg;
private final QueryStructUtils queryStructUtils;
@@ -57,20 +56,21 @@ public class ParserDefaultConverter implements SemanticConverter {
sqlCommend.setVariables(queryStructCmd.getParams().stream()
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
sqlCommend.setLimit(queryStructCmd.getLimit());
String rootPath = catalog.getModelFullPath(queryStructCmd.getModelId());
String rootPath = catalog.getModelFullPath(queryStructCmd.getModelIds());
sqlCommend.setRootPath(rootPath);
// todo tmp delete
// support detail query
if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
String internalMetricName = queryStructUtils.generateInternalMetricName(
queryStructCmd.getModelId(), queryStructCmd.getGroups());
sqlCommend.getMetrics().add(internalMetricName);
for (Long modelId : queryStructCmd.getModelIds()) {
String internalMetricName = queryStructUtils.generateInternalMetricName(
modelId, queryStructCmd.getGroups());
sqlCommend.getMetrics().add(internalMetricName);
}
}
return sqlCommend;
}
}

84
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -26,15 +26,6 @@ import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -43,6 +34,15 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
@Slf4j
public class QueryReqConverter {
@@ -57,13 +57,13 @@ public class QueryReqConverter {
@Autowired
private Catalog catalog;
public QueryStatement convert(QueryS2SQLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception {
public QueryStatement convert(QueryS2SQLReq databaseReq, List<ModelSchemaResp> modelSchemaResps) throws Exception {
if (Objects.isNull(modelSchemaResp)) {
if (CollectionUtils.isEmpty(modelSchemaResps)) {
return new QueryStatement();
}
//1.convert name to bizName
convertNameToBizName(databaseReq, modelSchemaResp);
convertNameToBizName(databaseReq, modelSchemaResps);
//2.functionName corrector
functionNameCorrector(databaseReq);
//3.correct tableName
@@ -75,21 +75,23 @@ public class QueryReqConverter {
}
//4.build MetricTables
List<String> allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql());
List<String> metrics = getMetrics(modelSchemaResp, allFields);
List<String> metrics = getMetrics(modelSchemaResps, allFields);
QueryStructReq queryStructCmd = new QueryStructReq();
MetricTable metricTable = new MetricTable();
metricTable.setMetrics(metrics);
Set<String> dimensions = getDimensions(modelSchemaResp, allFields);
Set<String> dimensions = getDimensions(modelSchemaResps, allFields);
metricTable.setDimensions(new ArrayList<>(dimensions));
metricTable.setAlias(tableName.toLowerCase());
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>(Arrays.asList(
queryStructUtils.generateInternalMetricName(databaseReq.getModelId(),
metricTable.getDimensions()))));
metricTable.setMetrics(new ArrayList<>());
for (Long modelId : databaseReq.getModelIds()) {
metricTable.getMetrics().add(queryStructUtils.generateInternalMetricName(modelId,
metricTable.getDimensions()));
}
} else {
queryStructCmd.setAggregators(
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
@@ -102,9 +104,10 @@ public class QueryReqConverter {
//4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq();
BeanUtils.copyProperties(databaseReq, result);
result.setRootPath(domainService.getModelFullPathMap().get(databaseReq.getModelId()));
result.setRootPath(catalog.getModelFullPath(databaseReq.getModelIds()));
result.setTables(tables);
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId());
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0));
if (!queryStructUtils.isSupportWith(EngineTypeEnum.valueOf(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
@@ -112,7 +115,7 @@ public class QueryReqConverter {
}
//5.physicalSql by ParseSqlReq
queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql()));
queryStructCmd.setModelId(databaseReq.getModelId());
queryStructCmd.setModelIds(databaseReq.getModelIds().stream().collect(Collectors.toSet()));
queryStructCmd.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd);
QueryStatement queryStatement = parserService.physicalSql(queryStructCmd, result);
@@ -132,8 +135,8 @@ public class QueryReqConverter {
return AggOption.DEFAULT;
}
private void convertNameToBizName(QueryS2SQLReq databaseReq, ModelSchemaResp modelSchemaResp) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp);
private void convertNameToBizName(QueryS2SQLReq databaseReq, List<ModelSchemaResp> modelSchemaResps) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResps);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
@@ -141,26 +144,28 @@ public class QueryReqConverter {
databaseReq.setSql(replaceFields);
}
private Set<String> getDimensions(ModelSchemaResp modelSchemaResp, List<String> allFields) {
Map<String, String> dimensionLowerToNameMap = modelSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry.getBizName()));
private Set<String> getDimensions(List<ModelSchemaResp> modelSchemaResps, List<String> allFields) {
Map<String, String> dimensionLowerToNameMap = modelSchemaResps.stream().flatMap(modelSchemaResp
-> modelSchemaResp.getDimensions().stream())
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName));
Map<String, String> internalLowerToNameMap = QueryStructUtils.internalCols.stream()
.collect(Collectors.toMap(a -> a.toLowerCase(), a -> a));
.collect(Collectors.toMap(String::toLowerCase, a -> a));
dimensionLowerToNameMap.putAll(internalLowerToNameMap);
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet());
}
private List<String> getMetrics(ModelSchemaResp modelSchemaResp, List<String> allFields) {
Map<String, String> metricLowerToNameMap = modelSchemaResp.getMetrics().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry.getBizName()));
private List<String> getMetrics(List<ModelSchemaResp> modelSchemaResps, List<String> allFields) {
Map<String, String> metricLowerToNameMap = modelSchemaResps.stream()
.flatMap(modelSchemaResp -> modelSchemaResp.getMetrics().stream())
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName));
return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
}
private void functionNameCorrector(QueryS2SQLReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId());
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0));
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
}
@@ -174,16 +179,17 @@ public class QueryReqConverter {
}
}
protected Map<String, String> getFieldNameToBizNameMap(ModelSchemaResp modelSchemaResp) {
protected Map<String, String> getFieldNameToBizNameMap(List<ModelSchemaResp> modelSchemaResps) {
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = modelSchemaResp.getDimensions().stream()
Map<String, String> dimensionResults = modelSchemaResps.stream().flatMap(modelSchemaResp
-> modelSchemaResp.getDimensions().stream())
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1));
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
Map<String, String> metricResults = modelSchemaResp.getMetrics().stream()
Map<String, String> metricResults = modelSchemaResps.stream().flatMap(modelSchemaResp
-> modelSchemaResp.getMetrics().stream())
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1));
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
@@ -204,8 +210,10 @@ public class QueryReqConverter {
}
public void correctTableName(QueryS2SQLReq databaseReq) {
String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(),
Constants.TABLE_PREFIX + databaseReq.getModelId());
String sql = databaseReq.getSql();
for (Long modelId : databaseReq.getModelIds()) {
sql = SqlParserReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + modelId);
}
databaseReq.setSql(sql);
}

5
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/pojo/QueryStatement.java
View File

@@ -2,14 +2,15 @@ package com.tencent.supersonic.semantic.query.persistence.pojo;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import java.util.List;
import lombok.Data;
import org.apache.commons.lang3.tuple.ImmutablePair;
import java.util.List;
@Data
public class QueryStatement {
private Long modelId = 0L;
private List<Long> modelIds;
private String sql = "";
private String sourceId = "";
private String errMsg = "";

23
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/SchemaController.java
View File

@@ -9,23 +9,24 @@ import com.tencent.supersonic.semantic.api.materialization.response.Materializat
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.query.service.MaterializationService;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/schema")

60
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
@@ -16,7 +16,6 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
@@ -26,20 +25,18 @@ import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.util.Sets;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@@ -57,64 +54,62 @@ public class AuthCommonService {
@Autowired
private ModelService modelService;
public boolean doModelAdmin(User user, Long modelId) {
public boolean doModelAdmin(User user, List<Long> modelIds) {
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
} else {
Map<Long, List<ModelResp>> id2modelResp = modelListAdmin.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
return !CollectionUtils.isEmpty(id2modelResp) && id2modelResp.containsKey(modelId);
Set<Long> modelAdmins = modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet());
return !CollectionUtils.isEmpty(modelAdmins) && modelAdmins.containsAll(modelIds);
}
}
public void doModelVisible(User user, Long modelId) {
public void doModelVisible(User user, List<Long> modelIds) {
Boolean visible = true;
List<ModelResp> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE);
if (CollectionUtils.isEmpty(modelListVisible)) {
visible = false;
} else {
Map<Long, List<ModelResp>> id2domainDesc = modelListVisible.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
if (!CollectionUtils.isEmpty(id2domainDesc) && !id2domainDesc.containsKey(modelId)) {
Set<Long> modelVisibles = modelListVisible.stream().map(ModelResp::getId).collect(Collectors.toSet());
if (!CollectionUtils.isEmpty(modelVisibles) && !modelVisibles.containsAll(modelIds)) {
visible = false;
}
}
if (!visible) {
ModelResp modelResp = modelService.getModel(modelId);
ModelResp modelResp = modelService.getModel(modelIds.get(0));
String modelName = modelResp.getName();
List<String> admins = modelService.getModelAdmin(modelResp.getId());
String message = String.format("您没有主题域[%s]权限，请联系管理员%s开通", modelName, admins);
String message = String.format("您没有模型[%s]权限，请联系管理员%s开通", modelName, admins);
throw new InvalidPermissionException(message);
}
}
public Set<String> getHighSensitiveColsByModelId(Long modelId) {
public Set<String> getHighSensitiveColsByModelId(List<Long> modelIds) {
Set<String> highSensitiveCols = new HashSet<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
metaFilter.setModelIds(modelIds);
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
List<DimensionResp> highSensitiveDimensions = dimensionService.getDimensions(metaFilter);
List<MetricResp> highSensitiveMetrics = metricService.getMetrics(metaFilter);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName()));
highSensitiveDimensions.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
if (!CollectionUtils.isEmpty(highSensitiveMetrics)) {
highSensitiveMetrics.stream().forEach(metric -> highSensitiveCols.add(metric.getBizName()));
highSensitiveMetrics.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
}
return highSensitiveCols;
}
public AuthorizedResourceResp getAuthorizedResource(User user, Long modelId,
public AuthorizedResourceResp getAuthorizedResource(User user, List<Long> modelIds,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelId, res)));
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelIds.get(0), res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelId(modelId);
queryAuthResReq.setModelIds(modelIds);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelId,
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelIds,
authorizedResource);
return authorizedResource;
}
@@ -124,13 +119,13 @@ public class AuthCommonService {
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) {
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, List<Long> modelIds) {
Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> modelId.equals(col.getModelId()))
cols.stream().filter(col -> modelIds.contains(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName()));
}
@@ -148,8 +143,9 @@ public class AuthCommonService {
}
public QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns,
Long domainId, AuthorizedResourceResp authResource) {
addPromptInfoInfo(domainId, resultWithColumns, authResource, Sets.newHashSet());
List<Long> modelIds,
AuthorizedResourceResp authResource) {
addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
@@ -245,20 +241,20 @@ public class AuthCommonService {
return queryResultWithColumns;
}
public void addPromptInfoInfo(Long modelId, QueryResultWithSchemaResp queryResultWithColumns,
public void addPromptInfoInfo(List<Long> modelIds, QueryResultWithSchemaResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
return;
}
List<String> admins = modelService.getModelAdmin(modelId);
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
if (!CollectionUtils.isEmpty(need2Apply)) {
String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins);
queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
}
if (!CollectionUtils.isEmpty(filters)) {
log.debug("dimensionFilters:{}", filters);
ModelResp modelResp = modelService.getModel(modelId);
ModelResp modelResp = modelService.getModel(modelIds.get(0));
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
filters.stream().forEach(filter -> {

17
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/MaterializationServiceImpl.java
View File

@@ -12,10 +12,13 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.materialization.domain.MaterializationConfService;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -24,9 +27,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service("MaterializationService")
@Slf4j
@@ -34,19 +34,17 @@ public class MaterializationServiceImpl implements MaterializationService {
protected final MaterializationConfService materializationConfService;
protected final ModelService modelService;
protected final DatasourceService datasourceService;
protected final Catalog catalog;
protected final QueryStructUtils queryStructUtils;
protected final QueryService queryService;
public MaterializationServiceImpl(
MaterializationConfService materializationConfService,
ModelService modelService, DatasourceService datasourceService,
ModelService modelService,
Catalog catalog, QueryStructUtils queryStructUtils,
QueryService queryService) {
this.materializationConfService = materializationConfService;
this.modelService = modelService;
this.datasourceService = datasourceService;
this.catalog = catalog;
this.queryStructUtils = queryStructUtils;
this.queryService = queryService;
@@ -72,11 +70,12 @@ public class MaterializationServiceImpl implements MaterializationService {
ModelSchemaFilterReq modelFilter = new ModelSchemaFilterReq();
modelFilter.setModelIds(Arrays.asList(materializationSourceResp.getModelId()));
List<ModelSchemaResp> modelSchemaRespList = modelService.fetchModelSchema(modelFilter);
List<MeasureResp> measureRespList = datasourceService.getMeasureListOfModel(
//todo
List<MeasureResp> measureRespList = modelService.getMeasureListOfModel(
Lists.newArrayList(materializationSourceResp.getModelId()));
modelSchemaRespList.stream().forEach(m -> {
m.getDimensions().stream()
.filter(mm -> mm.getDatasourceId().equals(materializationSourceReq.getDataSourceId())
.filter(mm -> mm.getModelId().equals(materializationSourceReq.getDataSourceId())
&& materializationSourceResp.getDimensions().keySet().contains(mm.getId())
).forEach(mm -> {
dimensionFields.add(mm.getBizName());

50
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -4,42 +4,43 @@ import com.google.common.cache.CacheBuilder;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.QueryUtils;
import com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation;
import com.tencent.supersonic.semantic.query.utils.StatUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
@@ -75,11 +76,11 @@ public class QueryServiceImpl implements QueryService {
@Override
@S2SQLPermissionAnnotation
@SneakyThrows
public Object queryBySql(QueryS2SQLReq querySqlCmd, User user) {
statUtils.initStatInfo(querySqlCmd, user);
public Object queryBySql(QueryS2SQLReq queryS2SQLReq, User user) {
statUtils.initStatInfo(queryS2SQLReq, user);
QueryStatement queryStatement = new QueryStatement();
try {
queryStatement = convertToQueryStatement(querySqlCmd, user);
queryStatement = convertToQueryStatement(queryS2SQLReq, user);
} catch (Exception e) {
log.info("convertToQueryStatement has a exception:{}", e.toString());
}
@@ -95,18 +96,11 @@ public class QueryServiceImpl implements QueryService {
private QueryStatement convertToQueryStatement(QueryS2SQLReq querySqlCmd, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = new ArrayList<>();
modelIds.add(querySqlCmd.getModelId());
filter.setModelIds(modelIds);
filter.setModelIds(querySqlCmd.getModelIds());
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> domainSchemas = schemaService.fetchModelSchema(filter, user);
ModelSchemaResp domainSchema = null;
if (CollectionUtils.isNotEmpty(domainSchemas)) {
domainSchema = domainSchemas.get(0);
}
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchema);
queryStatement.setModelId(querySqlCmd.getModelId());
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, modelSchemaResps);
queryStatement.setModelIds(querySqlCmd.getModelIds());
return queryStatement;
}
@@ -116,7 +110,7 @@ public class QueryServiceImpl implements QueryService {
log.info("[queryStructCmd:{}]", queryStructCmd);
try {
statUtils.initStatInfo(queryStructCmd, user);
String cacheKey = cacheUtils.generateCacheKey(queryStructCmd.getModelId().toString(),
String cacheKey = cacheUtils.generateCacheKey(getKeyByModelIds(queryStructCmd.getModelIds()),
queryStructCmd.generateCommandMd5());
handleGlobalCacheDisable(queryStructCmd);
boolean isCache = isCache(queryStructCmd);
@@ -160,7 +154,7 @@ public class QueryServiceImpl implements QueryService {
throws Exception {
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
String cacheKey = cacheUtils.generateCacheKey(
queryMultiStructReq.getQueryStructReqs().get(0).getModelId().toString(),
getKeyByModelIds(queryMultiStructReq.getQueryStructReqs().get(0).getModelIds()),
queryMultiStructReq.generateCommandMd5());
boolean isCache = isCache(queryMultiStructReq);
QueryResultWithSchemaResp queryResultWithColumns;
@@ -323,5 +317,9 @@ public class QueryServiceImpl implements QueryService {
return queryStructReq;
}
private String getKeyByModelIds(List<Long> modelIds) {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaService.java
View File

@@ -6,11 +6,11 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import java.util.List;

25
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java
View File

@@ -1,34 +1,35 @@
package com.tencent.supersonic.semantic.query.service;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
@Slf4j
@Service
public class SchemaServiceImpl implements SchemaService {

7
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SemanticQueryEngineImpl.java
View File

@@ -13,6 +13,7 @@ import com.tencent.supersonic.semantic.query.utils.ComponentFactory;
import com.tencent.supersonic.semantic.query.utils.QueryUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@Component
@@ -36,8 +37,8 @@ public class SemanticQueryEngineImpl implements SemanticQueryEngine {
if (queryExecutor != null) {
queryResultWithColumns = queryExecutor.execute(catalog, queryStatement);
queryResultWithColumns.setSql(queryStatement.getSql());
if (queryStatement.getModelId() > 0) {
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelId());
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelIds());
}
}
return queryResultWithColumns;
@@ -46,7 +47,7 @@ public class SemanticQueryEngineImpl implements SemanticQueryEngine {
public QueryStatement plan(QueryStructReq queryStructCmd) throws Exception {
QueryStatement queryStatement = queryParser.logicSql(queryStructCmd);
queryUtils.checkSqlParse(queryStatement);
queryStatement.setModelId(queryStructCmd.getModelId());
queryStatement.setModelIds(queryStructCmd.getModelIds());
log.info("queryStatement:{}", queryStatement);
return optimize(queryStructCmd, queryStatement);
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java
View File

@@ -12,7 +12,6 @@ import com.tencent.supersonic.semantic.query.parser.calcite.CalciteSqlParser;
import com.tencent.supersonic.semantic.query.parser.convert.CalculateAggConverter;
import com.tencent.supersonic.semantic.query.parser.convert.DefaultDimValueConverter;
import com.tencent.supersonic.semantic.query.parser.convert.MetricCheckConverter;
import com.tencent.supersonic.semantic.query.parser.convert.MultiSourceJoin;
import com.tencent.supersonic.semantic.query.parser.convert.ParserDefaultConverter;
import java.util.ArrayList;
import java.util.HashMap;
@@ -83,7 +82,6 @@ public class ComponentFactory {
semanticConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
semanticConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
semanticConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
semanticConverters.add(getBean("MultiSourceJoin", MultiSourceJoin.class));
}
private static void initQueryExecutors() {

315
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -1,35 +1,32 @@
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import java.text.SimpleDateFormat;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.query.service.AuthCommonService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -37,40 +34,21 @@ import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.assertj.core.util.Sets;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Component
@Aspect
@Slf4j
public class DataPermissionAOP {
private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat(
new SimpleDateFormat(Constants.DAY_FORMAT));
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private AuthService authService;
@Autowired
private DimensionService dimensionService;
@Autowired
private MetricService metricService;
@Autowired
private ModelService modelService;
@Autowired
private AuthCommonService authCommonService;
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@@ -93,27 +71,29 @@ public class DataPermissionAOP {
throw new RuntimeException("lease provide user information");
}
//1. determine whether admin of the model
if (doModelAdmin(user, queryStructReq)) {
if (authCommonService.doModelAdmin(user, queryStructReq.getModelIds())) {
return point.proceed();
}
// 2. determine whether the subject field is visible
doModelVisible(user, queryStructReq);
authCommonService.doModelVisible(user, queryStructReq.getModelIds());
// 3. fetch data permission meta information
Long modelId = queryStructReq.getModelId();
List<Long> modelIds = queryStructReq.getModelIds();
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelId);
Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelIds);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelId, sensitiveResReq);
log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelId, sensitiveResReq);
AuthorizedResourceResp authorizedResource = authCommonService.getAuthorizedResource(user,
modelIds, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, queryStructReq.getModelId());
Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource,
queryStructReq.getModelIds());
// 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
@@ -127,115 +107,20 @@ public class DataPermissionAOP {
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelId, authorizedResource);
return authCommonService.getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
}
// 6.if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
QueryResultWithSchemaResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns,
need2Apply);
addPromptInfoInfo(modelId, queryResultAfterDesensitization, authorizedResource, need2Apply);
QueryResultWithSchemaResp queryResultAfterDesensitization =
authCommonService.desensitizationData(queryResultWithColumns, need2Apply);
authCommonService.addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
private boolean doModelAdmin(User user, QueryStructReq queryStructReq) {
Long modelId = queryStructReq.getModelId();
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
} else {
Map<Long, List<ModelResp>> id2modelResp = modelListAdmin.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
return !CollectionUtils.isEmpty(id2modelResp) && id2modelResp.containsKey(modelId);
}
}
private void doModelVisible(User user, QueryStructReq queryStructReq) {
Boolean visible = true;
Long modelId = queryStructReq.getModelId();
List<ModelResp> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE);
if (CollectionUtils.isEmpty(modelListVisible)) {
visible = false;
} else {
Map<Long, List<ModelResp>> id2domainDesc = modelListVisible.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
if (!CollectionUtils.isEmpty(id2domainDesc) && !id2domainDesc.containsKey(modelId)) {
visible = false;
}
}
if (!visible) {
ModelResp modelResp = modelService.getModel(modelId);
String modelName = modelResp.getName();
List<String> admins = modelService.getModelAdmin(modelResp.getId());
String message = String.format("您没有主题域[%s]权限，请联系管理员%s开通", modelName, admins);
throw new InvalidPermissionException(message);
}
}
private QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns,
Long domainId, AuthorizedResourceResp authResource) {
addPromptInfoInfo(domainId, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
private void addPromptInfoInfo(Long modelId, QueryResultWithSchemaResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
return;
}
List<String> admins = modelService.getModelAdmin(modelId);
if (!CollectionUtils.isEmpty(need2Apply)) {
String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins);
queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
}
if (!CollectionUtils.isEmpty(filters)) {
log.debug("dimensionFilters:{}", filters);
ModelResp modelResp = modelService.getModel(modelId);
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
filters.stream().forEach(filter -> {
descList.add(filter.getDescription());
exprList.add(filter.getExpressions().toString());
});
String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s";
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
log.info("queryResultWithColumns:{}", queryResultWithColumns);
}
}
private QueryResultWithSchemaResp deepCopyResult(QueryResultWithSchemaResp raw) throws Exception {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
BeanUtils.copyProperties(raw, queryResultWithColumns);
List<QueryColumn> columns = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getColumns())) {
String columnsStr = MAPPER.writeValueAsString(raw.getColumns());
columns = MAPPER.readValue(columnsStr, new TypeReference<List<QueryColumn>>() {
});
queryResultWithColumns.setColumns(columns);
}
queryResultWithColumns.setColumns(columns);
List<Map<String, Object>> resultData = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getResultList())) {
for (Map<String, Object> line : raw.getResultList()) {
Map<String, Object> newLine = new HashMap<>();
newLine.putAll(line);
resultData.add(newLine);
}
}
queryResultWithColumns.setResultList(resultData);
return queryResultWithColumns;
}
private boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
if (resAuthSet.containsAll(sensitiveResReq)) {
return true;
@@ -244,50 +129,6 @@ public class DataPermissionAOP {
return false;
}
private Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) {
Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> modelId.equals(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName()));
}
});
log.info("resAuthName:{}", resAuthName);
return resAuthName;
}
private AuthorizedResourceResp getAuthorizedResource(User user, Long domainId,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId, res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelId(domainId);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId,
authorizedResource);
return authorizedResource;
}
private Set<String> getHighSensitiveColsByModelId(Long modelId) {
Set<String> highSensitiveCols = new HashSet<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
List<DimensionResp> highSensitiveDimensions = dimensionService.getDimensions(metaFilter);
List<MetricResp> highSensitiveMetrics = metricService.getMetrics(metaFilter);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
if (!CollectionUtils.isEmpty(highSensitiveMetrics)) {
highSensitiveMetrics.stream().forEach(metric -> highSensitiveCols.add(metric.getBizName()));
}
return highSensitiveCols;
}
private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
@@ -320,64 +161,6 @@ public class DataPermissionAOP {
}
private QueryResultWithSchemaResp desensitizationData(QueryResultWithSchemaResp raw, Set<String> need2Apply) {
log.debug("start desensitizationData logic");
if (CollectionUtils.isEmpty(need2Apply)) {
log.info("user has all sensitiveRes");
return raw;
}
List<QueryColumn> columns = raw.getColumns();
boolean doDesensitization = false;
for (QueryColumn queryColumn : columns) {
if (need2Apply.contains(queryColumn.getNameEn())) {
doDesensitization = true;
break;
}
}
if (!doDesensitization) {
return raw;
}
QueryResultWithSchemaResp queryResultWithColumns = raw;
try {
queryResultWithColumns = deepCopyResult(raw);
} catch (Exception e) {
log.warn("deepCopyResult: ", e);
}
addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply);
desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply);
return queryResultWithColumns;
}
private void addAuthorizedSchemaInfo(List<QueryColumn> columns, Set<String> need2Apply) {
if (CollectionUtils.isEmpty(need2Apply)) {
return;
}
columns.stream().forEach(col -> {
if (need2Apply.contains(col.getNameEn())) {
col.setAuthorized(false);
}
});
}
private void desensitizationInternal(List<Map<String, Object>> result, Set<String> need2Apply) {
log.info("start desensitizationInternal logic");
for (int i = 0; i < result.size(); i++) {
Map<String, Object> row = result.get(i);
Map<String, Object> newRow = new HashMap<>();
for (String col : row.keySet()) {
if (need2Apply.contains(col)) {
newRow.put(col, "****");
} else {
newRow.put(col, row.get(col));
}
}
result.set(i, newRow);
}
}
private void doFilterCheckLogic(QueryStructReq queryStructReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
@@ -385,32 +168,18 @@ public class DataPermissionAOP {
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = new ArrayList<>();
modelIds.add(queryStructReq.getModelId());
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
Map<Long, ModelResp> modelRespMap = modelService.getModelMap();
List<Long> modelIds = Lists.newArrayList(queryStructReq.getModelIds());
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds));
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));
.forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
ModelResp modelResp = modelInfos.get(0);
List<String> admins = modelService.getModelAdmin(modelResp.getId());
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
}
}
private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) {
log.info("queryAuthResReq:{}", queryAuthResReq);
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
}

25
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -1,7 +1,9 @@
package com.tencent.supersonic.semantic.query.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -9,19 +11,9 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Objects;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -34,6 +26,14 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Aspect
@Component
@Slf4j
@@ -56,7 +56,7 @@ public class DimValueAspect {
}
Object[] args = joinPoint.getArgs();
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelId()));
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelIds()));
String sql = queryS2SQLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue.
@@ -147,8 +147,7 @@ public class DimValueAspect {
Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0];
Long modelId = queryStructReq.getModelId();
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId));
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryStructReq.getModelIds()));
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions);
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions);

53
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -1,15 +1,11 @@
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
@@ -23,7 +19,6 @@ import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
@@ -33,6 +28,14 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -46,14 +49,12 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.logging.log4j.util.Strings;
import org.assertj.core.util.Lists;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
@Slf4j
@@ -95,7 +96,7 @@ public class QueryStructUtils {
private List<Long> getDimensionIds(QueryStructReq queryStructCmd) {
List<Long> dimensionIds = new ArrayList<>();
List<DimensionResp> dimensions = catalog.getDimensions(queryStructCmd.getModelId());
List<DimensionResp> dimensions = catalog.getDimensions(queryStructCmd.getModelIds());
Map<String, List<DimensionResp>> pair = dimensions.stream()
.collect(Collectors.groupingBy(DimensionResp::getBizName));
for (String group : queryStructCmd.getGroups()) {
@@ -116,7 +117,7 @@ public class QueryStructUtils {
private List<Long> getMetricIds(QueryStructReq queryStructCmd) {
List<Long> metricIds = new ArrayList<>();
List<MetricResp> metrics = catalog.getMetrics(queryStructCmd.getModelId());
List<MetricResp> metrics = catalog.getMetrics(queryStructCmd.getModelIds());
Map<String, List<MetricResp>> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName));
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (pair.containsKey(agg.getColumn())) {
@@ -214,7 +215,7 @@ public class QueryStructUtils {
Set<String> resNameSet = getResName(queryS2SQLReq);
Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelId());
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds());
filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -256,14 +257,16 @@ public class QueryStructUtils {
if (CollectionUtils.isEmpty(groups)) {
log.warn("group is empty!");
} else {
String group = groups.get(0).equalsIgnoreCase("sys_imp_date")
? groups.get(1) : groups.get(0);
DimensionResp dimension = catalog.getDimension(group, modelId);
String datasourceBizName = dimension.getDatasourceBizName();
if (Strings.isNotEmpty(datasourceBizName)) {
internalMetricNamePrefix = datasourceBizName + UNDERLINE;
for (int i = 0; i < groups.size(); i++) {
if (groups.get(i).equalsIgnoreCase("sys_imp_date")) {
continue;
}
DimensionResp dimension = catalog.getDimension(groups.get(i), modelId);
if (Objects.nonNull(dimension) && Strings.isNotEmpty(dimension.getModelBizName())) {
internalMetricNamePrefix = dimension.getModelBizName() + UNDERLINE;
break;
}
}
}
String internalMetricName = internalMetricNamePrefix + internalMetricNameSuffix;
return internalMetricName;

27
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java
View File

@@ -1,20 +1,24 @@
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -26,12 +30,9 @@ import java.util.concurrent.CompletableFuture;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
@Slf4j
@@ -67,9 +68,9 @@ public class QueryUtils {
}
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, Long modelId) {
List<MetricResp> metricDescList = catalog.getMetrics(modelId);
List<DimensionResp> dimensionDescList = catalog.getDimensions(modelId);
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, List<Long> modelIds) {
List<MetricResp> metricDescList = catalog.getMetrics(modelIds);
List<DimensionResp> dimensionDescList = catalog.getDimensions(modelIds);
Map<String, MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, String> namePair = new HashMap<>();

48
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2SQLDataAspect.java
View File

@@ -1,8 +1,7 @@
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.common.pojo.Constants;
@@ -15,14 +14,8 @@ import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.ModelFilter;
import com.tencent.supersonic.semantic.query.service.AuthCommonService;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -38,6 +31,16 @@ import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Component
@Aspect
@Order(1)
@@ -72,29 +75,29 @@ public class S2SQLDataAspect {
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
Long modelId = queryS2SQLReq.getModelId();
List<Long> modelIds = queryS2SQLReq.getModelIds();
//1. determine whether admin of the model
if (authCommonService.doModelAdmin(user, modelId)) {
if (authCommonService.doModelAdmin(user, modelIds)) {
log.info("determine whether admin of the model!");
return joinPoint.proceed();
}
// 2. determine whether the subject field is visible
authCommonService.doModelVisible(user, modelId);
authCommonService.doModelVisible(user, modelIds);
// 3. fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2SQLReq, user);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId);
Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelIds);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelId, sensitiveResReq);
log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = authCommonService
.getAuthorizedResource(user, modelId, sensitiveResReq);
.getAuthorizedResource(user, modelIds, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId);
Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelIds);
// 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryS2SQLReq, resAuthSet, sensitiveResReq);
@@ -109,7 +112,7 @@ public class S2SQLDataAspect {
.allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return authCommonService.getQueryResultWithColumns(queryResultWithColumns, modelId, authorizedResource);
return authCommonService.getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
}
// 6.if the column has no permission, hit *
@@ -118,7 +121,7 @@ public class S2SQLDataAspect {
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
QueryResultWithSchemaResp queryResultAfterDesensitization = authCommonService
.desensitizationData(queryResultWithColumns, need2Apply);
authCommonService.addPromptInfoInfo(modelId, queryResultAfterDesensitization, authorizedResource, need2Apply);
authCommonService.addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
@@ -163,9 +166,10 @@ public class S2SQLDataAspect {
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = new ArrayList<>();
modelIds.add(queryS2SQLReq.getModelId());
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setModelIds(modelIds);
List<ModelResp> modelInfos = modelService.getModelList(modelFilter);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();

31
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java
View File

@@ -7,6 +7,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.enums.QueryOptMode;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeBackEnum;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
@@ -17,23 +18,20 @@ import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.api.model.enums.QueryOptMode;
import com.tencent.supersonic.semantic.query.persistence.repository.StatRepository;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@Component
@Slf4j
@@ -91,23 +89,20 @@ public class StatUtils {
public void initStatInfo(QueryS2SQLReq queryS2SQLReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
queryStatInfo.setModelId(queryS2SQLReq.getModelId());
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelId());
queryStatInfo.setModelId(queryS2SQLReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelIds().get(0));
List<String> dimensions = new ArrayList<>();
if (Objects.nonNull(modelSchemaResp)) {
dimensions = getFieldNames(allFields, modelSchemaResp.getDimensions());
}
List<String> metrics = new ArrayList<>();
if (Objects.nonNull(modelSchemaResp)) {
dimensions = getFieldNames(allFields, modelSchemaResp.getDimensions());
metrics = getFieldNames(allFields, modelSchemaResp.getMetrics());
}
String userName = getUserName(facadeUser);
try {
queryStatInfo.setTraceId("")
.setModelId(queryS2SQLReq.getModelId())
.setModelId(queryS2SQLReq.getModelIds().get(0))
.setUser(userName)
.setQueryType(QueryTypeEnum.SQL.getValue())
.setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())
@@ -137,7 +132,7 @@ public class StatUtils {
try {
queryStatInfo.setTraceId(traceId)
.setModelId(queryStructCmd.getModelId())
.setModelId(1L)
.setUser(user)
.setQueryType(QueryTypeEnum.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())

5
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/MaterializationQueryTest.java
View File

@@ -10,9 +10,10 @@ import com.tencent.supersonic.semantic.query.parser.QueryParser;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.ComponentFactory;
import com.tencent.supersonic.semantic.query.utils.QueryUtils;
import java.util.Arrays;
import lombok.extern.slf4j.Slf4j;
import java.util.Arrays;
@Slf4j
public class MaterializationQueryTest {
@@ -43,7 +44,7 @@ public class MaterializationQueryTest {
try {
QueryStatement queryStatement = queryParser.logicSql(queryStructReq);
queryUtils.checkSqlParse(queryStatement);
queryStatement.setModelId(queryStructReq.getModelId());
queryStatement.setModelIds(queryStructReq.getModelIds());
log.info("queryStatement:{}", queryStatement);
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStructReq, queryStatement);

6
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.domain.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DataModelYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl;
@@ -53,7 +53,7 @@ class SemanticParserServiceTest {
public void test() throws Exception {
DatasourceYamlTpl datasource = new DatasourceYamlTpl();
DataModelYamlTpl datasource = new DataModelYamlTpl();
datasource.setName("s2_pv_uv_statis");
datasource.setSourceId(1L);
datasource.setSqlQuery("SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis");
@@ -195,7 +195,7 @@ class SemanticParserServiceTest {
private static void addDepartment(SemanticSchema semanticSchema) {
DatasourceYamlTpl datasource = new DatasourceYamlTpl();
DataModelYamlTpl datasource = new DataModelYamlTpl();
datasource.setName("user_department");
datasource.setSourceId(1L);
datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");