[release][project] supersonic 0.7.3 version backend update (#40)

* [improvement] add some features

* [improvement] revise CHANGELOG

---------

Co-authored-by: zuopengge <hwzuopengge@tencent.com>
This commit is contained in:
mainmain
2023-08-29 20:06:34 +08:00
committed by GitHub
parent 6fe9ab79ed
commit e1911bc81b
260 changed files with 6466 additions and 7108 deletions

View File

@@ -79,7 +79,6 @@ public abstract class Renderer {
if (measure.get().getConstraint() != null && !measure.get().getConstraint().isEmpty()) {
metricNode.getMeasureFilter()
.put(measure.get().getName(), SemanticNode.parse(measure.get().getConstraint(), scope));
;
}
}
return metricNode;

View File

@@ -7,6 +7,7 @@ import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -17,6 +18,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.parser.SqlParser;
@@ -50,7 +52,7 @@ public class DataSourceNode extends SemanticNode {
}
public static void getQueryDimensionMeasure(SemanticSchema schema, MetricReq metricCommand,
Set<String> queryDimension, List<String> measures) {
Set<String> queryDimension, List<String> measures) {
queryDimension.addAll(metricCommand.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] : d)
.collect(Collectors.toSet()));
@@ -62,17 +64,18 @@ public class DataSourceNode extends SemanticNode {
}
public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricReq metricCommand,
Set<String> queryDimension, List<String> measures, SqlValidatorScope scope) throws Exception {
Set<String> queryDimension, List<String> measures,
SqlValidatorScope scope) throws Exception {
if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
Set<String> filterConditions = new HashSet<>();
FilterNode.getFilterField(parse(metricCommand.getWhere(), scope), filterConditions);
Set<String> queryMeasures = new HashSet<>(measures);
Set<String> schemaMetricName = schema.getMetrics().stream().map(m -> m.getName())
.collect(Collectors.toSet());
Set<String> schemaMetricName = schema.getMetrics().stream()
.map(m -> m.getName()).collect(Collectors.toSet());
for (String filterCondition : filterConditions) {
if (schemaMetricName.contains(filterCondition)) {
schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(filterCondition)).forEach(
m -> m.getMetricTypeParams().getMeasures().stream()
schema.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(filterCondition))
.forEach(m -> m.getMetricTypeParams().getMeasures().stream()
.forEach(mm -> queryMeasures.add(mm.getName())));
continue;
}
@@ -84,14 +87,13 @@ public class DataSourceNode extends SemanticNode {
}
public static List<DataSource> getMatchDataSources(SqlValidatorScope scope, SemanticSchema schema,
MetricReq metricCommand) throws Exception {
MetricReq metricCommand) throws Exception {
List<DataSource> dataSources = new ArrayList<>();
// check by metric
List<String> measures = new ArrayList<>();
Set<String> queryDimension = new HashSet<>();
getQueryDimensionMeasure(schema, metricCommand, queryDimension, measures);
String sourceName = "";
DataSource baseDataSource = null;
// one , match measure count
Map<String, Integer> dataSourceMeasures = new HashMap<>();
@@ -148,8 +150,12 @@ public class DataSourceNode extends SemanticNode {
return dataSources;
}
private static boolean checkMatch(Set<String> sourceMeasure, Set<String> queryDimension, List<String> measures,
Set<String> dimension, MetricReq metricCommand, SqlValidatorScope scope) throws Exception {
private static boolean checkMatch(Set<String> sourceMeasure,
Set<String> queryDimension,
List<String> measures,
Set<String> dimension,
MetricReq metricCommand,
SqlValidatorScope scope) throws Exception {
boolean isAllMatch = true;
sourceMeasure.retainAll(measures);
if (sourceMeasure.size() < measures.size()) {
@@ -173,8 +179,11 @@ public class DataSourceNode extends SemanticNode {
return isAllMatch;
}
private static List<DataSource> getLinkDataSources(Set<String> baseIdentifiers, Set<String> queryDimension,
List<String> measures, DataSource baseDataSource, SemanticSchema schema) {
private static List<DataSource> getLinkDataSources(Set<String> baseIdentifiers,
Set<String> queryDimension,
List<String> measures,
DataSource baseDataSource,
SemanticSchema schema) {
Set<String> linkDataSourceName = new HashSet<>();
List<DataSource> linkDataSources = new ArrayList<>();
for (Map.Entry<String, DataSource> entry : schema.getDatasource().entrySet()) {
@@ -200,7 +209,6 @@ public class DataSourceNode extends SemanticNode {
linkMeasure.retainAll(measures);
if (!linkMeasure.isEmpty()) {
isMatch = true;
;
}
}
if (isMatch) {

View File

@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -106,10 +105,7 @@ public class CalculateAggConverter implements SemanticConverter {
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {
return false;
}
//todo ck类型暂不拼with语句
if (queryStructCmd.getModelId().equals(34L)) {
return false;
}
int nonSumFunction = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) {
@@ -118,9 +114,7 @@ public class CalculateAggConverter implements SemanticConverter {
if (agg.getFunc().equals(AggOperatorEnum.UNKNOWN)) {
return false;
}
if (agg.getFunc() != null
// && !agg.getFunc().equalsIgnoreCase(MetricAggDefault)
) {
if (agg.getFunc() != null) {
nonSumFunction++;
}
}
@@ -129,7 +123,7 @@ public class CalculateAggConverter implements SemanticConverter {
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
MetricReq metricCommand) throws Exception {
DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelId());
ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd,
EngineTypeEnum.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
@@ -156,7 +150,7 @@ public class CalculateAggConverter implements SemanticConverter {
}
public ParseSqlReq generateRatioSqlCommand(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum,
String version)
String version)
throws Exception {
check(queryStructCmd);
ParseSqlReq sqlCommand = new ParseSqlReq();
@@ -178,15 +172,17 @@ public class CalculateAggConverter implements SemanticConverter {
sql = new H2EngineSql().sql(queryStructCmd, isOver, metricTableName);
break;
case MYSQL:
if (Objects.nonNull(version) && version.startsWith(mysqlLowVersion)) {
sqlCommand.setSupportWith(false);
sql = new MysqlEngineSql().sql(queryStructCmd, isOver, metricTableName);
break;
}
case DORIS:
case CLICKHOUSE:
sql = new CkEngineSql().sql(queryStructCmd, isOver, metricTableName);
if (engineTypeEnum.equals(EngineTypeEnum.MYSQL) && Objects.nonNull(version) && version.startsWith(
mysqlLowVersion)) {
sqlCommand.setSupportWith(false);
sql = new MysqlEngineSql().sql(queryStructCmd, isOver, metricTableName);
} else {
sql = new CkEngineSql().sql(queryStructCmd, isOver, metricTableName);
}
break;
default:
}
sqlCommand.setSql(sql);
return sqlCommand;
@@ -275,7 +271,7 @@ public class CalculateAggConverter implements SemanticConverter {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format("toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s','-01')),%s) ",
return String.format("toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
@@ -302,7 +298,8 @@ public class CalculateAggConverter implements SemanticConverter {
@Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, String metricSql) {
String sql = String.format(
",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s from t0 left join t1 on %s ) metric_tb_src %s %s ",
",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s "
+ "from t0 left join t1 on %s ) metric_tb_src %s %s ",
metricSql, metricSql, getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."),
getAllJoinSelect(queryStructCmd, "t1."),
getJoinOn(queryStructCmd, isOver, "t0.", "t1."),
@@ -330,8 +327,6 @@ public class CalculateAggConverter implements SemanticConverter {
}
public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format(
@@ -460,9 +455,6 @@ public class CalculateAggConverter implements SemanticConverter {
}
private void check(QueryStructReq queryStructCmd) throws Exception {
Set<String> aggFunctions = queryStructCmd.getAggregators().stream()
.filter(f -> f.getArgs() != null && f.getArgs().get(0) != null)
.map(agg -> agg.getArgs().get(0).toLowerCase()).collect(Collectors.toSet());
Long ratioOverNum = queryStructCmd.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
Long ratioRollNum = queryStructCmd.getAggregators().stream()

View File

@@ -8,11 +8,11 @@ import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
@Component("DefaultDimValueConverter")
@@ -24,8 +24,8 @@ public class DefaultDimValueConverter implements SemanticConverter {
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
public void converter(Catalog catalog, QueryStructReq queryStructCmd,
ParseSqlReq sqlCommend, MetricReq metricCommand) throws Exception {
List<DimensionResp> dimensionResps = catalog.getDimensions(queryStructCmd.getModelId());
//dimension which has default values
dimensionResps = dimensionResps.stream()

View File

@@ -132,6 +132,4 @@ public class MultiSourceJoin implements SemanticConverter {
MetricReq metricCommand) throws Exception {
buildJoinPrefix(queryStructCmd);
}
}

View File

@@ -1,9 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.convert;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.query.pojo.Param;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
@@ -13,9 +10,7 @@ import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -68,7 +63,8 @@ public class ParserDefaultConverter implements SemanticConverter {
// todo tmp delete
// support detail query
if (queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
String internalMetricName = queryStructUtils.generateInternalMetricName(queryStructCmd.getModelId(), queryStructCmd.getGroups());
String internalMetricName = queryStructUtils.generateInternalMetricName(
queryStructCmd.getModelId(), queryStructCmd.getGroups());
sqlCommend.getMetrics().add(internalMetricName);
}

View File

@@ -3,15 +3,15 @@ package com.tencent.supersonic.semantic.query.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -57,7 +57,6 @@ public class QueryController {
public SqlParserResp parseByStruct(@RequestBody ParseSqlReq parseSqlReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
QueryStatement queryStatement = semanticQueryEngine.physicalSql(parseSqlReq);
SqlParserResp sqlParserResp = new SqlParserResp();
BeanUtils.copyProperties(queryStatement, sqlParserResp);

View File

@@ -7,22 +7,22 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
@RestController
@RequestMapping("/api/semantic/schema")
@@ -33,24 +33,24 @@ public class SchemaController {
@PostMapping
public List<ModelSchemaResp> fetchModelSchema(@RequestBody ModelSchemaFilterReq filter,
HttpServletRequest request,
HttpServletResponse response) {
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.fetchModelSchema(filter, user);
}
@GetMapping("/domain/list")
public List<DomainResp> getDomainList(HttpServletRequest request,
HttpServletResponse response) {
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.getDomainList(user);
}
@GetMapping("/model/list")
public List<ModelResp> getModelList(@RequestParam("domainId") Long domainId,
@RequestParam("authType") String authType,
HttpServletRequest request,
HttpServletResponse response) {
@RequestParam("authType") String authType,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.getModelList(user, AuthType.valueOf(authType), domainId);
}
@@ -71,4 +71,4 @@ public class SchemaController {
return schemaService.queryMetric(pageMetricCmd, user);
}
}
}

View File

@@ -2,9 +2,8 @@ package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;

View File

@@ -6,11 +6,12 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import java.util.List;
public interface SchemaService {

View File

@@ -5,23 +5,23 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -40,10 +40,10 @@ public class SchemaServiceImpl implements SchemaService {
private final DomainService domainService;
public SchemaServiceImpl(QueryService queryService,
ModelService modelService,
DimensionService dimensionService,
MetricService metricService,
DomainService domainService) {
ModelService modelService,
DimensionService dimensionService,
MetricService metricService,
DomainService domainService) {
this.queryService = queryService;
this.modelService = modelService;
this.dimensionService = dimensionService;

View File

@@ -12,23 +12,20 @@ import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -39,13 +36,16 @@ import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.assertj.core.util.Sets;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@@ -133,14 +133,14 @@ public class DataPermissionAOP {
.collect(Collectors.toSet());
QueryResultWithSchemaResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns,
need2Apply);
addPromptInfoInfo(modelId, queryResultAfterDesensitization, authorizedResource);
addPromptInfoInfo(modelId, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
private boolean doModelAdmin(User user, QueryStructReq queryStructCmd) {
Long modelId = queryStructCmd.getModelId();
private boolean doModelAdmin(User user, QueryStructReq queryStructReq) {
Long modelId = queryStructReq.getModelId();
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user.getName(), null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
@@ -151,69 +151,60 @@ public class DataPermissionAOP {
}
}
private void doModelVisible(User user, QueryStructReq queryStructCmd) {
private void doModelVisible(User user, QueryStructReq queryStructReq) {
Boolean visible = true;
Long domainId = queryStructCmd.getModelId();
Long modelId = queryStructReq.getModelId();
List<ModelResp> modelListVisible = modelService.getModelListWithAuth(user.getName(), null, AuthType.VISIBLE);
if (CollectionUtils.isEmpty(modelListVisible)) {
visible = false;
} else {
Map<Long, List<ModelResp>> id2domainDesc = modelListVisible.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
if (!CollectionUtils.isEmpty(id2domainDesc) && !id2domainDesc.containsKey(domainId)) {
if (!CollectionUtils.isEmpty(id2domainDesc) && !id2domainDesc.containsKey(modelId)) {
visible = false;
}
}
if (!visible) {
List<Long> modelIds = new ArrayList<>();
modelIds.add(domainId);
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
if (CollectionUtils.isEmpty(modelInfos)) {
throw new InvalidArgumentException(
"invalid domainId:" + domainId + ", please contact admin for details");
}
String domainName = modelInfos.get(0).getName();
throw new InvalidPermissionException(
"You do not have domain:" + domainName + " permission, please contact admin for details");
ModelResp modelResp = modelService.getModel(modelId);
String modelName = modelResp.getName();
List<String> admins = modelService.getModelAdmin(modelResp.getId());
String message = String.format("您没有主题域[%s]权限,请联系管理员%s开通", modelName, admins);
throw new InvalidArgumentException(message);
}
}
private QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns,
Long domainId, AuthorizedResourceResp authResource) {
addPromptInfoInfo(domainId, resultWithColumns, authResource);
addPromptInfoInfo(domainId, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
private void addPromptInfoInfo(Long domainId, QueryResultWithSchemaResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource) {
private void addPromptInfoInfo(Long modelId, QueryResultWithSchemaResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
return;
}
List<String> admins = modelService.getModelAdmin(modelId);
if (!CollectionUtils.isEmpty(need2Apply)) {
String promptInfo = String.format("当前结果已经过脱敏处理, 申请权限请联系管理员%s", admins);
queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
}
if (!CollectionUtils.isEmpty(filters)) {
log.debug("dimensionFilters:{}", filters);
List<Long> modelIds = new ArrayList<>();
modelIds.add(domainId);
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = "";
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
ModelResp modelResp = modelService.getModel(modelId);
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
filters.stream().forEach(filter -> {
descList.add(filter.getDescription());
exprList.add(filter.getExpressions().toString());
});
String promptInfo = "the current data has been controlled by permissions,"
+ " related information:%s, please contact admin for details";
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList);
String promptInfo = "当前结果已经过行权限过滤,详细过滤条件如下:%s, 申请权限请联系管理员%s";
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(modelNameCn, exprList, descList, message));
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
log.info("queryResultWithColumns:{}", queryResultWithColumns);
}
}
@@ -292,7 +283,7 @@ public class DataPermissionAOP {
return highSensitiveCols;
}
private void doRowPermission(QueryStructReq queryStructCmd, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -313,13 +304,13 @@ public class DataPermissionAOP {
});
if (StringUtils.isNotEmpty(joiner.toString())) {
log.info("before doRowPermission, queryStructCmd:{}", queryStructCmd);
log.info("before doRowPermission, queryStructReq:{}", queryStructReq);
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString());
List<Filter> filters = Objects.isNull(queryStructCmd.getOriginalFilter()) ? new ArrayList<>()
: queryStructCmd.getOriginalFilter();
List<Filter> filters = Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>()
: queryStructReq.getOriginalFilter();
filters.add(filter);
queryStructCmd.setDimensionFilters(filters);
log.info("after doRowPermission, queryStructCmd:{}", queryStructCmd);
queryStructReq.setDimensionFilters(filters);
log.info("after doRowPermission, queryStructReq:{}", queryStructReq);
}
}
@@ -348,7 +339,7 @@ public class DataPermissionAOP {
try {
queryResultWithColumns = deepCopyResult(raw);
} catch (Exception e) {
log.warn("deepCopyResult, e:{}", e);
log.warn("deepCopyResult: ", e);
}
addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply);
desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply);
@@ -382,30 +373,32 @@ public class DataPermissionAOP {
}
}
private void doFilterCheckLogic(QueryStructReq queryStructCmd, Set<String> resAuthName,
private void doFilterCheckLogic(QueryStructReq queryStructReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructCmd);
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = new ArrayList<>();
modelIds.add(queryStructCmd.getModelId());
modelIds.add(queryStructReq.getModelId());
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryStructCmd.getModelId());
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryStructReq.getModelId());
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
log.warn("in doFilterLogic, need2Apply:{}", need2Apply);
throw new InvalidPermissionException(
"you do not have data permission:" + nameCnSet + ", please contact admin for details");
ModelResp modelResp = modelInfos.get(0);
List<String> admins = modelService.getModelAdmin(modelResp.getId());
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
}
}
@@ -415,4 +408,4 @@ public class DataPermissionAOP {
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
}
}

View File

@@ -202,6 +202,19 @@ public class DateUtils {
* @return
*/
public String betweenDateStr(ItemDateResp dateDate, DateConf dateInfo) {
if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
LocalDate endData = LocalDate.parse(dateInfo.getEndDate(),
DateTimeFormatter.ofPattern(DAY_FORMAT));
LocalDate startData = LocalDate.parse(dateInfo.getStartDate(),
DateTimeFormatter.ofPattern(DAY_FORMAT));
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(MONTH_FORMAT);
return String.format("%s >= '%s' and %s <= '%s'",
sysDateMonthCol, startData.format(formatter), sysDateMonthCol, endData.format(formatter));
}
if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
return String.format("%s >= '%s' and %s <= '%s'",
sysDateWeekCol, dateInfo.getStartDate(), sysDateWeekCol, dateInfo.getEndDate());
}
return String.format("%s >= '%s' and %s <= '%s'",
sysDateCol, dateInfo.getStartDate(), sysDateCol, dateInfo.getEndDate());
}

View File

@@ -7,12 +7,6 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.aspectj.lang.ProceedingJoinPoint;
@@ -23,6 +17,13 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
@Aspect
@Component
@Slf4j
@@ -34,9 +35,9 @@ public class DimValueAspect {
@Autowired
private DimensionService dimensionService;
@Around("execution(* com.tencent.supersonic.semantic.query.rest.QueryController.queryByStruct(..))" +
" || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStruct(..))" +
" || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStructWithAuth(..))")
@Around("execution(* com.tencent.supersonic.semantic.query.rest.QueryController.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStructWithAuth(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {
@@ -65,15 +66,15 @@ public class DimValueAspect {
}
private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (!selectDimValueMap(queryResultWithColumns.getColumns(), dimAndTechNameAndBizNamePair)) {
return;
}
log.debug("start rewriteDimValue for resultList");
for (Map<String, Object> line : queryResultWithColumns.getResultList()) {
for (String bizName : line.keySet()) {
String techName = line.get(bizName).toString();
if (dimAndTechNameAndBizNamePair.containsKey(bizName)) {
if (dimAndTechNameAndBizNamePair.containsKey(bizName) && Objects.nonNull(line.get(bizName))) {
String techName = line.get(bizName).toString();
Map<String, String> techAndBizPair = dimAndTechNameAndBizNamePair.get(bizName);
if (!CollectionUtils.isEmpty(techAndBizPair) && techAndBizPair.containsKey(techName)) {
String bizValueName = techAndBizPair.get(techName);
@@ -86,10 +87,10 @@ public class DimValueAspect {
}
}
private boolean selectDimValueMap(List<QueryColumn> columns,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair) || CollectionUtils.isEmpty(
dimAndTechNameAndBizNamePair)) {
private boolean selectDimValueMap(List<QueryColumn> columns, Map<String,
Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)
|| CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)) {
return false;
}
@@ -118,9 +119,11 @@ public class DimValueAspect {
List<String> values = (List) value;
List<String> valuesNew = new ArrayList<>();
for (String valueSingle : values) {
boolean f =
aliasPair.containsKey(valueSingle) ? valuesNew.add(aliasPair.get(valueSingle))
: valuesNew.add(valueSingle);
if (aliasPair.containsKey(valueSingle)) {
valuesNew.add(aliasPair.get(valueSingle));
} else {
valuesNew.add(valueSingle);
}
}
filter.setValue(valuesNew);
}
@@ -138,9 +141,9 @@ public class DimValueAspect {
}
}
private void generateAliasAndTechNamePair(List<DimensionResp> dimensions
, Map<String, Map<String, String>> dimAndAliasAndTechNamePair
, Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
private void generateAliasAndTechNamePair(List<DimensionResp> dimensions,
Map<String, Map<String, String>> dimAndAliasAndTechNamePair,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
@@ -184,4 +187,4 @@ public class DimValueAspect {
});
}
}
}

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.util.jsqlparser.CCJSqlParserUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
@@ -38,8 +38,8 @@ public class QueryReqConverter {
MetricTable metricTable = new MetricTable();
String sql = databaseReq.getSql();
List<String> allFields = CCJSqlParserUtils.getAllFields(sql);
String tableName = CCJSqlParserUtils.getTableName(sql);
List<String> allFields = SqlParserSelectHelper.getAllFields(sql);
String tableName = SqlParserSelectHelper.getTableName(sql);
if (CollectionUtils.isEmpty(domainSchemas) || StringUtils.isEmpty(tableName)) {
return new QueryStatement();

View File

@@ -33,17 +33,14 @@ import org.springframework.util.CollectionUtils;
@Slf4j
@Component
public class QueryStructUtils {
public static Set<String> internalCols = new HashSet<>(
Arrays.asList("dayno", "plat_sys_var", "sys_imp_date", "sys_imp_week", "sys_imp_month"));
private final DateUtils dateUtils;
private final SqlFilterUtils sqlFilterUtils;
private final Catalog catalog;
@Value("${internal.metric.cnt.suffix:internal_cnt}")
private String internalMetricNameSuffix;
public static Set<String> internalCols = new HashSet<>(
Arrays.asList("dayno", "plat_sys_var", "sys_imp_date", "sys_imp_week", "sys_imp_month"));
public QueryStructUtils(
DateUtils dateUtils,
SqlFilterUtils sqlFilterUtils, Catalog catalog) {

View File

@@ -40,16 +40,6 @@ public class QueryUtils {
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
@PostConstruct
public void fillPattern() {
Set<String> aggFunctions = new HashSet<>(Arrays.asList("MAX", "MIN", "SUM", "AVG"));
String patternStr = "\\s*(%s\\((.*)\\)) AS";
for (String agg : aggFunctions) {
patterns.add(Pattern.compile(String.format(patternStr, agg)));
}
}
private final CacheUtils cacheUtils;
private final StatUtils statUtils;
@@ -63,12 +53,20 @@ public class QueryUtils {
this.catalog = catalog;
}
@PostConstruct
public void fillPattern() {
Set<String> aggFunctions = new HashSet<>(Arrays.asList("MAX", "MIN", "SUM", "AVG"));
String patternStr = "\\s*(%s\\((.*)\\)) AS";
for (String agg : aggFunctions) {
patterns.add(Pattern.compile(String.format(patternStr, agg)));
}
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, Long modelId) {
List<MetricResp> metricDescList = catalog.getMetrics(modelId);
List<DimensionResp> dimensionDescList = catalog.getDimensions(modelId);
Map<String,MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a,(k1, k2)->k1));
Map<String, MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, String> namePair = new HashMap<>();
Map<String, String> nameTypePair = new HashMap<>();
addSysTimeDimension(namePair, nameTypePair);
@@ -95,24 +93,13 @@ public class QueryUtils {
if (!nameTypePair.containsKey(nameEn) && isNumberType(column.getType())) {
column.setShowType("NUMBER");
}
if(metricRespMap.containsKey(nameEn)){
if (metricRespMap.containsKey(nameEn)) {
column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType());
column.setDataFormat(metricRespMap.get(nameEn).getDataFormat());
}
});
}
private boolean isNumberType(String type) {
if (StringUtils.isBlank(type)) {
return false;
}
if (type.equalsIgnoreCase("int") || type.equalsIgnoreCase("bigint")
|| type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")) {
return true;
}
return false;
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns,
QueryMultiStructReq queryMultiStructCmd) {
List<Aggregator> aggregators = queryMultiStructCmd.getQueryStructReqs().stream()
@@ -152,6 +139,17 @@ public class QueryUtils {
});
}
private boolean isNumberType(String type) {
if (StringUtils.isBlank(type)) {
return false;
}
if (type.equalsIgnoreCase("int") || type.equalsIgnoreCase("bigint")
|| type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")) {
return true;
}
return false;
}
private Map<String, String> getMetricNameFromAgg(List<Aggregator> aggregators) {
Map<String, String> map = new HashMap<>();
if (CollectionUtils.isEmpty(aggregators)) {