mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-13 04:57:28 +00:00
(improvement)(project) support for modifying filter conditions and fix group by pushdown and add windows scipt (#49)
Co-authored-by: lexluo <lexluo@tencent.com>
This commit is contained in:
lexluo09
@@ -4,6 +4,5 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
|
||||
|
||||
public interface QueryOptimizer {
|
||||
|
||||
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,5 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
|
||||
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
|
||||
|
||||
public interface SqlParser {
|
||||
|
||||
QueryStatement explain(MetricReq metricReq, boolean isAgg, Catalog catalog) throws Exception;
|
||||
}
|
||||
|
||||
@@ -3,15 +3,14 @@ package com.tencent.supersonic.semantic.query.parser.calcite;
|
||||
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
|
||||
import com.tencent.supersonic.semantic.model.domain.Catalog;
|
||||
import com.tencent.supersonic.semantic.query.parser.SqlParser;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component("CalciteSqlParser")
|
||||
public class CalciteSqlParser implements SqlParser {
|
||||
|
||||
private final SemanticSchemaManager semanticSchemaManager;
|
||||
|
||||
public CalciteSqlParser(
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.tencent.supersonic.semantic.query.parser.calcite;
|
||||
|
||||
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
|
||||
import java.util.Properties;
|
||||
import org.apache.calcite.avatica.util.Casing;
|
||||
import org.apache.calcite.avatica.util.Quoting;
|
||||
|
||||
@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner;
|
||||
|
||||
|
||||
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.FilterRender;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.OutputRender;
|
||||
|
||||
@@ -2,15 +2,15 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql;
|
||||
|
||||
|
||||
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@@ -2,8 +2,8 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
|
||||
|
||||
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Optimization;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
|
||||
@@ -2,14 +2,14 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
|
||||
|
||||
|
||||
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
|
||||
|
||||
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.AggFunctionNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.AggFunctionNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
|
||||
|
||||
|
||||
import com.tencent.supersonic.common.pojo.ColumnOrder;
|
||||
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.common.pojo.ColumnOrder;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@@ -4,7 +4,6 @@ import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class QueryStatement {
|
||||
|
||||
private Long modelId = 0L;
|
||||
private String sql = "";
|
||||
private String sourceId = "";
|
||||
|
||||
@@ -4,10 +4,10 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
|
||||
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
|
||||
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.semantic.query.persistence.mapper.StatMapper;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.semantic.query.utils;
|
||||
package com.tencent.supersonic.semantic.query.service;
|
||||
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq;
|
||||
@@ -8,7 +8,7 @@ import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
|
||||
import com.tencent.supersonic.semantic.model.domain.ModelService;
|
||||
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
|
||||
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
@@ -25,10 +25,8 @@ public class QueryReqConverter {
|
||||
|
||||
@Autowired
|
||||
private ModelService domainService;
|
||||
|
||||
@Autowired
|
||||
private SemanticQueryEngine parserService;
|
||||
|
||||
@Autowired
|
||||
private QueryStructUtils queryStructUtils;
|
||||
|
||||
@@ -3,8 +3,8 @@ package com.tencent.supersonic.semantic.query.service;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
|
||||
import java.util.List;
|
||||
|
||||
@@ -15,7 +15,6 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
|
||||
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
|
||||
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.semantic.query.utils.QueryReqConverter;
|
||||
import com.tencent.supersonic.semantic.query.utils.QueryUtils;
|
||||
import com.tencent.supersonic.semantic.query.utils.StatUtils;
|
||||
import java.util.ArrayList;
|
||||
|
||||
@@ -59,11 +59,9 @@ public class ComponentFactory {
|
||||
public static void setSqlParser(SqlParser parser) {
|
||||
sqlParser = parser;
|
||||
}
|
||||
|
||||
private static void initQueryOptimizer() {
|
||||
queryOptimizers.add(getBean("DetailQuery", DetailQuery.class));
|
||||
}
|
||||
|
||||
private static void initSemanticConverter() {
|
||||
semanticConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
|
||||
semanticConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
|
||||
|
||||
@@ -5,9 +5,16 @@ import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.semantic.model.domain.DimensionService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.aspectj.lang.ProceedingJoinPoint;
|
||||
import org.aspectj.lang.annotation.Around;
|
||||
@@ -17,13 +24,6 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@Aspect
|
||||
@Component
|
||||
@Slf4j
|
||||
@@ -32,9 +32,32 @@ public class DimValueAspect {
|
||||
@Value("${dimension.value.map.enable:true}")
|
||||
private Boolean dimensionValueMapEnable;
|
||||
|
||||
@Value("${dimension.value.map.sql.enable:true}")
|
||||
private Boolean dimensionValueMapSqlEnable;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
|
||||
@Around("execution(* com.tencent.supersonic.semantic.query.service.QueryServiceImpl.queryBySql(..))")
|
||||
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||
if (!dimensionValueMapSqlEnable) {
|
||||
log.debug("sql dimensionValueMapEnable is false, skip dimensionValueMap");
|
||||
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
Object[] args = joinPoint.getArgs();
|
||||
QueryDslReq queryDslReq = (QueryDslReq) args[0];
|
||||
|
||||
List<DimensionResp> dimensions = dimensionService.getDimensions(queryDslReq.getModelId());
|
||||
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
|
||||
|
||||
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
|
||||
if (Objects.nonNull(queryResultWithColumns)) {
|
||||
rewriteDimValue(queryResultWithColumns, techNameToBizName);
|
||||
}
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
|
||||
@Around("execution(* com.tencent.supersonic.semantic.query.rest.QueryController.queryByStruct(..))"
|
||||
+ " || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStruct(..))"
|
||||
+ " || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStructWithAuth(..))")
|
||||
@@ -48,12 +71,11 @@ public class DimValueAspect {
|
||||
|
||||
Object[] args = joinPoint.getArgs();
|
||||
QueryStructReq queryStructReq = (QueryStructReq) args[0];
|
||||
Long domainId = queryStructReq.getModelId();
|
||||
Long modelId = queryStructReq.getModelId();
|
||||
|
||||
List<DimensionResp> dimensions = dimensionService.getDimensions(domainId);
|
||||
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = new ConcurrentHashMap<>();
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = new ConcurrentHashMap<>();
|
||||
generateAliasAndTechNamePair(dimensions, dimAndAliasAndTechNamePair, dimAndTechNameAndBizNamePair);
|
||||
List<DimensionResp> dimensions = dimensionService.getDimensions(modelId);
|
||||
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions);
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions);
|
||||
|
||||
rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair);
|
||||
|
||||
@@ -66,7 +88,7 @@ public class DimValueAspect {
|
||||
}
|
||||
|
||||
private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns,
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
|
||||
if (!selectDimValueMap(queryResultWithColumns.getColumns(), dimAndTechNameAndBizNamePair)) {
|
||||
return;
|
||||
}
|
||||
@@ -141,50 +163,78 @@ public class DimValueAspect {
|
||||
}
|
||||
}
|
||||
|
||||
private void generateAliasAndTechNamePair(List<DimensionResp> dimensions,
|
||||
Map<String, Map<String, String>> dimAndAliasAndTechNamePair,
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
|
||||
private Map<String, Map<String, String>> getAliasAndBizNameToTechName(List<DimensionResp> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensions)) {
|
||||
return;
|
||||
return new HashMap<>();
|
||||
}
|
||||
dimensions.stream().forEach(dimension -> {
|
||||
if (Objects.nonNull(dimension) && Strings.isNotEmpty(dimension.getBizName())
|
||||
&& !CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
|
||||
String bizName = dimension.getBizName();
|
||||
Map<String, Map<String, String>> result = new HashMap<>();
|
||||
for (DimensionResp dimension : dimensions) {
|
||||
if (needSkipDimension(dimension)) {
|
||||
continue;
|
||||
}
|
||||
String bizName = dimension.getBizName();
|
||||
List<DimValueMap> dimValueMaps = dimension.getDimValueMaps();
|
||||
Map<String, String> aliasAndBizNameToTechName = new HashMap<>();
|
||||
|
||||
List<DimValueMap> dimValueMaps = dimension.getDimValueMaps();
|
||||
Map<String, String> innerPairTech = new HashMap<>();
|
||||
Map<String, String> innerPairBiz = new HashMap<>();
|
||||
|
||||
dimValueMaps.stream().forEach(dimValueMap -> {
|
||||
if (Objects.nonNull(dimValueMap) && !CollectionUtils.isEmpty(dimValueMap.getAlias())
|
||||
&& Strings.isNotEmpty(dimValueMap.getTechName())) {
|
||||
|
||||
// add bizName and techName pair
|
||||
if (Strings.isNotEmpty(dimValueMap.getBizName())) {
|
||||
innerPairTech.put(dimValueMap.getBizName(), dimValueMap.getTechName());
|
||||
}
|
||||
|
||||
dimValueMap.getAlias().stream().forEach(alias -> {
|
||||
if (Strings.isNotEmpty(alias)) {
|
||||
innerPairTech.put(alias, dimValueMap.getTechName());
|
||||
}
|
||||
});
|
||||
}
|
||||
if (Objects.nonNull(dimValueMap) && Strings.isNotEmpty(dimValueMap.getTechName())) {
|
||||
innerPairBiz.put(dimValueMap.getTechName(), dimValueMap.getBizName());
|
||||
}
|
||||
});
|
||||
|
||||
if (!CollectionUtils.isEmpty(innerPairTech)) {
|
||||
dimAndAliasAndTechNamePair.put(bizName, innerPairTech);
|
||||
for (DimValueMap dimValueMap : dimValueMaps) {
|
||||
if (needSkipDimValue(dimValueMap)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!CollectionUtils.isEmpty(innerPairBiz)) {
|
||||
dimAndTechNameAndBizNamePair.put(bizName, innerPairBiz);
|
||||
if (Strings.isNotEmpty(dimValueMap.getBizName())) {
|
||||
aliasAndBizNameToTechName.put(dimValueMap.getBizName(), dimValueMap.getTechName());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(dimValueMap.getAlias())) {
|
||||
dimValueMap.getAlias().stream().forEach(alias -> {
|
||||
if (Strings.isNotEmpty(alias)) {
|
||||
aliasAndBizNameToTechName.put(alias, dimValueMap.getTechName());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (!CollectionUtils.isEmpty(aliasAndBizNameToTechName)) {
|
||||
result.put(bizName, aliasAndBizNameToTechName);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean needSkipDimValue(DimValueMap dimValueMap) {
|
||||
return Objects.isNull(dimValueMap) || Strings.isEmpty(dimValueMap.getTechName());
|
||||
}
|
||||
|
||||
|
||||
private Map<String, Map<String, String>> getTechNameToBizName(List<DimensionResp> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensions)) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
Map<String, Map<String, String>> result = new HashMap<>();
|
||||
for (DimensionResp dimension : dimensions) {
|
||||
if (needSkipDimension(dimension)) {
|
||||
continue;
|
||||
}
|
||||
String bizName = dimension.getBizName();
|
||||
List<DimValueMap> dimValueMaps = dimension.getDimValueMaps();
|
||||
Map<String, String> techNameToBizName = new HashMap<>();
|
||||
|
||||
for (DimValueMap dimValueMap : dimValueMaps) {
|
||||
if (needSkipDimValue(dimValueMap)) {
|
||||
continue;
|
||||
}
|
||||
if (StringUtils.isNotEmpty(dimValueMap.getBizName())) {
|
||||
techNameToBizName.put(dimValueMap.getTechName(), dimValueMap.getBizName());
|
||||
}
|
||||
}
|
||||
|
||||
if (!CollectionUtils.isEmpty(techNameToBizName)) {
|
||||
result.put(bizName, techNameToBizName);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean needSkipDimension(DimensionResp dimension) {
|
||||
return Objects.isNull(dimension) || Strings.isEmpty(dimension.getBizName()) || CollectionUtils.isEmpty(
|
||||
dimension.getDimValueMaps());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,10 +5,10 @@ import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.semantic.api.query.pojo.Criterion;
|
||||
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
package com.tencent.supersonic.semantic.query.utils;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
|
||||
@@ -4,13 +4,13 @@ import com.alibaba.ttl.TransmittableThreadLocal;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeBackEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
|
||||
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.semantic.query.persistence.repository.StatRepository;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<mapper namespace="com.tencent.supersonic.semantic.query.persistence.mapper.StatMapper">
|
||||
|
||||
<resultMap id="QueryStatDO"
|
||||
type="com.tencent.supersonic.semantic.api.model.pojo.QueryStat">
|
||||
type="com.tencent.supersonic.semantic.api.model.pojo.QueryStat">
|
||||
<id column="id" property="id"/>
|
||||
<result column="trace_id" property="traceId"/>
|
||||
<result column="model_id" property="modelId"/>
|
||||
@@ -40,18 +40,17 @@
|
||||
|
||||
<insert id="createRecord">
|
||||
insert into s2_query_stat_info
|
||||
(trace_id, model_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5,
|
||||
query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
|
||||
elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics,
|
||||
select_cols, agg_cols, filter_cols, group_by_cols,
|
||||
order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key)
|
||||
values (#{traceId}, #{modelId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd},
|
||||
#{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5},
|
||||
#{queryEngine},
|
||||
#{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate},
|
||||
#{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols},
|
||||
#{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey},
|
||||
#{resultCacheKey})
|
||||
(
|
||||
trace_id, model_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
|
||||
elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics, select_cols, agg_cols, filter_cols, group_by_cols,
|
||||
order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key
|
||||
)
|
||||
values
|
||||
(
|
||||
#{traceId}, #{modelId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
|
||||
#{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate}, #{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols},
|
||||
#{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey}, #{resultCacheKey}
|
||||
)
|
||||
</insert>
|
||||
|
||||
<select id="getStatInfo"
|
||||
|
||||
@@ -1,35 +1,34 @@
|
||||
CREATE TABLE `s2_query_stat_info`
|
||||
(
|
||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`trace_id` varchar(200) DEFAULT NULL COMMENT '查询标识',
|
||||
`domain_id` bigint(20) DEFAULT NULL COMMENT '主题域ID',
|
||||
`user` varchar(200) DEFAULT NULL COMMENT '执行sql的用户',
|
||||
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
`query_type` varchar(200) DEFAULT NULL COMMENT '查询对应的场景',
|
||||
`query_type_back` int(10) DEFAULT '0' COMMENT '查询类型, 0-正常查询, 1-预刷类型',
|
||||
`query_sql_cmd` mediumtext COMMENT '对应查询的struct',
|
||||
`sql_cmd_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
|
||||
`query_struct_cmd` mediumtext COMMENT '对应查询的struct',
|
||||
`struct_cmd_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
|
||||
`sql` mediumtext COMMENT '对应查询的sql',
|
||||
`sql_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
|
||||
`query_engine` varchar(20) DEFAULT NULL COMMENT '查询引擎',
|
||||
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT '查询耗时',
|
||||
`query_state` varchar(20) DEFAULT NULL COMMENT '查询最终状态',
|
||||
`native_query` int(10) DEFAULT NULL COMMENT '1-明细查询,0-聚合查询',
|
||||
`start_date` varchar(50) DEFAULT NULL COMMENT 'sql开始日期',
|
||||
`end_date` varchar(50) DEFAULT NULL COMMENT 'sql结束日期',
|
||||
`dimensions` mediumtext COMMENT 'sql 涉及的维度',
|
||||
`metrics` mediumtext COMMENT 'sql 涉及的指标',
|
||||
`select_cols` mediumtext COMMENT 'sql select部分涉及的标签',
|
||||
`agg_cols` mediumtext COMMENT 'sql agg部分涉及的标签',
|
||||
`filter_cols` mediumtext COMMENT 'sql where部分涉及的标签',
|
||||
`group_by_cols` mediumtext COMMENT 'sql grouy by部分涉及的标签',
|
||||
`order_by_cols` mediumtext COMMENT 'sql order by部分涉及的标签',
|
||||
`use_result_cache` tinyint(1) DEFAULT '-1' COMMENT '是否命中sql缓存',
|
||||
`use_sql_cache` tinyint(1) DEFAULT '-1' COMMENT '是否命中sql缓存',
|
||||
`sql_cache_key` mediumtext COMMENT '缓存的key',
|
||||
`result_cache_key` mediumtext COMMENT '缓存的key',
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `domain_index` (`domain_id`)
|
||||
CREATE TABLE `s2_query_stat_info` (
|
||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`trace_id` varchar(200) DEFAULT NULL COMMENT '查询标识',
|
||||
`domain_id` bigint(20) DEFAULT NULL COMMENT '主题域ID',
|
||||
`user` varchar(200) DEFAULT NULL COMMENT '执行sql的用户',
|
||||
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
`query_type` varchar(200) DEFAULT NULL COMMENT '查询对应的场景',
|
||||
`query_type_back` int(10) DEFAULT '0' COMMENT '查询类型, 0-正常查询, 1-预刷类型',
|
||||
`query_sql_cmd` mediumtext COMMENT '对应查询的struct',
|
||||
`sql_cmd_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
|
||||
`query_struct_cmd` mediumtext COMMENT '对应查询的struct',
|
||||
`struct_cmd_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
|
||||
`sql` mediumtext COMMENT '对应查询的sql',
|
||||
`sql_md5` varchar(200) DEFAULT NULL COMMENT 'sql md5值',
|
||||
`query_engine` varchar(20) DEFAULT NULL COMMENT '查询引擎',
|
||||
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT '查询耗时',
|
||||
`query_state` varchar(20) DEFAULT NULL COMMENT '查询最终状态',
|
||||
`native_query` int(10) DEFAULT NULL COMMENT '1-明细查询,0-聚合查询',
|
||||
`start_date` varchar(50) DEFAULT NULL COMMENT 'sql开始日期',
|
||||
`end_date` varchar(50) DEFAULT NULL COMMENT 'sql结束日期',
|
||||
`dimensions` mediumtext COMMENT 'sql 涉及的维度',
|
||||
`metrics` mediumtext COMMENT 'sql 涉及的指标',
|
||||
`select_cols` mediumtext COMMENT 'sql select部分涉及的标签',
|
||||
`agg_cols` mediumtext COMMENT 'sql agg部分涉及的标签',
|
||||
`filter_cols` mediumtext COMMENT 'sql where部分涉及的标签',
|
||||
`group_by_cols` mediumtext COMMENT 'sql grouy by部分涉及的标签',
|
||||
`order_by_cols` mediumtext COMMENT 'sql order by部分涉及的标签',
|
||||
`use_result_cache` tinyint(1) DEFAULT '-1' COMMENT '是否命中sql缓存',
|
||||
`use_sql_cache` tinyint(1) DEFAULT '-1' COMMENT '是否命中sql缓存',
|
||||
`sql_cache_key` mediumtext COMMENT '缓存的key',
|
||||
`result_cache_key` mediumtext COMMENT '缓存的key',
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `domain_index` (`domain_id`)
|
||||
) COMMENT='查询统计信息表'
|
||||
|
||||
@@ -1,46 +1,18 @@
|
||||
CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info`
|
||||
(
|
||||
`id` bigint
|
||||
(
|
||||
20
|
||||
) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`trace_id` varchar
|
||||
(
|
||||
200
|
||||
) NOT NULL COMMENT '查询标识' ,
|
||||
`domain_id` bigint
|
||||
(
|
||||
20
|
||||
) NOT NULL COMMENT '主题域ID',
|
||||
CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info` (
|
||||
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`trace_id` varchar(200) NOT NULL COMMENT '查询标识' ,
|
||||
`domain_id` bigint(20) NOT NULL COMMENT '主题域ID',
|
||||
`dimensions` mediumtext COMMENT '查询相关的维度信息',
|
||||
`metrics` mediumtext COMMENT '查询相关的指标信息',
|
||||
`orders` mediumtext COMMENT '查询相关的排序信息',
|
||||
`filters` mediumtext COMMENT '查询相关的过滤信息',
|
||||
`date_info` mediumtext COMMENT '查询相关的日期信息',
|
||||
`limit` bigint
|
||||
(
|
||||
20
|
||||
) NOT NULL COMMENT '查询相关的limit信息',
|
||||
`native_query` tinyint
|
||||
(
|
||||
1
|
||||
) NOT NULL DEFAULT '0' COMMENT '1-明细查询,0-聚合查询',
|
||||
`limit` bigint(20) NOT NULL COMMENT'查询相关的limit信息',
|
||||
`native_query` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1-明细查询,0-聚合查询',
|
||||
`sql` mediumtext COMMENT '解析后的sql',
|
||||
`created_at` datetime NOT NULL COMMENT '创建时间',
|
||||
`created_by` varchar
|
||||
(
|
||||
100
|
||||
) NOT NULL COMMENT '创建人',
|
||||
`status` int
|
||||
(
|
||||
10
|
||||
) NOT NULL COMMENT '运行状态',
|
||||
`elapsed_ms` bigint
|
||||
(
|
||||
10
|
||||
) DEFAULT NULL COMMENT 'sql解析耗时',
|
||||
PRIMARY KEY
|
||||
(
|
||||
`id`
|
||||
)
|
||||
) COMMENT='语义层sql解析信息表'
|
||||
`created_at` datetime NOT NULL COMMENT '创建时间',
|
||||
`created_by` varchar(100) NOT NULL COMMENT '创建人',
|
||||
`status` int(10) NOT NULL COMMENT '运行状态',
|
||||
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT 'sql解析耗时',
|
||||
PRIMARY KEY (`id`)
|
||||
)COMMENT='语义层sql解析信息表'
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package com.tencent.supersonic.semantic.query.domain.calcite;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.ColumnOrder;
|
||||
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
|
||||
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
|
||||
import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl;
|
||||
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
|
||||
@@ -9,7 +7,9 @@ import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl;
|
||||
import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
|
||||
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
|
||||
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
|
||||
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
|
||||
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
|
||||
import com.tencent.supersonic.common.pojo.ColumnOrder;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
|
||||
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
|
||||
|
||||
Reference in New Issue
Block a user