topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 21:17:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)(chat) Add views and adapt chat and headless (#700)

Browse Source 
* (improvement)(headless)(chat) Add views and adapt chat and headless

---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-01-30 20:43:53 +08:00
committed by GitHub
parent 31f8c1df35
commit 24b442baef
237 changed files with 3205 additions and 4479 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java
View File

@@ -16,13 +16,9 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
@@ -46,11 +42,6 @@ public class AuthCheckBaseAspect {
new SimpleDateFormat(Constants.DAY_FORMAT));
@Autowired
private AuthService authService;
@Autowired
private DimensionService dimensionService;
@Autowired
private MetricService metricService;
@Autowired
private ModelService modelService;
@@ -85,18 +76,17 @@ public class AuthCheckBaseAspect {
}
public Set<String> getHighSensitiveColsByModelId(List<Long> modelIds) {
public Set<String> getHighSensitiveColsByModelId(SemanticSchemaResp semanticSchemaResp) {
Set<String> highSensitiveCols = new HashSet<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(modelIds);
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
List<DimensionResp> highSensitiveDimensions = dimensionService.getDimensions(metaFilter);
List<MetricResp> highSensitiveMetrics = metricService.getMetrics(metaFilter);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) {
semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp ->
SensitiveLevelEnum.HIGH.getCode().equals(dimSchemaResp.getSensitiveLevel()))
.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
if (!CollectionUtils.isEmpty(highSensitiveMetrics)) {
highSensitiveMetrics.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
if (!CollectionUtils.isEmpty(semanticSchemaResp.getMetrics())) {
semanticSchemaResp.getMetrics().stream().filter(metricSchemaResp ->
SensitiveLevelEnum.HIGH.getCode().equals(metricSchemaResp.getSensitiveLevel()))
.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
}
return highSensitiveCols;
}

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
@@ -17,13 +18,6 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
@@ -35,6 +29,14 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Aspect
@Component
@Slf4j
@@ -90,7 +92,7 @@ public class DimValueAspect {
log.info("correctorSql before replacing:{}", sql);
List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Set<String> fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet());
Set<String> fieldNames = dimensions.stream().map(SchemaItem::getName).collect(Collectors.toSet());
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
for (FieldExpression expression : fieldExpressionList) {
if (!fieldNames.contains(expression.getFieldName())) {

70
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.headless.server.aspect;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -12,28 +10,20 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -48,6 +38,15 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Component
@Aspect
@@ -108,23 +107,24 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
throws Throwable {
Object[] objects = joinPoint.getArgs();
User user = (User) objects[1];
List<Long> modelIds = querySqlReq.getModelIds();
// fetch data permission meta information
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, modelSchemaRespList);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
SchemaFilterReq filter = new SchemaFilterReq();
filter.setModelIds(querySqlReq.getModelIds());
filter.setViewId(querySqlReq.getViewId());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
List<Long> modelIdInView = semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, semanticSchemaResp);
log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIds, sensitiveResReq);
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInView, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIds);
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq);
@@ -138,7 +138,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource);
}
// if the column has no permission, hit *
@@ -147,7 +147,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
@@ -207,21 +207,25 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
Object[] args = point.getArgs();
User user = (User) args[1];
// fetch data permission meta information
List<Long> modelIds = queryStructReq.getModelIds();
SchemaFilterReq filter = new SchemaFilterReq();
filter.setModelIds(queryStructReq.getModelIds());
filter.setViewId(queryStructReq.getViewId());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
List<Long> modelIdInView = semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq);
log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInView, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
modelIds, sensitiveResReq);
modelIdInView, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource,
queryStructReq.getModelIds());
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, queryStructReq.getModelIds());
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
@@ -235,7 +239,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource);
}
// if the column has no permission, hit *
@@ -243,7 +247,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
.collect(Collectors.toSet());
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java
View File

@@ -8,9 +8,6 @@ import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.util.ComponentFactory;
import com.tencent.supersonic.common.util.embedding.EmbeddingQuery;
import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@@ -19,6 +16,10 @@ import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Component
@Slf4j
public class MetaEmbeddingListener implements ApplicationListener<DataEvent> {
@@ -42,7 +43,8 @@ public class MetaEmbeddingListener implements ApplicationListener<DataEvent> {
.map(dataItem -> {
EmbeddingQuery embeddingQuery = new EmbeddingQuery();
embeddingQuery.setQueryId(
dataItem.getId().toString() + DictWordType.NATURE_SPILT + dataItem.getType().getName());
dataItem.getId().toString() + DictWordType.NATURE_SPILT
+ dataItem.getType().name().toLowerCase());
embeddingQuery.setQuery(dataItem.getName());
Map meta = JSONObject.parseObject(JSONObject.toJSONString(dataItem), Map.class);
embeddingQuery.setMetadata(meta);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
View File

@@ -1,21 +1,20 @@
package com.tencent.supersonic.headless.server.manager;
import com.tencent.supersonic.headless.api.pojo.enums.DatasourceQuery;
import com.tencent.supersonic.headless.api.pojo.enums.ModelSourceType;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.enums.DatasourceQuery;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@@ -47,7 +46,6 @@ public class ModelYamlManager {
.collect(Collectors.toList()));
dataModelYamlTpl.setName(modelResp.getBizName());
dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
dataModelYamlTpl.setModelSourceTypeEnum(ModelSourceType.of(modelResp.getSourceType()));
if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) {
dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery());
} else {

91
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -1,12 +1,10 @@
package com.tencent.supersonic.headless.server.manager;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.Field;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
@@ -31,8 +29,12 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -41,50 +43,31 @@ import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j
@Service
public class SemanticSchemaManager {
@Autowired
private LoadingCache<String, SemanticModel> loadingCache;
private final Catalog catalog;
public SemanticSchemaManager(Catalog catalog) {
this.catalog = catalog;
}
public SemanticModel reload(String rootPath) {
public SemanticModel getSemanticModel(SemanticSchemaResp semanticSchemaResp) {
SemanticModel semanticModel = new SemanticModel();
semanticModel.setRootPath(rootPath);
Set<Long> modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim()))
.collect(Collectors.toSet());
if (modelIds.isEmpty()) {
log.error("get modelIds empty {}", rootPath);
return semanticModel;
}
semanticModel.setSchemaKey(semanticSchemaResp.getSchemaKey());
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
Map<Long, String> modelIdName = new HashMap<>();
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next());
catalog.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls,
dataModelYamlTpls, metricYamlTpls, modelIdName);
DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp();
semanticModel.setDatabase(DatabaseConverter.convert(databaseResp));
List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
if (!CollectionUtils.isEmpty(modelRelas)) {
semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) {
semanticModel.setJoinRelations(getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource)
@@ -104,15 +87,6 @@ public class SemanticSchemaManager {
return semanticModel;
}
public SemanticModel get(String rootPath) throws Exception {
rootPath = formatKey(rootPath);
SemanticModel schema = loadingCache.get(rootPath);
if (schema == null) {
return null;
}
return schema;
}
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
return getMetricsByMetricYamlTpl(t);
}
@@ -351,43 +325,4 @@ public class SemanticSchemaManager {
metrics.addAll(metricYamlTpls);
}
public static String formatKey(String key) {
key = key.trim();
if (key.startsWith("/")) {
key = key.substring(1);
}
if (key.endsWith("/")) {
key = key.substring(0, key.length() - 1);
}
return key;
}
@Configuration
@EnableCaching
public class GuavaCacheConfig {
@Value("${parser.cache.saveMinute:1}")
private Integer saveMinutes = 1;
@Value("${parser.cache.maximumSize:1000}")
private Integer maximumSize = 1000;
@Bean
public LoadingCache<String, SemanticModel> getCache() {
LoadingCache<String, SemanticModel> cache
= CacheBuilder.newBuilder()
.expireAfterWrite(saveMinutes, TimeUnit.MINUTES)
.initialCapacity(10)
.maximumSize(maximumSize).build(
new CacheLoader<String, SemanticModel>() {
@Override
public SemanticModel load(String key) {
log.info("load SemanticSchema [{}]", key);
return SemanticSchemaManager.this.reload(key);
}
}
);
return cache;
}
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDO.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java
View File

@@ -8,8 +8,8 @@ import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_view_info")
public class ViewInfoDO {
@TableName("s2_canvas")
public class CanvasDO {
/**
*/
@TableId(type = IdType.AUTO)

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java Normal file
View File

@@ -0,0 +1,46 @@
package com.tencent.supersonic.headless.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_view")
public class ViewDO {
@TableId(type = IdType.AUTO)
private Long id;
private Long domainId;
private String name;
private String bizName;
private String description;
private Integer status;
private String alias;
private String viewDetail;
private Date createdAt;
private String createdBy;
private Date updatedAt;
private String updatedBy;
private String filterSql;
private String queryConfig;
private String admin;
private String adminOrg;
}

740
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewInfoDOExample.java
View File

@@ -1,740 +0,0 @@
package com.tencent.supersonic.headless.server.persistence.dataobject;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
public class ViewInfoDOExample {
/**
* s2_view_info
*/
protected String orderByClause;
/**
* s2_view_info
*/
protected boolean distinct;
/**
* s2_view_info
*/
protected List<Criteria> oredCriteria;
/**
* s2_view_info
*/
protected Integer limitStart;
/**
* s2_view_info
*/
protected Integer limitEnd;
/**
* @mbg.generated
*/
public ViewInfoDOExample() {
oredCriteria = new ArrayList<Criteria>();
}
/**
* @mbg.generated
*/
public void setOrderByClause(String orderByClause) {
this.orderByClause = orderByClause;
}
/**
* @mbg.generated
*/
public String getOrderByClause() {
return orderByClause;
}
/**
* @mbg.generated
*/
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
/**
* @mbg.generated
*/
public boolean isDistinct() {
return distinct;
}
/**
* @mbg.generated
*/
public List<Criteria> getOredCriteria() {
return oredCriteria;
}
/**
* @mbg.generated
*/
public void or(Criteria criteria) {
oredCriteria.add(criteria);
}
/**
* @mbg.generated
*/
public Criteria or() {
Criteria criteria = createCriteriaInternal();
oredCriteria.add(criteria);
return criteria;
}
/**
* @mbg.generated
*/
public Criteria createCriteria() {
Criteria criteria = createCriteriaInternal();
if (oredCriteria.size() == 0) {
oredCriteria.add(criteria);
}
return criteria;
}
/**
* @mbg.generated
*/
protected Criteria createCriteriaInternal() {
Criteria criteria = new Criteria();
return criteria;
}
/**
* @mbg.generated
*/
public void clear() {
oredCriteria.clear();
orderByClause = null;
distinct = false;
}
/**
* @mbg.generated
*/
public void setLimitStart(Integer limitStart) {
this.limitStart = limitStart;
}
/**
* @mbg.generated
*/
public Integer getLimitStart() {
return limitStart;
}
/**
* @mbg.generated
*/
public void setLimitEnd(Integer limitEnd) {
this.limitEnd = limitEnd;
}
/**
* @mbg.generated
*/
public Integer getLimitEnd() {
return limitEnd;
}
/**
* s2_view_info null
*/
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
protected GeneratedCriteria() {
super();
criteria = new ArrayList<Criterion>();
}
public boolean isValid() {
return criteria.size() > 0;
}
public List<Criterion> getAllCriteria() {
return criteria;
}
public List<Criterion> getCriteria() {
return criteria;
}
protected void addCriterion(String condition) {
if (condition == null) {
throw new RuntimeException("Value for condition cannot be null");
}
criteria.add(new Criterion(condition));
}
protected void addCriterion(String condition, Object value, String property) {
if (value == null) {
throw new RuntimeException("Value for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(String condition, Object value1, Object value2, String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value1, value2));
}
public Criteria andIdIsNull() {
addCriterion("id is null");
return (Criteria) this;
}
public Criteria andIdIsNotNull() {
addCriterion("id is not null");
return (Criteria) this;
}
public Criteria andIdEqualTo(Long value) {
addCriterion("id =", value, "id");
return (Criteria) this;
}
public Criteria andIdNotEqualTo(Long value) {
addCriterion("id <>", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThan(Long value) {
addCriterion("id >", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThanOrEqualTo(Long value) {
addCriterion("id >=", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThan(Long value) {
addCriterion("id <", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThanOrEqualTo(Long value) {
addCriterion("id <=", value, "id");
return (Criteria) this;
}
public Criteria andIdIn(List<Long> values) {
addCriterion("id in", values, "id");
return (Criteria) this;
}
public Criteria andIdNotIn(List<Long> values) {
addCriterion("id not in", values, "id");
return (Criteria) this;
}
public Criteria andIdBetween(Long value1, Long value2) {
addCriterion("id between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andIdNotBetween(Long value1, Long value2) {
addCriterion("id not between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andModelIdIsNull() {
addCriterion("model_id is null");
return (Criteria) this;
}
public Criteria andModelIdIsNotNull() {
addCriterion("model_id is not null");
return (Criteria) this;
}
public Criteria andModelIdEqualTo(Long value) {
addCriterion("model_id =", value, "modelId");
return (Criteria) this;
}
public Criteria andModelIdNotEqualTo(Long value) {
addCriterion("model_id <>", value, "modelId");
return (Criteria) this;
}
public Criteria andModelIdGreaterThan(Long value) {
addCriterion("model_id >", value, "modelId");
return (Criteria) this;
}
public Criteria andModelIdGreaterThanOrEqualTo(Long value) {
addCriterion("model_id >=", value, "modelId");
return (Criteria) this;
}
public Criteria andModelIdLessThan(Long value) {
addCriterion("model_id <", value, "modelId");
return (Criteria) this;
}
public Criteria andModelIdLessThanOrEqualTo(Long value) {
addCriterion("model_id <=", value, "modelId");
return (Criteria) this;
}
public Criteria andModelIdIn(List<Long> values) {
addCriterion("model_id in", values, "modelId");
return (Criteria) this;
}
public Criteria andModelIdNotIn(List<Long> values) {
addCriterion("model_id not in", values, "modelId");
return (Criteria) this;
}
public Criteria andModelIdBetween(Long value1, Long value2) {
addCriterion("model_id between", value1, value2, "modelId");
return (Criteria) this;
}
public Criteria andModelIdNotBetween(Long value1, Long value2) {
addCriterion("model_id not between", value1, value2, "modelId");
return (Criteria) this;
}
public Criteria andTypeIsNull() {
addCriterion("type is null");
return (Criteria) this;
}
public Criteria andTypeIsNotNull() {
addCriterion("type is not null");
return (Criteria) this;
}
public Criteria andTypeEqualTo(String value) {
addCriterion("type =", value, "type");
return (Criteria) this;
}
public Criteria andTypeNotEqualTo(String value) {
addCriterion("type <>", value, "type");
return (Criteria) this;
}
public Criteria andTypeGreaterThan(String value) {
addCriterion("type >", value, "type");
return (Criteria) this;
}
public Criteria andTypeGreaterThanOrEqualTo(String value) {
addCriterion("type >=", value, "type");
return (Criteria) this;
}
public Criteria andTypeLessThan(String value) {
addCriterion("type <", value, "type");
return (Criteria) this;
}
public Criteria andTypeLessThanOrEqualTo(String value) {
addCriterion("type <=", value, "type");
return (Criteria) this;
}
public Criteria andTypeLike(String value) {
addCriterion("type like", value, "type");
return (Criteria) this;
}
public Criteria andTypeNotLike(String value) {
addCriterion("type not like", value, "type");
return (Criteria) this;
}
public Criteria andTypeIn(List<String> values) {
addCriterion("type in", values, "type");
return (Criteria) this;
}
public Criteria andTypeNotIn(List<String> values) {
addCriterion("type not in", values, "type");
return (Criteria) this;
}
public Criteria andTypeBetween(String value1, String value2) {
addCriterion("type between", value1, value2, "type");
return (Criteria) this;
}
public Criteria andTypeNotBetween(String value1, String value2) {
addCriterion("type not between", value1, value2, "type");
return (Criteria) this;
}
public Criteria andCreatedAtIsNull() {
addCriterion("created_at is null");
return (Criteria) this;
}
public Criteria andCreatedAtIsNotNull() {
addCriterion("created_at is not null");
return (Criteria) this;
}
public Criteria andCreatedAtEqualTo(Date value) {
addCriterion("created_at =", value, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtNotEqualTo(Date value) {
addCriterion("created_at <>", value, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtGreaterThan(Date value) {
addCriterion("created_at >", value, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtGreaterThanOrEqualTo(Date value) {
addCriterion("created_at >=", value, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtLessThan(Date value) {
addCriterion("created_at <", value, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtLessThanOrEqualTo(Date value) {
addCriterion("created_at <=", value, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtIn(List<Date> values) {
addCriterion("created_at in", values, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtNotIn(List<Date> values) {
addCriterion("created_at not in", values, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtBetween(Date value1, Date value2) {
addCriterion("created_at between", value1, value2, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedAtNotBetween(Date value1, Date value2) {
addCriterion("created_at not between", value1, value2, "createdAt");
return (Criteria) this;
}
public Criteria andCreatedByIsNull() {
addCriterion("created_by is null");
return (Criteria) this;
}
public Criteria andCreatedByIsNotNull() {
addCriterion("created_by is not null");
return (Criteria) this;
}
public Criteria andCreatedByEqualTo(String value) {
addCriterion("created_by =", value, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByNotEqualTo(String value) {
addCriterion("created_by <>", value, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByGreaterThan(String value) {
addCriterion("created_by >", value, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByGreaterThanOrEqualTo(String value) {
addCriterion("created_by >=", value, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByLessThan(String value) {
addCriterion("created_by <", value, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByLessThanOrEqualTo(String value) {
addCriterion("created_by <=", value, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByLike(String value) {
addCriterion("created_by like", value, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByNotLike(String value) {
addCriterion("created_by not like", value, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByIn(List<String> values) {
addCriterion("created_by in", values, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByNotIn(List<String> values) {
addCriterion("created_by not in", values, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByBetween(String value1, String value2) {
addCriterion("created_by between", value1, value2, "createdBy");
return (Criteria) this;
}
public Criteria andCreatedByNotBetween(String value1, String value2) {
addCriterion("created_by not between", value1, value2, "createdBy");
return (Criteria) this;
}
public Criteria andUpdatedAtIsNull() {
addCriterion("updated_at is null");
return (Criteria) this;
}
public Criteria andUpdatedAtIsNotNull() {
addCriterion("updated_at is not null");
return (Criteria) this;
}
public Criteria andUpdatedAtEqualTo(Date value) {
addCriterion("updated_at =", value, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtNotEqualTo(Date value) {
addCriterion("updated_at <>", value, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtGreaterThan(Date value) {
addCriterion("updated_at >", value, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtGreaterThanOrEqualTo(Date value) {
addCriterion("updated_at >=", value, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtLessThan(Date value) {
addCriterion("updated_at <", value, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtLessThanOrEqualTo(Date value) {
addCriterion("updated_at <=", value, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtIn(List<Date> values) {
addCriterion("updated_at in", values, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtNotIn(List<Date> values) {
addCriterion("updated_at not in", values, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtBetween(Date value1, Date value2) {
addCriterion("updated_at between", value1, value2, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedAtNotBetween(Date value1, Date value2) {
addCriterion("updated_at not between", value1, value2, "updatedAt");
return (Criteria) this;
}
public Criteria andUpdatedByIsNull() {
addCriterion("updated_by is null");
return (Criteria) this;
}
public Criteria andUpdatedByIsNotNull() {
addCriterion("updated_by is not null");
return (Criteria) this;
}
public Criteria andUpdatedByEqualTo(String value) {
addCriterion("updated_by =", value, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByNotEqualTo(String value) {
addCriterion("updated_by <>", value, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByGreaterThan(String value) {
addCriterion("updated_by >", value, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByGreaterThanOrEqualTo(String value) {
addCriterion("updated_by >=", value, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByLessThan(String value) {
addCriterion("updated_by <", value, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByLessThanOrEqualTo(String value) {
addCriterion("updated_by <=", value, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByLike(String value) {
addCriterion("updated_by like", value, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByNotLike(String value) {
addCriterion("updated_by not like", value, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByIn(List<String> values) {
addCriterion("updated_by in", values, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByNotIn(List<String> values) {
addCriterion("updated_by not in", values, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByBetween(String value1, String value2) {
addCriterion("updated_by between", value1, value2, "updatedBy");
return (Criteria) this;
}
public Criteria andUpdatedByNotBetween(String value1, String value2) {
addCriterion("updated_by not between", value1, value2, "updatedBy");
return (Criteria) this;
}
}
/**
* s2_view_info
*/
public static class Criteria extends GeneratedCriteria {
protected Criteria() {
super();
}
}
/**
* s2_view_info null
*/
public static class Criterion {
private String condition;
private Object value;
private Object secondValue;
private boolean noValue;
private boolean singleValue;
private boolean betweenValue;
private boolean listValue;
private String typeHandler;
protected Criterion(String condition) {
super();
this.condition = condition;
this.typeHandler = null;
this.noValue = true;
}
protected Criterion(String condition, Object value, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.typeHandler = typeHandler;
if (value instanceof List<?>) {
this.listValue = true;
} else {
this.singleValue = true;
}
}
protected Criterion(String condition, Object value) {
this(condition, value, null);
}
protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.secondValue = secondValue;
this.typeHandler = typeHandler;
this.betweenValue = true;
}
protected Criterion(String condition, Object value, Object secondValue) {
this(condition, value, secondValue, null);
}
public String getCondition() {
return condition;
}
public Object getValue() {
return value;
}
public Object getSecondValue() {
return secondValue;
}
public boolean isNoValue() {
return noValue;
}
public boolean isSingleValue() {
return singleValue;
}
public boolean isBetweenValue() {
return betweenValue;
}
public boolean isListValue() {
return listValue;
}
public String getTypeHandler() {
return typeHandler;
}
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewInfoDOMapper.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ViewInfoDOMapper extends BaseMapper<ViewInfoDO> {
public interface CanvasDOMapper extends BaseMapper<CanvasDO> {
}

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java Normal file
View File

@@ -0,0 +1,11 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ViewDOMapper extends BaseMapper<ViewDO> {
}

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/CanvasRepository.java Normal file
View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO;
import java.util.List;
public interface CanvasRepository {
List<CanvasDO> getCanvasList(Long domainId);
CanvasDO getCanvasById(Long id);
void deleteCanvas(Long id);
void createCanvas(CanvasDO canvasDO);
void updateCanvas(CanvasDO canvasDO);
}

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ViewInfoRepository.java
View File

@@ -1,18 +0,0 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO;
import java.util.List;
public interface ViewInfoRepository {
List<ViewInfoDO> getViewInfoList(Long domainId);
ViewInfoDO getViewInfoById(Long id);
void deleteViewInfo(Long id);
void createViewInfo(ViewInfoDO viewInfoDO);
void updateViewInfo(ViewInfoDO viewInfoDO);
}

48
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/CanvasRepositoryImpl.java Normal file
View File

@@ -0,0 +1,48 @@
package com.tencent.supersonic.headless.server.persistence.repository.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO;
import com.tencent.supersonic.headless.server.persistence.mapper.CanvasDOMapper;
import com.tencent.supersonic.headless.server.persistence.repository.CanvasRepository;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
public class CanvasRepositoryImpl implements CanvasRepository {
private CanvasDOMapper canvasDOMapper;
public CanvasRepositoryImpl(CanvasDOMapper canvasDOMapper) {
this.canvasDOMapper = canvasDOMapper;
}
@Override
public List<CanvasDO> getCanvasList(Long domainId) {
QueryWrapper<CanvasDO> wrapper = new QueryWrapper<>();
wrapper.lambda().eq(CanvasDO::getDomainId, domainId);
return canvasDOMapper.selectList(wrapper);
}
@Override
public CanvasDO getCanvasById(Long id) {
return canvasDOMapper.selectById(id);
}
@Override
public void deleteCanvas(Long id) {
canvasDOMapper.deleteById(id);
}
@Override
public void createCanvas(CanvasDO canvasDO) {
canvasDOMapper.insert(canvasDO);
}
@Override
public void updateCanvas(CanvasDO canvasDO) {
canvasDOMapper.updateById(canvasDO);
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java
View File

@@ -48,8 +48,8 @@ public class StatRepositoryImpl implements StatRepository {
statInfos.stream().forEach(stat -> {
String dimensions = stat.getDimensions();
String metrics = stat.getMetrics();
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.getName(), stat.getModelId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.getName(), stat.getModelId());
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getModelId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getModelId());
});
map.forEach((k, v) -> {

48
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ViewInfoRepositoryImpl.java
View File

@@ -1,48 +0,0 @@
package com.tencent.supersonic.headless.server.persistence.repository.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO;
import com.tencent.supersonic.headless.server.persistence.mapper.ViewInfoDOMapper;
import com.tencent.supersonic.headless.server.persistence.repository.ViewInfoRepository;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
public class ViewInfoRepositoryImpl implements ViewInfoRepository {
private ViewInfoDOMapper viewInfoDOMapper;
public ViewInfoRepositoryImpl(ViewInfoDOMapper viewInfoDOMapper) {
this.viewInfoDOMapper = viewInfoDOMapper;
}
@Override
public List<ViewInfoDO> getViewInfoList(Long domainId) {
QueryWrapper<ViewInfoDO> wrapper = new QueryWrapper<>();
wrapper.lambda().eq(ViewInfoDO::getDomainId, domainId);
return viewInfoDOMapper.selectList(wrapper);
}
@Override
public ViewInfoDO getViewInfoById(Long id) {
return viewInfoDOMapper.selectById(id);
}
@Override
public void deleteViewInfo(Long id) {
viewInfoDOMapper.deleteById(id);
}
@Override
public void createViewInfo(ViewInfoDO viewInfoDO) {
viewInfoDOMapper.insert(viewInfoDO);
}
@Override
public void updateViewInfo(ViewInfoDO viewInfoDO) {
viewInfoDOMapper.updateById(viewInfoDO);
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
View File

@@ -19,6 +19,10 @@ public class MetaFilter {
private List<Long> modelIds;
private Long domainId;
private Long viewId;
private Integer sensitiveLevel;
private Integer status;

57
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/CanvasController.java Normal file
View File

@@ -0,0 +1,57 @@
package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.CanvasReq;
import com.tencent.supersonic.headless.api.pojo.response.CanvasSchemaResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO;
import com.tencent.supersonic.headless.server.service.impl.CanvasServiceImpl;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/viewInfo")
public class CanvasController {
private CanvasServiceImpl canvasService;
public CanvasController(CanvasServiceImpl canvasService) {
this.canvasService = canvasService;
}
@PostMapping("/createOrUpdateViewInfo")
public CanvasDO createOrUpdateCanvas(@RequestBody CanvasReq canvasReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return canvasService.createOrUpdateCanvas(canvasReq, user);
}
@GetMapping("/getViewInfoList/{domainId}")
public List<CanvasDO> getCanvasList(@PathVariable("domainId") Long domainId) {
return canvasService.getCanvasList(domainId);
}
@DeleteMapping("/deleteViewInfo/{id}")
public void deleteCanvas(@PathVariable("id") Long id) {
canvasService.deleteCanvas(id);
}
@GetMapping("/getDomainSchemaRela/{domainId}")
public List<CanvasSchemaResp> getDomainSchema(@PathVariable("domainId") Long domainId,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return canvasService.getCanvasSchema(domainId, user);
}
}

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java
View File

@@ -4,15 +4,15 @@ import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.server.service.SchemaService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
@@ -34,11 +34,8 @@ public class SchemaController {
private SchemaService schemaService;
@PostMapping
public List<ModelSchemaResp> fetchModelSchema(@RequestBody ModelSchemaFilterReq filter,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.fetchModelSchema(filter, user);
public List<ViewSchemaResp> fetchViewSchema(@RequestBody ViewFilterReq filter) {
return schemaService.fetchViewSchema(filter);
}
@GetMapping("/domain/list")

64
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java Normal file
View File

@@ -0,0 +1,64 @@
package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.ViewService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/view")
public class ViewController {
@Autowired
private ViewService viewService;
@PostMapping
public ViewResp save(@RequestBody ViewReq viewReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return viewService.save(viewReq, user);
}
@PutMapping
public ViewResp update(@RequestBody ViewReq viewReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return viewService.update(viewReq, user);
}
@GetMapping("/{id}")
public ViewResp getView(@PathVariable("id") Long id) {
return viewService.getView(id);
}
@GetMapping("/getViewList")
public List<ViewResp> getViewList(@RequestParam("domainId") Long domainId) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDomainId(domainId);
return viewService.getViewList(metaFilter);
}
@DeleteMapping("/{id}")
public Boolean delete(@PathVariable("id") Long id,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
viewService.delete(id, user);
return true;
}
}

57
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewInfoController.java
View File

@@ -1,57 +0,0 @@
package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.ViewInfoReq;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaRelaResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO;
import com.tencent.supersonic.headless.server.service.impl.ViewInfoServiceImpl;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/viewInfo")
public class ViewInfoController {
private ViewInfoServiceImpl viewInfoServiceImpl;
public ViewInfoController(ViewInfoServiceImpl viewInfoServiceImpl) {
this.viewInfoServiceImpl = viewInfoServiceImpl;
}
@PostMapping("/createOrUpdateViewInfo")
public ViewInfoDO createOrUpdateViewInfo(@RequestBody ViewInfoReq viewInfoReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return viewInfoServiceImpl.createOrUpdateViewInfo(viewInfoReq, user);
}
@GetMapping("/getViewInfoList/{domainId}")
public List<ViewInfoDO> getViewInfoList(@PathVariable("domainId") Long domainId) {
return viewInfoServiceImpl.getViewInfoList(domainId);
}
@DeleteMapping("/deleteViewInfo/{id}")
public void deleteViewInfo(@PathVariable("id") Long id) {
viewInfoServiceImpl.deleteViewInfo(id);
}
@GetMapping("/getDomainSchemaRela/{domainId}")
public List<ModelSchemaRelaResp> getDomainSchema(@PathVariable("domainId") Long domainId,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return viewInfoServiceImpl.getDomainSchema(domainId, user);
}
}

32
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java
View File

@@ -1,28 +1,25 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
public interface Catalog {
DatabaseResp getDatabase(Long id);
DatabaseResp getDatabaseByModelId(Long modelId);
DimensionResp getDimension(String bizName, Long modelId);
DimensionResp getDimension(Long id);
@@ -33,16 +30,17 @@ public interface Catalog {
MetricResp getMetric(Long id);
List<ModelRela> getModelRela(List<Long> modelIds);
List<ModelSchemaResp> getModelSchema(List<Long> modelIds);
void getModelYamlTplByModelIds(Set<Long> modelIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList, List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName);
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
List<ModelResp> getModelList(List<Long> modelIds);
void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList,
List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName);
SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) throws ExecutionException;
}

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java
View File

@@ -7,19 +7,13 @@ import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;
import java.util.Map;
import java.util.Set;
public interface ModelService {
@@ -27,7 +21,7 @@ public interface ModelService {
ModelResp updateModel(ModelReq datasourceReq, User user) throws Exception;
List<ModelResp> getModelList(ModelFilter modelFilter);
List<ModelResp> getModelList(MetaFilter metaFilter);
Map<Long, ModelResp> getModelMap();
@@ -47,16 +41,8 @@ public interface ModelService {
List<String> getModelAdmin(Long id);
ModelSchemaResp fetchSingleModelSchema(Long modelId);
List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq modelSchemaFilterReq);
DatabaseResp getDatabaseByModelId(Long modelId);
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
void getModelYamlTplByModelIds(Set<Long> modelIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList, List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName);
}

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java
View File

@@ -3,21 +3,27 @@ package com.tencent.supersonic.headless.server.service;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import java.util.List;
import java.util.concurrent.ExecutionException;
public interface SchemaService {
List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq filter, User user);
List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter);
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq, User user);
@@ -28,4 +34,10 @@ public interface SchemaService {
List<DomainResp> getDomainList(User user);
List<ModelResp> getModelList(User user, AuthType authType, Long domainId);
List<ViewResp> getViewList(Long domainId);
SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) throws ExecutionException;
}

21
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java Normal file
View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;
public interface ViewService {
ViewResp save(ViewReq viewReq, User user);
ViewResp update(ViewReq viewReq, User user);
ViewResp getView(Long id);
List<ViewResp> getViewList(MetaFilter metaFilter);
void delete(Long id, User user);
}

60
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewInfoServiceImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java
View File

@@ -4,13 +4,13 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.request.ViewInfoReq;
import com.tencent.supersonic.headless.api.pojo.request.CanvasReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaRelaResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewInfoDO;
import com.tencent.supersonic.headless.server.persistence.repository.ViewInfoRepository;
import com.tencent.supersonic.headless.api.pojo.response.CanvasSchemaResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO;
import com.tencent.supersonic.headless.server.persistence.repository.CanvasRepository;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
@@ -22,9 +22,9 @@ import java.util.Date;
import java.util.List;
@Service
public class ViewInfoServiceImpl {
public class CanvasServiceImpl {
private ViewInfoRepository viewInfoRepository;
private CanvasRepository viewInfoRepository;
private ModelService modelService;
@@ -32,58 +32,58 @@ public class ViewInfoServiceImpl {
private MetricService metricService;
public ViewInfoServiceImpl(ViewInfoRepository viewInfoRepository, ModelService modelService,
MetricService metricService, DimensionService dimensionService) {
public CanvasServiceImpl(CanvasRepository viewInfoRepository, ModelService modelService,
MetricService metricService, DimensionService dimensionService) {
this.viewInfoRepository = viewInfoRepository;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.modelService = modelService;
}
public List<ViewInfoDO> getViewInfoList(Long domainId) {
return viewInfoRepository.getViewInfoList(domainId);
public List<CanvasDO> getCanvasList(Long domainId) {
return viewInfoRepository.getCanvasList(domainId);
}
public List<ModelSchemaRelaResp> getDomainSchema(Long domainId, User user) {
List<ModelSchemaRelaResp> domainSchemaRelaResps = Lists.newArrayList();
public List<CanvasSchemaResp> getCanvasSchema(Long domainId, User user) {
List<CanvasSchemaResp> canvasSchemaResps = Lists.newArrayList();
List<ModelResp> modelResps = modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN);
for (ModelResp modelResp : modelResps) {
ModelSchemaRelaResp domainSchemaRelaResp = new ModelSchemaRelaResp();
CanvasSchemaResp canvasSchemaResp = new CanvasSchemaResp();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelResp.getId()));
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
domainSchemaRelaResp.setModel(modelResp);
domainSchemaRelaResp.setDimensions(dimensionResps);
domainSchemaRelaResp.setMetrics(metricResps);
domainSchemaRelaResp.setDomainId(domainId);
domainSchemaRelaResps.add(domainSchemaRelaResp);
canvasSchemaResp.setModel(modelResp);
canvasSchemaResp.setDimensions(dimensionResps);
canvasSchemaResp.setMetrics(metricResps);
canvasSchemaResp.setDomainId(domainId);
canvasSchemaResps.add(canvasSchemaResp);
}
return domainSchemaRelaResps;
return canvasSchemaResps;
}
public ViewInfoDO createOrUpdateViewInfo(ViewInfoReq viewInfoReq, User user) {
if (viewInfoReq.getId() == null) {
ViewInfoDO viewInfoDO = new ViewInfoDO();
BeanUtils.copyProperties(viewInfoReq, viewInfoDO);
public CanvasDO createOrUpdateCanvas(CanvasReq canvasReq, User user) {
if (canvasReq.getId() == null) {
CanvasDO viewInfoDO = new CanvasDO();
BeanUtils.copyProperties(canvasReq, viewInfoDO);
viewInfoDO.setCreatedAt(new Date());
viewInfoDO.setCreatedBy(user.getName());
viewInfoDO.setUpdatedAt(new Date());
viewInfoDO.setUpdatedBy(user.getName());
viewInfoRepository.createViewInfo(viewInfoDO);
viewInfoRepository.createCanvas(viewInfoDO);
return viewInfoDO;
}
Long id = viewInfoReq.getId();
ViewInfoDO viewInfoDO = viewInfoRepository.getViewInfoById(id);
BeanUtils.copyProperties(viewInfoReq, viewInfoDO);
Long id = canvasReq.getId();
CanvasDO viewInfoDO = viewInfoRepository.getCanvasById(id);
BeanUtils.copyProperties(canvasReq, viewInfoDO);
viewInfoDO.setUpdatedAt(new Date());
viewInfoDO.setUpdatedBy(user.getName());
viewInfoRepository.updateViewInfo(viewInfoDO);
viewInfoRepository.updateCanvas(viewInfoDO);
return viewInfoDO;
}
public void deleteViewInfo(Long id) {
viewInfoRepository.deleteViewInfo(id);
public void deleteCanvas(Long id) {
viewInfoRepository.deleteCanvas(id);
}
}

87
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java
View File

@@ -1,24 +1,31 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.server.manager.MetricYamlManager;
import com.tencent.supersonic.headless.server.manager.ModelYamlManager;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.ViewService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Component;
@@ -26,7 +33,8 @@ import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@Slf4j
@Component
@@ -35,28 +43,21 @@ public class CatalogImpl implements Catalog {
private final DatabaseService databaseService;
private final ModelService modelService;
private final DimensionService dimensionService;
private final ModelService datasourceService;
private final MetricService metricService;
private final ModelRelaService modelRelaService;
private final ViewService viewService;
private final SchemaService schemaService;
public CatalogImpl(DatabaseService databaseService,
ModelService modelService, DimensionService dimensionService,
ModelService datasourceService,
public CatalogImpl(DatabaseService databaseService, SchemaService schemaService,
ModelService modelService, DimensionService dimensionService, ViewService viewService,
MetricService metricService, ModelRelaService modelRelaService) {
this.databaseService = databaseService;
this.modelService = modelService;
this.dimensionService = dimensionService;
this.datasourceService = datasourceService;
this.viewService = viewService;
this.metricService = metricService;
this.modelRelaService = modelRelaService;
}
public DatabaseResp getDatabase(Long id) {
return databaseService.getDatabase(id);
}
public DatabaseResp getDatabaseByModelId(Long modelId) {
return modelService.getDatabaseByModelId(modelId);
this.schemaService = schemaService;
}
@Override
@@ -69,11 +70,6 @@ public class CatalogImpl implements Catalog {
return dimensionService.getDimension(id);
}
@Override
public List<ModelRela> getModelRela(List<Long> modelIds) {
return modelRelaService.getModelRela(modelIds);
}
@Override
public List<DimensionResp> getDimensions(MetaFilter metaFilter) {
return dimensionService.getDimensions(metaFilter);
@@ -91,7 +87,7 @@ public class CatalogImpl implements Catalog {
@Override
public ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric) {
return datasourceService.getItemDate(dimension, metric);
return modelService.getItemDate(dimension, metric);
}
@Override
@@ -106,18 +102,43 @@ public class CatalogImpl implements Catalog {
}
@Override
public List<ModelSchemaResp> getModelSchema(List<Long> modelIds) {
ModelSchemaFilterReq modelSchemaFilterReq = new ModelSchemaFilterReq();
modelSchemaFilterReq.setModelIds(modelIds);
return modelService.fetchModelSchema(modelSchemaFilterReq);
public void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList,
List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName) {
List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
if (CollectionUtils.isEmpty(modelResps)) {
return;
}
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
Long databaseId = modelResps.get(0).getDatabaseId();
DatabaseResp databaseResp = databaseService.getDatabase(databaseId);
for (ModelResp modelResp : modelResps) {
modelIdName.put(modelResp.getId(), modelResp.getBizName());
dataModelYamlTplList.add(ModelYamlManager.convert2YamlObj(modelResp, databaseResp));
if (!dimensionYamlMap.containsKey(modelResp.getBizName())) {
dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>());
}
List<DimensionResp> dimensionRespList = dimensionResps.stream()
.filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName()))
.collect(Collectors.toList());
dimensionYamlMap.get(modelResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml(
dimensionRespList));
}
List<MetricResp> metricResps = new ArrayList<>(semanticSchemaResp.getMetrics());
metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(metricResps));
}
@Override
public void getModelYamlTplByModelIds(Set<Long> modelIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList, List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName) {
datasourceService.getModelYamlTplByModelIds(modelIds, dimensionYamlMap, dataModelYamlTplList,
metricYamlTplList, modelIdName);
public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) {
return schemaService.fetchSemanticSchema(schemaFilterReq);
}
@Override
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) throws ExecutionException {
return schemaService.getStatInfo(itemUseReq);
}
}

24
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View File

@@ -24,6 +24,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
@@ -32,6 +33,7 @@ import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
@@ -63,6 +65,8 @@ public class DimensionServiceImpl implements DimensionService {
private ModelRelaService modelRelaService;
private ViewService viewService;
@Autowired
private ApplicationEventPublisher eventPublisher;
@@ -71,12 +75,14 @@ public class DimensionServiceImpl implements DimensionService {
ModelService modelService,
ChatGptHelper chatGptHelper,
DatabaseService databaseService,
ModelRelaService modelRelaService) {
ModelRelaService modelRelaService,
ViewService viewService) {
this.modelService = modelService;
this.dimensionRepository = dimensionRepository;
this.chatGptHelper = chatGptHelper;
this.databaseService = databaseService;
this.modelRelaService = modelRelaService;
this.viewService = viewService;
}
@Override
@@ -94,7 +100,7 @@ public class DimensionServiceImpl implements DimensionService {
return;
}
Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensionInSameDomain(modelId);
List<DimensionResp> dimensionResps = getDimensions(modelId);
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> nameMap = dimensionResps.stream()
@@ -219,6 +225,10 @@ public class DimensionServiceImpl implements DimensionService {
if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
return filterByField(dimensionResps, metaFilter.getFieldsDepend());
}
if (metaFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(metaFilter.getViewId());
return DimensionConverter.filterByView(dimensionResps, viewResp);
}
return dimensionResps;
}
@@ -317,17 +327,9 @@ public class DimensionServiceImpl implements DimensionService {
return dimValueMapsResp;
}
private List<DimensionResp> getDimensionInSameDomain(Long modelId) {
ModelResp modelResp = modelService.getModel(modelId);
Long domainId = modelResp.getDomainId();
List<ModelResp> modelResps = modelService.getModelByDomainIds(Lists.newArrayList(domainId));
List<Long> modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList());
return getDimensions(new MetaFilter(modelIds));
}
private void checkExist(List<DimensionReq> dimensionReqs) {
Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensionInSameDomain(modelId);
List<DimensionResp> dimensionResps = getDimensions(modelId);
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> nameMap = dimensionResps.stream()

88
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -12,21 +12,21 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.pojo.request.DownloadStructReq;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.utils.DataTransformUtils;
import com.tencent.supersonic.headless.server.pojo.DataDownload;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DownloadService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.QueryService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -42,6 +42,7 @@ import java.io.OutputStream;
import java.net.URLEncoder;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -58,12 +59,16 @@ public class DownloadServiceImpl implements DownloadService {
private static final long downloadSize = 10000;
private ModelService modelService;
private MetricService metricService;
private DimensionService dimensionService;
private QueryService queryService;
public DownloadServiceImpl(ModelService modelService, QueryService queryService) {
this.modelService = modelService;
public DownloadServiceImpl(MetricService metricService,
DimensionService dimensionService, QueryService queryService) {
this.metricService = metricService;
this.dimensionService = dimensionService;
this.queryService = queryService;
}
@@ -74,7 +79,7 @@ public class DownloadServiceImpl implements DownloadService {
File file = FileUtils.createTmpFile(fileName);
try {
QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq, true);
SemanticQueryResp queryResult = (SemanticQueryResp) queryService.queryByReq(querySqlReq, user);
SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
} catch (RuntimeException e) {
@@ -100,21 +105,30 @@ public class DownloadServiceImpl implements DownloadService {
public void batchDownload(BatchDownloadReq batchDownloadReq, User user, File file) throws Exception {
List<Long> metricIds = batchDownloadReq.getMetricIds();
List<ModelSchemaResp> modelSchemaRespList = modelService.fetchModelSchema(new ModelSchemaFilterReq());
Map<String, List<MetricSchemaResp>> metricSchemaMap = getMetricSchemaMap(modelSchemaRespList, metricIds);
Map<Long, DimSchemaResp> dimensionRespMap = getDimensionMap(modelSchemaRespList);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(metricIds);
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
Map<String, List<MetricResp>> metricMap = getMetricMap(metricResps);
List<Long> dimensionIds = metricResps.stream().map(MetricResp::getRelateDimension)
.map(RelateDimension::getDrillDownDimensions).flatMap(Collection::stream)
.map(DrillDownDimension::getDimensionId).collect(Collectors.toList());
metaFilter.setIds(dimensionIds);
Map<Long, DimensionResp> dimensionRespMap = dimensionService.getDimensions(metaFilter)
.stream().collect(Collectors.toMap(DimensionResp::getId, d -> d));
ExcelWriter excelWriter = EasyExcel.write(file).build();
int sheetCount = 1;
for (List<MetricSchemaResp> metrics : metricSchemaMap.values()) {
for (List<MetricResp> metrics : metricMap.values()) {
if (CollectionUtils.isEmpty(metrics)) {
continue;
}
MetricSchemaResp metricSchemaResp = metrics.get(0);
List<DimSchemaResp> dimensions = getMetricRelaDimensions(metricSchemaResp, dimensionRespMap);
for (MetricSchemaResp metric : metrics) {
MetricResp metricResp = metrics.get(0);
List<DimensionResp> dimensions = getMetricRelaDimensions(metricResp, dimensionRespMap);
for (MetricResp metric : metrics) {
try {
DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq);
SemanticQueryResp queryResult = queryService.queryByReq(downloadStructReq, user);
DownloadStructReq downloadStructReq = buildDownloadReq(dimensions, metric, batchDownloadReq);
QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq);
querySqlReq.setNeedAuth(true);
SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
.head(dataDownload.getHeaders()).build();
@@ -219,14 +233,14 @@ public class DownloadServiceImpl implements DownloadService {
}
}
private DownloadStructReq buildDownloadStructReq(List<DimSchemaResp> dimensionResps, MetricResp metricResp,
private DownloadStructReq buildDownloadReq(List<DimensionResp> dimensionResps, MetricResp metricResp,
BatchDownloadReq batchDownloadReq) {
DateConf dateConf = batchDownloadReq.getDateInfo();
Set<Long> modelIds = dimensionResps.stream().map(DimSchemaResp::getModelId).collect(Collectors.toSet());
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
modelIds.add(metricResp.getModelId());
DownloadStructReq downloadStructReq = new DownloadStructReq();
downloadStructReq.setGroups(dimensionResps.stream()
.map(DimSchemaResp::getBizName).collect(Collectors.toList()));
.map(DimensionResp::getBizName).collect(Collectors.toList()));
downloadStructReq.getGroups().add(0, getTimeDimension(dateConf));
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricResp.getBizName());
@@ -248,31 +262,27 @@ public class DownloadServiceImpl implements DownloadService {
}
}
private Map<String, List<MetricSchemaResp>> getMetricSchemaMap(List<ModelSchemaResp> modelSchemaRespList,
List<Long> metricIds) {
return modelSchemaRespList.stream().flatMap(modelSchemaResp
-> modelSchemaResp.getMetrics().stream())
.filter(metricSchemaResp -> metricIds.contains(metricSchemaResp.getId()))
.collect(Collectors.groupingBy(MetricSchemaResp::getRelaDimensionIdKey));
}
private Map<Long, DimSchemaResp> getDimensionMap(List<ModelSchemaResp> modelSchemaRespList) {
return modelSchemaRespList.stream().flatMap(modelSchemaResp
-> modelSchemaResp.getDimensions().stream())
.collect(Collectors.toMap(DimensionResp::getId, dimensionResp -> dimensionResp));
private Map<String, List<MetricResp>> getMetricMap(List<MetricResp> metricResps) {
for (MetricResp metricResp : metricResps) {
List<DrillDownDimension> drillDownDimensions = metricService.getDrillDownDimension(metricResp.getId());
RelateDimension relateDimension = RelateDimension.builder()
.drillDownDimensions(drillDownDimensions).build();
metricResp.setRelateDimension(relateDimension);
}
return metricResps.stream().collect(Collectors.groupingBy(MetricResp::getRelaDimensionIdKey));
}
private Map<String, String> getDimensionNameMap(List<QueryColumn> queryColumns) {
return queryColumns.stream().collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn));
}
private List<DimSchemaResp> getMetricRelaDimensions(MetricSchemaResp metricSchemaResp,
Map<Long, DimSchemaResp> dimensionRespMap) {
if (metricSchemaResp.getRelateDimension() == null
|| CollectionUtils.isEmpty(metricSchemaResp.getRelateDimension().getDrillDownDimensions())) {
private List<DimensionResp> getMetricRelaDimensions(MetricResp metricResp,
Map<Long, DimensionResp> dimensionRespMap) {
if (metricResp.getRelateDimension() == null
|| CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) {
return Lists.newArrayList();
}
return metricSchemaResp.getRelateDimension().getDrillDownDimensions()
return metricResp.getRelateDimension().getDrillDownDimensions()
.stream().map(drillDownDimension -> dimensionRespMap.get(drillDownDimension.getDimensionId()))
.filter(Objects::nonNull)
.collect(Collectors.toList());

47
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -25,6 +25,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
@@ -35,6 +36,7 @@ import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import lombok.extern.slf4j.Slf4j;
@@ -65,6 +67,8 @@ public class MetricServiceImpl implements MetricService {
private CollectService collectService;
private ViewService viewService;
private ApplicationEventPublisher eventPublisher;
public MetricServiceImpl(MetricRepository metricRepository,
@@ -72,6 +76,7 @@ public class MetricServiceImpl implements MetricService {
DomainService domainService,
ChatGptHelper chatGptHelper,
CollectService collectService,
ViewService viewService,
ApplicationEventPublisher eventPublisher) {
this.domainService = domainService;
this.metricRepository = metricRepository;
@@ -79,6 +84,7 @@ public class MetricServiceImpl implements MetricService {
this.chatGptHelper = chatGptHelper;
this.eventPublisher = eventPublisher;
this.collectService = collectService;
this.viewService = viewService;
}
@Override
@@ -98,7 +104,7 @@ public class MetricServiceImpl implements MetricService {
return;
}
Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricResps = getMetricInSameDomain(modelId);
List<MetricResp> metricResps = getMetrics(new MetaFilter(Lists.newArrayList(modelId)));
Map<String, MetricResp> bizNameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap = metricResps.stream()
@@ -186,7 +192,11 @@ public class MetricServiceImpl implements MetricService {
List<CollectDO> collectList = collectService.getCollectList(user.getName());
List<Long> collectIds = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList());
if (pageMetricReq.isHasCollect()) {
metricFilter.setIds(collectIds);
if (CollectionUtils.isEmpty(collectIds)) {
metricFilter.setIds(Lists.newArrayList(-1L));
} else {
metricFilter.setIds(collectIds);
}
}
PageInfo<MetricDO> metricDOPageInfo = PageHelper.startPage(pageMetricReq.getCurrent(),
pageMetricReq.getPageSize())
@@ -211,6 +221,10 @@ public class MetricServiceImpl implements MetricService {
if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
return filterByField(metricResps, metaFilter.getFieldsDepend());
}
if (metaFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(metaFilter.getViewId());
return MetricConverter.filterByView(metricResps, viewResp);
}
return metricResps;
}
@@ -321,16 +335,27 @@ public class MetricServiceImpl implements MetricService {
@Override
public List<DrillDownDimension> getDrillDownDimension(Long metricId) {
List<DrillDownDimension> drillDownDimensions = Lists.newArrayList();
MetricResp metricResp = getMetric(metricId);
if (metricResp == null) {
return Lists.newArrayList();
return drillDownDimensions;
}
if (metricResp.getRelateDimension() != null
&& !CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) {
return metricResp.getRelateDimension().getDrillDownDimensions();
drillDownDimensions.addAll(metricResp.getRelateDimension().getDrillDownDimensions());
}
ModelResp modelResp = modelService.getModel(metricResp.getModelId());
return modelResp.getDrillDownDimensions();
if (modelResp.getDrillDownDimensions() == null) {
return drillDownDimensions;
}
for (DrillDownDimension drillDownDimension : modelResp.getDrillDownDimensions()) {
if (!drillDownDimensions.stream().map(DrillDownDimension::getDimensionId)
.collect(Collectors.toList()).contains(drillDownDimension.getDimensionId())) {
drillDownDimension.setInheritedFromModel(true);
drillDownDimensions.add(drillDownDimension);
}
}
return drillDownDimensions;
}
@Override
@@ -361,7 +386,9 @@ public class MetricServiceImpl implements MetricService {
private void checkExist(List<MetricBaseReq> metricReqs) {
Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricResps = getMetricInSameDomain(modelId);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metaFilter);
Map<String, MetricResp> bizNameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap = metricResps.stream()
@@ -384,14 +411,6 @@ public class MetricServiceImpl implements MetricService {
}
}
private List<MetricResp> getMetricInSameDomain(Long modelId) {
ModelResp modelResp = modelService.getModel(modelId);
Long domainId = modelResp.getDomainId();
List<ModelResp> modelResps = modelService.getModelByDomainIds(Lists.newArrayList(domainId));
List<Long> modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList());
return getMetrics(new MetaFilter(modelIds));
}
private List<MetricResp> convertList(List<MetricDO> metricDOS) {
return convertList(metricDOS, Lists.newArrayList());
}

141
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -4,7 +4,6 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
@@ -14,29 +13,19 @@ import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.request.DateInfoReq;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.server.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.server.manager.MetricYamlManager;
import com.tencent.supersonic.headless.server.manager.ModelYamlManager;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;
@@ -49,6 +38,7 @@ import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.ModelConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
@@ -67,7 +57,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -90,6 +79,8 @@ public class ModelServiceImpl implements ModelService {
private UserService userService;
private ViewService viewService;
private DateInfoRepository dateInfoRepository;
public ModelServiceImpl(ModelRepository modelRepository,
@@ -99,6 +90,7 @@ public class ModelServiceImpl implements ModelService {
ModelRelaService modelRelaService,
DomainService domainService,
UserService userService,
ViewService viewService,
DateInfoRepository dateInfoRepository) {
this.modelRepository = modelRepository;
this.databaseService = databaseService;
@@ -107,6 +99,7 @@ public class ModelServiceImpl implements ModelService {
this.domainService = domainService;
this.modelRelaService = modelRelaService;
this.userService = userService;
this.viewService = viewService;
this.dateInfoRepository = dateInfoRepository;
}
@@ -153,8 +146,16 @@ public class ModelServiceImpl implements ModelService {
}
@Override
public List<ModelResp> getModelList(ModelFilter modelFilter) {
return ModelConverter.convertList(modelRepository.getModelList(modelFilter));
public List<ModelResp> getModelList(MetaFilter metaFilter) {
ModelFilter modelFilter = new ModelFilter();
BeanUtils.copyProperties(metaFilter, modelFilter);
List<ModelResp> modelResps = ModelConverter.convertList(modelRepository.getModelList(modelFilter));
if (modelFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(modelFilter.getViewId());
return modelResps.stream().filter(modelResp -> viewResp.getAllModels().contains(modelResp.getId()))
.collect(Collectors.toList());
}
return modelResps;
}
@Override
@@ -383,52 +384,6 @@ public class ModelServiceImpl implements ModelService {
return Lists.newArrayList();
}
@Override
public ModelSchemaResp fetchSingleModelSchema(Long modelId) {
ModelResp modelResp = getModel(modelId);
ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
BeanUtils.copyProperties(modelResp, modelSchemaResp);
modelSchemaResp.setDimensions(generateDimSchema(modelId));
modelSchemaResp.setMetrics(generateMetricSchema(modelId, modelResp));
return modelSchemaResp;
}
@Override
public List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq modelSchemaFilterReq) {
List<ModelSchemaResp> modelSchemaRespList = new ArrayList<>();
List<Long> modelIds = modelSchemaFilterReq.getModelIds();
if (CollectionUtils.isEmpty(modelIds)) {
modelIds = generateModelIdsReq(modelSchemaFilterReq);
}
MetaFilter metaFilter = new MetaFilter(modelIds);
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetrics(metaFilter)
.stream().collect(Collectors.groupingBy(MetricResp::getModelId));
Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensions(metaFilter)
.stream().collect(Collectors.groupingBy(DimensionResp::getModelId));
List<ModelRela> modelRelas = modelRelaService.getModelRela(modelIds);
for (Long modelId : modelIds) {
ModelResp modelResp = getModelMap().get(modelId);
if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) {
continue;
}
List<MetricResp> metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList());
List<MetricSchemaResp> metricSchemaResps = metricResps.stream().map(metricResp ->
convert(metricResp, modelResp)).collect(Collectors.toList());
List<DimSchemaResp> dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList())
.stream().map(this::convert).collect(Collectors.toList());
ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
BeanUtils.copyProperties(modelResp, modelSchemaResp);
modelSchemaResp.setDimensions(dimensionResps);
modelSchemaResp.setMetrics(metricSchemaResps);
modelSchemaResp.setModelRelas(modelRelas.stream().filter(modelRela
-> modelRela.getFromModelId().equals(modelId) || modelRela.getToModelId().equals(modelId))
.collect(Collectors.toList()));
modelSchemaRespList.add(modelSchemaResp);
}
return modelSchemaRespList;
}
@Override
public DatabaseResp getDatabaseByModelId(Long modelId) {
ModelResp modelResp = getModel(modelId);
@@ -464,37 +419,6 @@ public class ModelServiceImpl implements ModelService {
return modelRepository.getModelById(id);
}
private List<MetricSchemaResp> generateMetricSchema(Long modelId, ModelResp modelResp) {
List<MetricSchemaResp> metricSchemaDescList = new ArrayList<>();
List<MetricResp> metricResps = metricService.getMetrics(new MetaFilter(Lists.newArrayList(modelId)));
metricResps.forEach(metricResp -> metricSchemaDescList.add(convert(metricResp, modelResp)));
return metricSchemaDescList;
}
private List<DimSchemaResp> generateDimSchema(Long modelId) {
List<DimensionResp> dimDescList = dimensionService.getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
return dimDescList.stream().map(this::convert).collect(Collectors.toList());
}
private DimSchemaResp convert(DimensionResp dimensionResp) {
DimSchemaResp dimSchemaResp = new DimSchemaResp();
BeanUtils.copyProperties(dimensionResp, dimSchemaResp);
dimSchemaResp.setUseCnt(0L);
return dimSchemaResp;
}
private MetricSchemaResp convert(MetricResp metricResp, ModelResp modelResp) {
MetricSchemaResp metricSchemaResp = new MetricSchemaResp();
BeanUtils.copyProperties(metricResp, metricSchemaResp);
RelateDimension relateDimension = metricResp.getRelateDimension();
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
metricSchemaResp.setRelateDimension(RelateDimension.builder()
.drillDownDimensions(modelResp.getDrillDownDimensions()).build());
}
metricSchemaResp.setUseCnt(0L);
return metricSchemaResp;
}
private List<DateInfoReq> convert(List<DateInfoDO> dateInfoDOList) {
List<DateInfoReq> dateInfoCommendList = new ArrayList<>();
dateInfoDOList.forEach(dateInfoDO -> {
@@ -506,13 +430,6 @@ public class ModelServiceImpl implements ModelService {
return dateInfoCommendList;
}
private List<Long> generateModelIdsReq(ModelSchemaFilterReq filter) {
if (Objects.nonNull(filter) && !CollectionUtils.isEmpty(filter.getModelIds())) {
return filter.getModelIds();
}
return new ArrayList<>(getModelMap().keySet());
}
public static boolean checkAdminPermission(Set<String> orgIds, User user, ModelResp modelResp) {
List<String> admins = modelResp.getAdmins();
List<String> adminOrgs = modelResp.getAdminOrgs();
@@ -561,30 +478,4 @@ public class ModelServiceImpl implements ModelService {
return false;
}
@Override
public void getModelYamlTplByModelIds(Set<Long> modelIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList, List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName) {
for (Long modelId : modelIds) {
ModelResp modelResp = getModel(modelId);
modelIdName.put(modelId, modelResp.getBizName());
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId));
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(metricResps));
Long databaseId = modelResp.getDatabaseId();
DatabaseResp databaseResp = databaseService.getDatabase(databaseId);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
dataModelYamlTplList.add(ModelYamlManager.convert2YamlObj(modelResp, databaseResp));
if (!dimensionYamlMap.containsKey(modelResp.getBizName())) {
dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>());
}
List<DimensionResp> dimensionRespList = dimensionResps.stream()
.filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName()))
.collect(Collectors.toList());
dimensionYamlMap.get(modelResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml(
dimensionRespList));
}
}
}

93
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -12,19 +10,17 @@ import com.tencent.supersonic.common.pojo.enums.ApiItemType;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Item;
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
@@ -33,8 +29,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.cache.QueryCache;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.parser.DefaultQueryParser;
@@ -49,32 +45,27 @@ import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.service.AppService;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Slf4j
public class QueryServiceImpl implements QueryService {
protected final Cache<String, List<ItemUseResp>> itemUseCache =
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
private final StatUtils statUtils;
private StatUtils statUtils;
private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter;
private final Catalog catalog;
@@ -124,7 +115,7 @@ public class QueryServiceImpl implements QueryService {
}
StatUtils.get().setUseResultCache(false);
//3 query
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
QueryStatement queryStatement = buildQueryStatement(queryReq);
SemanticQueryResp result = query(queryStatement);
//4 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(cacheKey, result);
@@ -145,49 +136,51 @@ public class QueryServiceImpl implements QueryService {
}
}
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
filter.setModelIds(querySqlReq.getModelIds());
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, modelSchemaResps);
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq) throws Exception {
SchemaFilterReq filter = buildSchemaFilterReq(querySqlReq);
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, semanticSchemaResp);
queryStatement.setModelIds(querySqlReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
SemanticModel semanticModel = semanticSchemaManager.get(querySqlReq.getModelIdStr());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp);
queryStatement.setSemanticModel(semanticModel);
return queryStatement;
}
private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) throws Exception {
private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq) throws Exception {
if (semanticQueryReq instanceof QuerySqlReq) {
return buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user);
return buildSqlQueryStatement((QuerySqlReq) semanticQueryReq);
}
if (semanticQueryReq instanceof QueryStructReq) {
return buildStructQueryStatement((QueryStructReq) semanticQueryReq, user);
return buildStructQueryStatement((QueryStructReq) semanticQueryReq);
}
if (semanticQueryReq instanceof QueryMultiStructReq) {
return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user);
return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq);
}
return null;
}
private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq, User user) throws Exception {
private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq) throws Exception {
SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq);
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setIsS2SQL(false);
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
SemanticModel semanticModel = semanticSchemaManager.get(queryStructReq.getModelIdStr());
queryStatement.setViewId(queryStatement.getQueryStructReq().getViewId());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp);
queryStatement.setSemanticModel(semanticModel);
return queryStatement;
}
private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq, User user)
private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq)
throws Exception {
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq, user);
SemanticModel semanticModel = semanticSchemaManager.get(queryStructReq.getModelIdStr());
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
SemanticModel semanticModel = queryStatement.getSemanticModel();
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
queryStatement.setSemanticModel(semanticModel);
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
@@ -198,6 +191,13 @@ public class QueryServiceImpl implements QueryService {
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
}
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
schemaFilterReq.setViewId(semanticQueryReq.getViewId());
schemaFilterReq.setModelIds(semanticQueryReq.getModelIds());
return schemaFilterReq;
}
@Override
@SneakyThrows
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
@@ -208,20 +208,13 @@ public class QueryServiceImpl implements QueryService {
@Override
@SneakyThrows
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
if (itemUseReq.getCacheEnable()) {
return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> {
List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
itemUseCache.put(JsonUtil.toString(itemUseReq), data);
return data;
});
}
return statUtils.getStatInfo(itemUseReq);
return catalog.getStatInfo(itemUseReq);
}
@Override
public <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception {
T queryReq = explainSqlReq.getQueryReq();
QueryStatement queryStatement = buildQueryStatement((SemanticQueryReq) queryReq, user);
QueryStatement queryStatement = buildQueryStatement((SemanticQueryReq) queryReq);
queryStatement = plan(queryStatement);
return getExplainResp(queryStatement);
}
@@ -297,7 +290,7 @@ public class QueryServiceImpl implements QueryService {
}
private void authCheck(AppDetailResp appDetailResp, List<Long> ids, ApiItemType type) {
Set<Long> idsInApp = appDetailResp.getConfig().getAllItems().stream()
Set<Long> idsInApp = appDetailResp.allItems().stream()
.filter(item -> type.equals(item.getType())).map(Item::getId).collect(Collectors.toSet());
if (!idsInApp.containsAll(ids)) {
throw new InvalidArgumentException("查询范围超过应用申请范围, 请检查");
@@ -346,9 +339,7 @@ public class QueryServiceImpl implements QueryService {
//3 execute
if (queryExecutor != null) {
semanticQueryResp = queryExecutor.execute(queryStatement);
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
}
queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getSemanticSchemaResp());
}
return semanticQueryResp;
} catch (Exception e) {

258
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -1,15 +1,24 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.github.pagehelper.PageInfo;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.enums.SchemaType;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
@@ -18,19 +27,33 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
@@ -39,37 +62,115 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
@Service
public class SchemaServiceImpl implements SchemaService {
private final QueryService queryService;
protected final Cache<String, List<ItemUseResp>> itemUseCache =
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
private final StatUtils statUtils;
private final ModelService modelService;
private final DimensionService dimensionService;
private final MetricService metricService;
private final DomainService domainService;
private final ViewService viewService;
private final ModelRelaService modelRelaService;
public SchemaServiceImpl(QueryService queryService,
ModelService modelService,
public SchemaServiceImpl(ModelService modelService,
DimensionService dimensionService,
MetricService metricService,
DomainService domainService) {
this.queryService = queryService;
DomainService domainService,
ViewService viewService,
ModelRelaService modelRelaService,
StatUtils statUtils) {
this.modelService = modelService;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.domainService = domainService;
this.viewService = viewService;
this.modelRelaService = modelRelaService;
this.statUtils = statUtils;
}
@SneakyThrows
@Override
public List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq filter, User user) {
List<ModelSchemaResp> domainSchemaDescList = modelService.fetchModelSchema(filter);
ItemUseReq itemUseCommend = new ItemUseReq();
itemUseCommend.setModelIds(filter.getModelIds());
List<ItemUseResp> statInfos = queryService.getStatInfo(itemUseCommend);
log.debug("statInfos:{}", statInfos);
fillCnt(domainSchemaDescList, statInfos);
return domainSchemaDescList;
public List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter) {
List<ViewSchemaResp> viewSchemaResps = new ArrayList<>();
List<Long> viewIds = filter.getViewIds();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
metaFilter.setIds(viewIds);
List<ViewResp> viewResps = viewService.getViewList(metaFilter);
List<Long> modelIds = viewResps.stream().map(ViewResp::getAllModels)
.flatMap(Collection::stream).collect(Collectors.toList());
metaFilter.setModelIds(modelIds);
metaFilter.setIds(Lists.newArrayList());
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
metaFilter.setIds(modelIds);
List<ModelResp> modelResps = modelService.getModelList(metaFilter);
Map<Long, ViewResp> viewRespMap = getViewMap(viewResps);
for (Long viewId : viewRespMap.keySet()) {
ViewResp viewResp = viewRespMap.get(viewId);
if (viewResp == null || !StatusEnum.ONLINE.getCode().equals(viewResp.getStatus())) {
continue;
}
List<MetricSchemaResp> metricSchemaResps = MetricConverter.filterByView(metricResps, viewResp)
.stream().map(this::convert).collect(Collectors.toList());
List<DimSchemaResp> dimSchemaResps = DimensionConverter.filterByView(dimensionResps, viewResp)
.stream().map(this::convert).collect(Collectors.toList());
ViewSchemaResp viewSchemaResp = new ViewSchemaResp();
BeanUtils.copyProperties(viewResp, viewSchemaResp);
viewSchemaResp.setDimensions(dimSchemaResps);
viewSchemaResp.setMetrics(metricSchemaResps);
viewSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
viewResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));
viewSchemaResps.add(viewSchemaResp);
}
fillStaticInfo(viewSchemaResps);
return viewSchemaResps;
}
private void fillCnt(List<ModelSchemaResp> domainSchemaDescList, List<ItemUseResp> statInfos) {
public ViewSchemaResp fetchViewSchema(Long viewId) {
if (viewId == null) {
return null;
}
return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null);
}
public List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds) {
List<ModelSchemaResp> modelSchemaResps = Lists.newArrayList();
if (CollectionUtils.isEmpty(modelIds)) {
return modelSchemaResps;
}
MetaFilter metaFilter = new MetaFilter(modelIds);
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetrics(metaFilter)
.stream().collect(Collectors.groupingBy(MetricResp::getModelId));
Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensions(metaFilter)
.stream().collect(Collectors.groupingBy(DimensionResp::getModelId));
List<ModelRela> modelRelas = modelRelaService.getModelRela(modelIds);
Map<Long, ModelResp> modelMap = modelService.getModelMap();
for (Long modelId : modelIds) {
ModelResp modelResp = modelMap.get(modelId);
if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) {
continue;
}
List<MetricResp> metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList());
List<MetricSchemaResp> metricSchemaResps = metricResps.stream()
.map(this::convert).collect(Collectors.toList());
List<DimSchemaResp> dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList())
.stream().map(this::convert).collect(Collectors.toList());
ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
BeanUtils.copyProperties(modelResp, modelSchemaResp);
modelSchemaResp.setDimensions(dimensionResps);
modelSchemaResp.setMetrics(metricSchemaResps);
modelSchemaResp.setModelRelas(modelRelas.stream().filter(modelRela
-> modelRela.getFromModelId().equals(modelId) || modelRela.getToModelId().equals(modelId))
.collect(Collectors.toList()));
modelSchemaResps.add(modelSchemaResp);
}
return modelSchemaResps;
}
private void fillCnt(List<ViewSchemaResp> viewSchemaResps, List<ItemUseResp> statInfos) {
Map<String, ItemUseResp> typeIdAndStatPair = statInfos.stream()
.collect(Collectors.toMap(
@@ -77,43 +178,45 @@ public class SchemaServiceImpl implements SchemaService {
itemUseInfo -> itemUseInfo,
(item1, item2) -> item1));
log.debug("typeIdAndStatPair:{}", typeIdAndStatPair);
for (ModelSchemaResp domainSchemaDesc : domainSchemaDescList) {
fillDimCnt(domainSchemaDesc, typeIdAndStatPair);
fillMetricCnt(domainSchemaDesc, typeIdAndStatPair);
for (ViewSchemaResp viewSchemaResp : viewSchemaResps) {
fillDimCnt(viewSchemaResp, typeIdAndStatPair);
fillMetricCnt(viewSchemaResp, typeIdAndStatPair);
}
}
private void fillMetricCnt(ModelSchemaResp domainSchemaDesc, Map<String, ItemUseResp> typeIdAndStatPair) {
List<MetricSchemaResp> metrics = domainSchemaDesc.getMetrics();
if (CollectionUtils.isEmpty(domainSchemaDesc.getMetrics())) {
private void fillMetricCnt(ViewSchemaResp viewSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<MetricSchemaResp> metrics = viewSchemaResp.getMetrics();
if (CollectionUtils.isEmpty(viewSchemaResp.getMetrics())) {
return;
}
if (!CollectionUtils.isEmpty(metrics)) {
metrics.stream().forEach(metric -> {
String key = TypeEnums.METRIC.getName() + AT_SYMBOL + AT_SYMBOL + metric.getBizName();
String key = TypeEnums.METRIC.name().toLowerCase()
+ AT_SYMBOL + AT_SYMBOL + metric.getBizName();
if (typeIdAndStatPair.containsKey(key)) {
metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
}
});
}
domainSchemaDesc.setMetrics(metrics);
viewSchemaResp.setMetrics(metrics);
}
private void fillDimCnt(ModelSchemaResp domainSchemaDesc, Map<String, ItemUseResp> typeIdAndStatPair) {
List<DimSchemaResp> dimensions = domainSchemaDesc.getDimensions();
if (CollectionUtils.isEmpty(domainSchemaDesc.getDimensions())) {
private void fillDimCnt(ViewSchemaResp viewSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<DimSchemaResp> dimensions = viewSchemaResp.getDimensions();
if (CollectionUtils.isEmpty(viewSchemaResp.getDimensions())) {
return;
}
if (!CollectionUtils.isEmpty(dimensions)) {
dimensions.stream().forEach(dim -> {
String key = TypeEnums.DIMENSION.getName() + AT_SYMBOL + AT_SYMBOL + dim.getBizName();
String key = TypeEnums.DIMENSION.name().toLowerCase()
+ AT_SYMBOL + AT_SYMBOL + dim.getBizName();
if (typeIdAndStatPair.containsKey(key)) {
dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
}
});
}
domainSchemaDesc.setDimensions(dimensions);
viewSchemaResp.setDimensions(dimensions);
}
@Override
@@ -135,7 +238,7 @@ public class SchemaServiceImpl implements SchemaService {
} else if (TypeEnums.DIMENSION.equals(schemaItemQueryReq.getType())) {
return dimensionService.getDimensions(metaFilter);
}
throw new InvalidArgumentException("暂不支持的类型" + schemaItemQueryReq.getType().getName());
throw new InvalidArgumentException("暂不支持的类型" + schemaItemQueryReq.getType().name());
}
@Override
@@ -148,4 +251,97 @@ public class SchemaServiceImpl implements SchemaService {
return modelService.getModelListWithAuth(user, domainId, authTypeEnum);
}
@Override
public List<ViewResp> getViewList(Long domainId) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDomainId(domainId);
return viewService.getViewList(metaFilter);
}
@Override
public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) {
SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp();
semanticSchemaResp.setViewId(schemaFilterReq.getViewId());
semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds());
if (schemaFilterReq.getViewId() != null) {
ViewSchemaResp viewSchemaResp = fetchViewSchema(schemaFilterReq.getViewId());
BeanUtils.copyProperties(viewSchemaResp, semanticSchemaResp);
List<Long> modelIds = viewSchemaResp.getAllModels();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(modelIds);
List<ModelResp> modelList = modelService.getModelList(metaFilter);
metaFilter.setModelIds(modelIds);
List<ModelRela> modelRelas = modelRelaService.getModelRela(modelIds);
semanticSchemaResp.setModelResps(modelList);
semanticSchemaResp.setModelRelas(modelRelas);
semanticSchemaResp.setModelIds(modelIds);
semanticSchemaResp.setSchemaType(SchemaType.VIEW);
} else if (!CollectionUtils.isEmpty(schemaFilterReq.getModelIds())) {
List<ModelSchemaResp> modelSchemaResps = fetchModelSchemaResps(schemaFilterReq.getModelIds());
semanticSchemaResp.setMetrics(modelSchemaResps.stream().map(ModelSchemaResp::getMetrics)
.flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp.setDimensions(modelSchemaResps.stream().map(ModelSchemaResp::getDimensions)
.flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp.setModelRelas(modelSchemaResps.stream().map(ModelSchemaResp::getModelRelas)
.flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
semanticSchemaResp.setSchemaType(SchemaType.MODEL);
}
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0));
semanticSchemaResp.setDatabaseResp(databaseResp);
}
return semanticSchemaResp;
}
@SneakyThrows
@Override
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
if (itemUseReq.getCacheEnable()) {
return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> {
List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
itemUseCache.put(JsonUtil.toString(itemUseReq), data);
return data;
});
}
return statUtils.getStatInfo(itemUseReq);
}
private void fillStaticInfo(List<ViewSchemaResp> viewSchemaResps) throws ExecutionException {
List<Long> viewIds = viewSchemaResps.stream()
.map(ViewSchemaResp::getId).collect(Collectors.toList());
ItemUseReq itemUseReq = new ItemUseReq();
itemUseReq.setModelIds(viewIds);
List<ItemUseResp> statInfos = getStatInfo(itemUseReq);
log.debug("statInfos:{}", statInfos);
fillCnt(viewSchemaResps, statInfos);
}
private Map<Long, ViewResp> getViewMap(List<ViewResp> viewResps) {
if (CollectionUtils.isEmpty(viewResps)) {
return new HashMap<>();
}
return viewResps.stream().collect(
Collectors.toMap(ViewResp::getId, viewResp -> viewResp));
}
private DimSchemaResp convert(DimensionResp dimensionResp) {
DimSchemaResp dimSchemaResp = new DimSchemaResp();
BeanUtils.copyProperties(dimensionResp, dimSchemaResp);
return dimSchemaResp;
}
private MetricSchemaResp convert(MetricResp metricResp) {
MetricSchemaResp metricSchemaResp = new MetricSchemaResp();
BeanUtils.copyProperties(metricResp, metricSchemaResp);
return metricSchemaResp;
}
private ModelResp convert(ModelSchemaResp modelSchemaResp) {
ModelResp modelResp = new ModelResp();
BeanUtils.copyProperties(modelSchemaResp, modelResp);
return modelResp;
}
}

100
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java Normal file
View File

@@ -0,0 +1,100 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.ViewDetail;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO;
import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.ViewService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
@Service
public class ViewServiceImpl
extends ServiceImpl<ViewDOMapper, ViewDO> implements ViewService {
@Override
public ViewResp save(ViewReq viewReq, User user) {
viewReq.createdBy(user.getName());
ViewDO viewDO = convert(viewReq);
viewDO.setStatus(StatusEnum.ONLINE.getCode());
save(viewDO);
return convert(viewDO);
}
@Override
public ViewResp update(ViewReq viewReq, User user) {
viewReq.updatedBy(user.getName());
ViewDO viewDO = convert(viewReq);
updateById(viewDO);
return convert(viewDO);
}
@Override
public ViewResp getView(Long id) {
ViewDO viewDO = getById(id);
return convert(viewDO);
}
@Override
public List<ViewResp> getViewList(MetaFilter metaFilter) {
QueryWrapper<ViewDO> wrapper = new QueryWrapper<>();
if (metaFilter.getDomainId() != null) {
wrapper.lambda().eq(ViewDO::getDomainId, metaFilter.getDomainId());
}
if (!CollectionUtils.isEmpty(metaFilter.getIds())) {
wrapper.lambda().in(ViewDO::getId, metaFilter.getIds());
}
wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode());
return list(wrapper).stream().map(this::convert).collect(Collectors.toList());
}
@Override
public void delete(Long id, User user) {
ViewDO viewDO = getById(id);
viewDO.setStatus(StatusEnum.DELETED.getCode());
viewDO.setUpdatedBy(user.getName());
viewDO.setUpdatedAt(new Date());
updateById(viewDO);
}
private ViewResp convert(ViewDO viewDO) {
ViewResp viewResp = new ViewResp();
BeanMapper.mapper(viewDO, viewResp);
viewResp.setViewDetail(JSONObject.parseObject(viewDO.getViewDetail(), ViewDetail.class));
if (viewDO.getQueryConfig() != null) {
viewResp.setQueryConfig(JSONObject.parseObject(viewDO.getQueryConfig(), QueryConfig.class));
}
viewResp.setAdmins(StringUtils.isBlank(viewDO.getAdmin())
? Lists.newArrayList() : Arrays.asList(viewDO.getAdmin().split(",")));
viewResp.setAdminOrgs(StringUtils.isBlank(viewDO.getAdminOrg())
? Lists.newArrayList() : Arrays.asList(viewDO.getAdminOrg().split(",")));
viewResp.setTypeEnum(TypeEnums.VIEW);
return viewResp;
}
private ViewDO convert(ViewReq viewReq) {
ViewDO viewDO = new ViewDO();
BeanMapper.mapper(viewReq, viewDO);
viewDO.setViewDetail(JSONObject.toJSONString(viewReq.getViewDetail()));
viewDO.setQueryConfig(JSONObject.toJSONString(viewReq.getQueryConfig()));
return viewDO;
}
}

9
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java
View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
@@ -19,6 +20,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
public class DimensionConverter {
@@ -77,4 +79,11 @@ public class DimensionConverter {
return dimensionResp;
}
public static List<DimensionResp> filterByView(List<DimensionResp> dimensionResps, ViewResp viewResp) {
return dimensionResps.stream().filter(dimensionResp ->
viewResp.getAllDimensions().contains(dimensionResp.getId())
|| viewResp.getAllIncludeAllModels().contains(dimensionResp.getModelId()))
.collect(Collectors.toList());
}
}

9
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
View File

@@ -14,12 +14,14 @@ import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import org.springframework.beans.BeanUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class MetricConverter {
@@ -96,4 +98,11 @@ public class MetricConverter {
return metricResp;
}
public static List<MetricResp> filterByView(List<MetricResp> metricResps, ViewResp viewResp) {
return metricResps.stream().filter(metricResp ->
viewResp.getAllMetrics().contains(metricResp.getId())
|| viewResp.getAllIncludeAllModels().contains(metricResp.getModelId()))
.collect(Collectors.toList());
}
}

134
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -9,25 +9,33 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -37,14 +45,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
@@ -59,36 +59,31 @@ public class QueryReqConverter {
@Autowired
private SqlGenerateUtils sqlGenerateUtils;
@Autowired
private Catalog catalog;
public QueryStatement convert(QuerySqlReq querySQLReq,
SemanticSchemaResp semanticSchemaResp) throws Exception {
public QueryStatement convert(QuerySqlReq querySqlReq,
List<ModelSchemaResp> modelSchemaResps) throws Exception {
if (CollectionUtils.isEmpty(modelSchemaResps)) {
if (semanticSchemaResp == null) {
return new QueryStatement();
}
Map<Long, ModelSchemaResp> modelSchemaRespMap = modelSchemaResps.stream()
.collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp));
//1.convert name to bizName
convertNameToBizName(querySqlReq, modelSchemaResps);
convertNameToBizName(querySQLReq, semanticSchemaResp);
//2.functionName corrector
functionNameCorrector(querySqlReq);
functionNameCorrector(querySQLReq, semanticSchemaResp);
//3.correct tableName
correctTableName(querySqlReq);
correctTableName(querySQLReq);
String tableName = SqlParserSelectHelper.getTableName(querySqlReq.getSql());
String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql());
if (StringUtils.isEmpty(tableName)) {
return new QueryStatement();
}
//4.build MetricTables
List<String> allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql());
List<String> metrics = getMetrics(modelSchemaResps, allFields);
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql());
List<String> metrics = getMetrics(semanticSchemaResp, allFields);
QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable();
metricTable.setMetrics(metrics);
Set<String> dimensions = getDimensions(modelSchemaResps, allFields);
Set<String> dimensions = getDimensions(semanticSchemaResp, allFields);
metricTable.setDimensions(new ArrayList<>(dimensions));
@@ -96,36 +91,33 @@ public class QueryReqConverter {
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>());
for (Long modelId : querySqlReq.getModelIds()) {
ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId);
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName()));
}
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
semanticSchemaResp.getModelResps().get(0).getBizName()));
} else {
queryStructReq.setAggregators(
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
Collectors.toList()));
}
AggOption aggOption = getAggOption(querySqlReq);
AggOption aggOption = getAggOption(querySQLReq);
metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq();
BeanUtils.copyProperties(querySqlReq, result);
BeanUtils.copyProperties(querySQLReq, result);
result.setRootPath(querySqlReq.getModelIdStr());
result.setTables(tables);
DatabaseResp database = catalog.getDatabaseByModelId(querySqlReq.getModelIds().get(0));
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
result.setWithAlias(false);
}
//5. do deriveMetric
generateDerivedMetric(querySqlReq.getModelIds(), modelSchemaResps, aggOption, result);
generateDerivedMetric(semanticSchemaResp, aggOption, result);
//6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(querySqlReq.getModelIds()));
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setViewId(querySQLReq.getViewId());
queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryStatement queryStatement = new QueryStatement();
@@ -133,7 +125,7 @@ public class QueryReqConverter {
queryStatement.setParseSqlReq(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setModelIds(querySqlReq.getModelIds());
queryStatement.setViewId(querySQLReq.getViewId());
queryStatement.setEnableLimitWrapper(limitWrapper);
return queryStatement;
@@ -151,8 +143,8 @@ public class QueryReqConverter {
return AggOption.DEFAULT;
}
private void convertNameToBizName(QuerySqlReq databaseReq, List<ModelSchemaResp> modelSchemaResps) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResps);
private void convertNameToBizName(QuerySqlReq databaseReq, SemanticSchemaResp semanticSchemaResp) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
@@ -160,9 +152,8 @@ public class QueryReqConverter {
databaseReq.setSql(replaceFields);
}
private Set<String> getDimensions(List<ModelSchemaResp> modelSchemaResps, List<String> allFields) {
Map<String, String> dimensionLowerToNameMap = modelSchemaResps.stream()
.flatMap(modelSchemaResp -> modelSchemaResp.getDimensions().stream())
private Set<String> getDimensions(SemanticSchemaResp semanticSchemaResp, List<String> allFields) {
Map<String, String> dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName,
(k1, k2) -> k1));
Map<String, String> internalLowerToNameMap = QueryStructUtils.internalCols.stream()
@@ -173,16 +164,15 @@ public class QueryReqConverter {
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet());
}
private List<String> getMetrics(List<ModelSchemaResp> modelSchemaResps, List<String> allFields) {
Map<String, String> metricLowerToNameMap = modelSchemaResps.stream()
.flatMap(modelSchemaResp -> modelSchemaResp.getMetrics().stream())
private List<String> getMetrics(SemanticSchemaResp semanticSchemaResp, List<String> allFields) {
Map<String, String> metricLowerToNameMap = semanticSchemaResp.getMetrics().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName));
return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
}
private void functionNameCorrector(QuerySqlReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0));
private void functionNameCorrector(QuerySqlReq databaseReq, SemanticSchemaResp semanticSchemaResp) {
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
}
@@ -194,15 +184,13 @@ public class QueryReqConverter {
}
}
protected Map<String, String> getFieldNameToBizNameMap(List<ModelSchemaResp> modelSchemaResps) {
protected Map<String, String> getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) {
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = modelSchemaResps.stream().flatMap(modelSchemaResp
-> modelSchemaResp.getDimensions().stream())
Map<String, String> dimensionResults = semanticSchemaResp.getDimensions().stream()
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
Map<String, String> metricResults = modelSchemaResps.stream().flatMap(modelSchemaResp
-> modelSchemaResp.getMetrics().stream())
Map<String, String> metricResults = semanticSchemaResp.getMetrics().stream()
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
@@ -224,12 +212,11 @@ public class QueryReqConverter {
return elements.stream();
}
public void correctTableName(QuerySqlReq databaseReq) {
String sql = databaseReq.getSql();
for (Long modelId : databaseReq.getModelIds()) {
sql = SqlParserReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + modelId);
}
databaseReq.setSql(sql);
public void correctTableName(QuerySqlReq querySqlReq) {
String sql = querySqlReq.getSql();
sql = SqlParserReplaceHelper.replaceTable(sql,
Constants.TABLE_PREFIX + querySqlReq.getViewId());
querySqlReq.setSql(sql);
}
private QueryType getQueryType(AggOption aggOption) {
@@ -241,13 +228,13 @@ public class QueryReqConverter {
return queryType;
}
private void generateDerivedMetric(List<Long> modelIds, List<ModelSchemaResp> modelSchemaResps, AggOption aggOption,
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
ParseSqlReq parseSqlReq) {
String sql = parseSqlReq.getSql();
for (MetricTable metricTable : parseSqlReq.getTables()) {
List<String> measures = new ArrayList<>();
Map<String, String> replaces = new HashMap<>();
generateDerivedMetric(modelIds, modelSchemaResps, aggOption, metricTable.getMetrics(),
generateDerivedMetric(semanticSchemaResp, aggOption, metricTable.getMetrics(),
metricTable.getDimensions(),
measures, replaces);
if (!CollectionUtils.isEmpty(replaces)) {
@@ -263,13 +250,11 @@ public class QueryReqConverter {
parseSqlReq.setSql(sql);
}
private void generateDerivedMetric(List<Long> modelIds, List<ModelSchemaResp> modelSchemaResps, AggOption aggOption,
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
List<String> metrics, List<String> dimensions,
List<String> measures, Map<String, String> replaces) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(modelIds);
List<MetricResp> metricResps = catalog.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = catalog.getDimensions(metaFilter);
List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
// check metrics has derived
if (!metricResps.stream()
.anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(),
@@ -279,14 +264,13 @@ public class QueryReqConverter {
log.info("begin to generateDerivedMetric {} [{}]", aggOption, metrics);
Set<String> allFields = new HashSet<>();
Map<String, Measure> allMeasures = new HashMap<>();
modelSchemaResps.stream().forEach(modelSchemaResp -> {
allFields.addAll(modelSchemaResp.getFieldList());
if (Objects.nonNull(modelSchemaResp.getModelDetail().getMeasures())) {
modelSchemaResp.getModelDetail().getMeasures().stream()
semanticSchemaResp.getModelResps().forEach(modelResp -> {
allFields.addAll(modelResp.getFieldList());
if (Objects.nonNull(modelResp.getModelDetail().getMeasures())) {
modelResp.getModelDetail().getMeasures().stream()
.forEach(mm -> allMeasures.put(mm.getBizName(), mm));
}
});
Set<String> deriveDimension = new HashSet<>();
Set<String> deriveMetric = new HashSet<>();
Set<String> visitedMetric = new HashSet<>();

50
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -1,10 +1,5 @@
package com.tencent.supersonic.headless.server.utils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
@@ -22,9 +17,16 @@ import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -38,11 +40,11 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
@Slf4j
@@ -65,26 +67,27 @@ public class QueryStructUtils {
public QueryStructUtils(
DateModeUtils dateModeUtils,
SqlFilterUtils sqlFilterUtils, Catalog catalog) {
SqlFilterUtils sqlFilterUtils, @Lazy Catalog catalog) {
this.dateModeUtils = dateModeUtils;
this.sqlFilterUtils = sqlFilterUtils;
this.catalog = catalog;
}
private List<Long> getDimensionIds(QueryStructReq queryStructCmd) {
private List<Long> getDimensionIds(QueryStructReq queryStructReq) {
List<Long> dimensionIds = new ArrayList<>();
MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds());
MetaFilter metaFilter = new MetaFilter();
metaFilter.setViewId(queryStructReq.getViewId());
List<DimensionResp> dimensions = catalog.getDimensions(metaFilter);
Map<String, List<DimensionResp>> pair = dimensions.stream()
.collect(Collectors.groupingBy(DimensionResp::getBizName));
for (String group : queryStructCmd.getGroups()) {
for (String group : queryStructReq.getGroups()) {
if (pair.containsKey(group)) {
dimensionIds.add(pair.get(group).get(0).getId());
}
}
List<String> filtersCols = sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter());
List<String> filtersCols = sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter());
for (String col : filtersCols) {
if (pair.containsKey(col)) {
dimensionIds.add(pair.get(col).get(0).getId());
@@ -95,7 +98,8 @@ public class QueryStructUtils {
private List<Long> getMetricIds(QueryStructReq queryStructCmd) {
List<Long> metricIds = new ArrayList<>();
MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds());
MetaFilter metaFilter = new MetaFilter();
metaFilter.setViewId(queryStructCmd.getViewId());
List<MetricResp> metrics = catalog.getMetrics(metaFilter);
Map<String, List<MetricResp>> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName));
for (Aggregator agg : queryStructCmd.getAggregators()) {
@@ -133,12 +137,12 @@ public class QueryStructUtils {
}
public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySqlReq,
List<ModelSchemaResp> modelSchemaRespList) {
SemanticSchemaResp semanticSchemaResp) {
Set<String> resNameSet = getResName(querySqlReq);
Set<String> resNameEnSet = new HashSet<>();
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
List<MetricSchemaResp> metrics = modelSchemaRespList.get(0).getMetrics();
List<DimSchemaResp> dimensions = modelSchemaRespList.get(0).getDimensions();
if (semanticSchemaResp != null) {
List<MetricSchemaResp> metrics = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensions = semanticSchemaResp.getDimensions();
metrics.stream().forEach(o -> {
if (resNameSet.contains(o.getName())) {
resNameEnSet.add(o.getBizName());
@@ -174,8 +178,8 @@ public class QueryStructUtils {
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
List<Long> metricIds = getMetricIds(queryStructCmd);
ItemDateResp dateDate = catalog.getItemDate(
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()),
new ItemDateFilter(metricIds, TypeEnums.METRIC.getName()));
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.name()),
new ItemDateFilter(metricIds, TypeEnums.METRIC.name()));
return dateDate;
}

39
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -1,20 +1,23 @@
package com.tencent.supersonic.headless.server.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -24,11 +27,9 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
@Slf4j
@@ -43,11 +44,6 @@ public class QueryUtils {
@Value("${query.optimizer.enable:true}")
private Boolean optimizeEnable;
private final Catalog catalog;
public QueryUtils(Catalog catalog) {
this.catalog = catalog;
}
@PostConstruct
public void fillPattern() {
@@ -58,10 +54,9 @@ public class QueryUtils {
}
}
public void fillItemNameInfo(SemanticQueryResp queryResultWithColumns, List<Long> modelIds) {
MetaFilter metaFilter = new MetaFilter(modelIds);
List<MetricResp> metricDescList = catalog.getMetrics(metaFilter);
List<DimensionResp> dimensionDescList = catalog.getDimensions(metaFilter);
public void fillItemNameInfo(SemanticQueryResp semanticQueryResp, SemanticSchemaResp semanticSchemaResp) {
List<MetricSchemaResp> metricDescList = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimSchemaResps = semanticSchemaResp.getDimensions();
Map<String, MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, String> namePair = new HashMap<>();
@@ -71,11 +66,11 @@ public class QueryUtils {
namePair.put(metricDesc.getBizName(), metricDesc.getName());
nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name());
});
dimensionDescList.forEach(dimensionDesc -> {
dimSchemaResps.forEach(dimensionDesc -> {
namePair.put(dimensionDesc.getBizName(), dimensionDesc.getName());
nameTypePair.put(dimensionDesc.getBizName(), dimensionDesc.getSemanticType());
});
List<QueryColumn> columns = queryResultWithColumns.getColumns();
List<QueryColumn> columns = semanticQueryResp.getColumns();
columns.forEach(column -> {
String nameEn = getName(column.getNameEn().toLowerCase());
if (nameEn.contains(JOIN_UNDERLINE)) {

48
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -7,18 +7,17 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.QueryOptMode;
import com.tencent.supersonic.headless.api.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.enums.QueryTypeBack;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
@@ -101,22 +100,15 @@ public class StatUtils {
public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
List<String> aggFields = SqlParserSelectHelper.getAggregateFields(querySqlReq.getSql());
List<String> allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql());
queryStatInfo.setModelId(querySqlReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySqlReq.getModelIds().get(0));
List<String> dimensions = new ArrayList<>();
List<String> metrics = new ArrayList<>();
if (Objects.nonNull(modelSchemaResp)) {
dimensions = getFieldNames(allFields, modelSchemaResp.getDimensions());
metrics = getFieldNames(allFields, modelSchemaResp.getMetrics());
}
List<String> dimensions = allFields.stream().filter(aggFields::contains).collect(Collectors.toList());
String userName = getUserName(facadeUser);
try {
queryStatInfo.setTraceId("")
.setModelId(querySqlReq.getModelIds().get(0))
.setUser(userName)
.setViewId(querySqlReq.getViewId())
.setQueryType(QueryType.SQL.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQuerySqlCmd(querySqlReq.toString())
@@ -124,43 +116,49 @@ public class StatUtils {
.setStartTime(System.currentTimeMillis())
.setUseResultCache(true)
.setUseSqlCache(true)
.setMetrics(objectMapper.writeValueAsString(metrics))
.setMetrics(objectMapper.writeValueAsString(aggFields))
.setDimensions(objectMapper.writeValueAsString(dimensions));
if (!CollectionUtils.isEmpty(querySqlReq.getModelIds())) {
queryStatInfo.setModelId(querySqlReq.getModelIds().get(0));
}
} catch (JsonProcessingException e) {
log.error("initStatInfo:{}", e);
}
StatUtils.set(queryStatInfo);
}
public void initStructStatInfo(QueryStructReq queryStructCmd, User facadeUser) {
public void initStructStatInfo(QueryStructReq queryStructReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
String traceId = "";
List<String> dimensions = queryStructCmd.getGroups();
List<String> dimensions = queryStructReq.getGroups();
List<String> metrics = new ArrayList<>();
queryStructCmd.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn()));
queryStructReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn()));
String user = getUserName(facadeUser);
try {
queryStatInfo.setTraceId(traceId)
.setModelId(1L)
.setViewId(queryStructReq.getViewId())
.setUser(user)
.setQueryType(QueryType.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQueryStructCmd(queryStructCmd.toString())
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructCmd.toString()))
.setQueryStructCmd(queryStructReq.toString())
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructReq.toString()))
.setStartTime(System.currentTimeMillis())
.setNativeQuery(queryStructCmd.getQueryType().isNativeAggQuery())
.setGroupByCols(objectMapper.writeValueAsString(queryStructCmd.getGroups()))
.setAggCols(objectMapper.writeValueAsString(queryStructCmd.getAggregators()))
.setOrderByCols(objectMapper.writeValueAsString(queryStructCmd.getOrders()))
.setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery())
.setGroupByCols(objectMapper.writeValueAsString(queryStructReq.getGroups()))
.setAggCols(objectMapper.writeValueAsString(queryStructReq.getAggregators()))
.setOrderByCols(objectMapper.writeValueAsString(queryStructReq.getOrders()))
.setFilterCols(objectMapper.writeValueAsString(
sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter())))
sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter())))
.setUseResultCache(true)
.setUseSqlCache(true)
.setMetrics(objectMapper.writeValueAsString(metrics))
.setDimensions(objectMapper.writeValueAsString(dimensions))
.setQueryOptMode(QueryOptMode.NONE.name());
if (!CollectionUtils.isEmpty(queryStructReq.getModelIds())) {
queryStatInfo.setModelId(queryStructReq.getModelIds().get(0));
}
} catch (JsonProcessingException e) {
e.printStackTrace();
}

5
headless/server/src/main/resources/mapper/StatMapper.xml
View File

@@ -9,6 +9,7 @@
<id column="id" property="id"/>
<result column="trace_id" property="traceId"/>
<result column="model_id" property="modelId"/>
<result column="view_id" property="viewId"/>
<result column="user" property="user"/>
<result column="created_at" property="createdAt"/>
<result column="query_type" property="queryType"/>
@@ -42,13 +43,13 @@
<insert id="createRecord">
insert into s2_query_stat_info
(
trace_id, model_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
trace_id, model_id, view_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics, select_cols, agg_cols, filter_cols, group_by_cols,
order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key, query_opt_mode
)
values
(
#{traceId}, #{modelId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
#{traceId}, #{modelId}, #{viewId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
#{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate}, #{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols},
#{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey}, #{resultCacheKey}, #{queryOptMode}
)

15
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -29,12 +29,8 @@ class HeadlessParserServiceTest {
private static Map<String, SemanticSchema> headlessSchemaMap = new HashMap<>();
public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricCommand, boolean isAgg) {
public static SqlParserResp parser(SemanticSchema semanticSchema, MetricQueryReq metricQueryReq, boolean isAgg) {
SqlParserResp sqlParser = new SqlParserResp();
if (metricCommand.getRootPath().isEmpty()) {
sqlParser.setErrMsg("rootPath empty");
return sqlParser;
}
try {
if (semanticSchema == null) {
sqlParser.setErrMsg("headlessSchema not found");
@@ -42,14 +38,14 @@ class HeadlessParserServiceTest {
}
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setMetricReq(metricCommand);
queryStatement.setMetricReq(metricQueryReq);
aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg));
EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType());
sqlParser.setSql(aggBuilder.getSql(engineType));
sqlParser.setSourceId(aggBuilder.getSourceId());
} catch (Exception e) {
sqlParser.setErrMsg(e.getMessage());
log.error("parser error MetricCommand[{}] error [{}]", metricCommand, e);
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
}
return sqlParser;
}
@@ -123,8 +119,7 @@ class HeadlessParserServiceTest {
identify.setType("primary");
identifies.add(identify);
datasource.setIdentifiers(identifies);
SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build();
SemanticSchema semanticSchema = SemanticSchema.newBuilder("1").build();
SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.getDatasource(datasource));
@@ -168,7 +163,6 @@ class HeadlessParserServiceTest {
//HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
MetricQueryReq metricCommand = new MetricQueryReq();
metricCommand.setRootPath("s2");
metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date")));
metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv")));
metricCommand.setWhere("user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
@@ -181,7 +175,6 @@ class HeadlessParserServiceTest {
addDepartment(semanticSchema);
MetricQueryReq metricCommand2 = new MetricQueryReq();
metricCommand2.setRootPath("s2");
metricCommand2.setDimensions(new ArrayList<>(
Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page")));
metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv")));

22
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java
View File

@@ -1,13 +1,7 @@
package com.tencent.supersonic.headless.server.service;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import com.alibaba.excel.util.FileUtils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq;
@@ -15,28 +9,18 @@ import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.impl.DownloadServiceImpl;
import java.io.File;
import java.util.Date;
import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
class DownloadServiceImplTest {
@Test
void testBatchDownload() throws Exception {
ModelService modelService = Mockito.mock(ModelService.class);
QueryService queryService = Mockito.mock(QueryService.class);
when(modelService.fetchModelSchema(any())).thenReturn(Lists.newArrayList(mockModelSchemaResp()));
when(queryService.queryByReq(any(), any())).thenReturn(mockQueryResult());
DownloadServiceImpl downloadService = new DownloadServiceImpl(modelService, queryService);
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
File file = FileUtils.createTmpFile(fileName);
downloadService.batchDownload(buildBatchDownloadReq(), User.getFakeUser(), file);
}
private ModelSchemaResp mockModelSchemaResp() {

4
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
View File

@@ -20,6 +20,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.service.impl.ViewServiceImpl;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -65,8 +66,9 @@ public class MetricServiceImplTest {
ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class);
CollectService collectService = Mockito.mock(CollectService.class);
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
ViewService viewService = Mockito.mock(ViewServiceImpl.class);
return new MetricServiceImpl(metricRepository, modelService, domainService,
chatGptHelper, collectService, eventPublisher);
chatGptHelper, collectService, viewService, eventPublisher);
}
private MetricReq buildMetricReq() {

4
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java
View File

@@ -76,8 +76,10 @@ class ModelServiceImplTest {
UserService userService = Mockito.mock(UserService.class);
ModelRelaService modelRelaService = Mockito.mock(ModelRelaService.class);
DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class);
ViewService viewService = Mockito.mock(ViewService.class);
return new ModelServiceImpl(modelRepository, databaseService,
dimensionService, metricService, modelRelaService, domainService, userService, dateInfoRepository);
dimensionService, metricService, modelRelaService, domainService,
userService, viewService, dateInfoRepository);
}
private ModelReq mockModelReq() {