topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 21:17:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Feature/model data embedding for chat and support status for metric and dimension (#311)

Browse Source 
* (improvement)(semantic) add offline status for metric and dimension

* (improvement)(chat) add metric recall

---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-02 18:44:58 +08:00
committed by GitHub
parent f4e3922f47
commit ad20380283
89 changed files with 1572 additions and 896 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.query.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
@@ -13,6 +14,7 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
@@ -22,6 +24,7 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.util.Sets;
import org.springframework.beans.BeanUtils;
@@ -89,8 +92,11 @@ public class AuthCommonService {
public Set<String> getHighSensitiveColsByModelId(Long modelId) {
Set<String> highSensitiveCols = new HashSet<>();
List<DimensionResp> highSensitiveDimensions = dimensionService.getHighSensitiveDimension(modelId);
List<MetricResp> highSensitiveMetrics = metricService.getHighSensitiveMetric(modelId);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
List<DimensionResp> highSensitiveDimensions = dimensionService.getDimensions(metaFilter);
List<MetricResp> highSensitiveMetrics = metricService.getMetrics(metaFilter);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java
View File

@@ -118,8 +118,8 @@ public class SchemaServiceImpl implements SchemaService {
}
@Override
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricCmd, User user) {
return metricService.queryMetric(pageMetricCmd, user);
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user) {
return metricService.queryMetric(pageMetricReq, user);
}
@Override

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -5,6 +5,7 @@ import static com.tencent.supersonic.common.pojo.Constants.MINUS;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
@@ -15,6 +16,7 @@ import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
@@ -38,6 +40,7 @@ import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
@@ -271,8 +274,11 @@ public class DataPermissionAOP {
private Set<String> getHighSensitiveColsByModelId(Long modelId) {
Set<String> highSensitiveCols = new HashSet<>();
List<DimensionResp> highSensitiveDimensions = dimensionService.getHighSensitiveDimension(modelId);
List<MetricResp> highSensitiveMetrics = metricService.getHighSensitiveMetric(modelId);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
List<DimensionResp> highSensitiveDimensions = dimensionService.getDimensions(metaFilter);
List<MetricResp> highSensitiveMetrics = metricService.getMetrics(metaFilter);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
@@ -387,7 +393,7 @@ public class DataPermissionAOP {
modelNameCn = modelInfos.get(0).getName();
}
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryStructReq.getModelId());
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds));
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.semantic.query.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
@@ -21,6 +22,7 @@ import java.util.Set;
import java.util.Objects;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
@@ -54,11 +56,12 @@ public class DimValueAspect {
}
Object[] args = joinPoint.getArgs();
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2QLReq.getModelId()));
String sql = queryS2QLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue.
List<FilterExpression> filterExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(queryS2QLReq.getModelId());
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Set<String> fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet());
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
filterExpressionList.stream().forEach(expression -> {
@@ -145,8 +148,8 @@ public class DimValueAspect {
Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0];
Long modelId = queryStructReq.getModelId();
List<DimensionResp> dimensions = dimensionService.getDimensions(modelId);
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId));
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions);
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions);

5
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java
View File

@@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.query.service.AuthCommonService;
import java.util.ArrayList;
import java.util.HashSet;
@@ -169,8 +170,8 @@ public class S2QLDataAspect {
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryS2QLReq.getModelId());
MetaFilter metaFilter = new MetaFilter(modelIds);
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(metaFilter);
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));