topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add metric and dimension name check (#149)

Browse Source 
* (improvement)(semantic) add metric and dimension name check

* (improvement)(chat) opt QueryResponder recalling history similar solved query

---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-09-26 20:17:52 +08:00
committed by GitHub
parent 4ad3e1d9cf
commit ff5479f1a2
8 changed files with 176 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.application;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
@@ -48,11 +49,14 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
@@ -84,25 +88,27 @@ public class DatasourceServiceImpl implements DatasourceService {
}
@Override
@Transactional
public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception {
preCheck(datasourceReq);
checkExist(datasourceReq);
Datasource datasource = DatasourceConverter.convert(datasourceReq);
log.info("[create datasource] object:{}", JSONObject.toJSONString(datasource));
saveDatasource(datasource, user);
Optional<DatasourceResp> datasourceDescOptional = getDatasource(datasourceReq.getModelId(),
datasourceReq.getBizName());
if (!datasourceDescOptional.isPresent()) {
throw new RuntimeException("create datasource failed");
throw new RuntimeException("创建数据源失败");
}
DatasourceResp datasourceDesc = datasourceDescOptional.get();
datasource.setId(datasourceDesc.getId());
DatasourceResp datasourceResp = datasourceDescOptional.get();
datasource.setId(datasourceResp.getId());
batchCreateDimension(datasource, user);
batchCreateMetric(datasource, user);
return datasourceDesc;
return datasourceResp;
}
@Override
@Transactional
public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception {
preCheck(datasourceReq);
Datasource datasource = DatasourceConverter.convert(datasourceReq);
@@ -124,20 +130,20 @@ public class DatasourceServiceImpl implements DatasourceService {
@Override
public List<MeasureResp> getMeasureListOfModel(Long modelId) {
List<DatasourceResp> datasourceDescs = getDatasourceList(modelId);
List<MeasureResp> measureDescs = Lists.newArrayList();
if (!CollectionUtils.isEmpty(datasourceDescs)) {
for (DatasourceResp datasourceDesc : datasourceDescs) {
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
List<MeasureResp> measureResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(datasourceResps)) {
for (DatasourceResp datasourceDesc : datasourceResps) {
DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail();
List<Measure> measures = datasourceDetail.getMeasures();
if (!CollectionUtils.isEmpty(measures)) {
measureDescs.addAll(
measureResps.addAll(
measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc))
.collect(Collectors.toList()));
}
}
}
return measureDescs;
return measureResps;
}
@@ -153,11 +159,11 @@ public class DatasourceServiceImpl implements DatasourceService {
private Optional<DatasourceResp> getDatasource(Long modelId, String bizName) {
List<DatasourceResp> datasourceDescs = getDatasourceList(modelId);
if (CollectionUtils.isEmpty(datasourceDescs)) {
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
if (CollectionUtils.isEmpty(datasourceResps)) {
return Optional.empty();
}
for (DatasourceResp datasourceDesc : datasourceDescs) {
for (DatasourceResp datasourceDesc : datasourceResps) {
if (datasourceDesc.getBizName().equals(bizName)) {
return Optional.of(datasourceDesc);
}
@@ -175,18 +181,51 @@ public class DatasourceServiceImpl implements DatasourceService {
private void preCheck(DatasourceReq datasourceReq) {
if (NameCheckUtils.containsSpecialCharacters(datasourceReq.getName())) {
String message = String.format("数据源名称[%s]包含特殊字符, 请修改", datasourceReq.getName());
throw new InvalidArgumentException(message);
}
List<Dim> dims = datasourceReq.getDimensions();
List<Measure> measures = datasourceReq.getMeasures();
List<Dim> timeDims = datasourceReq.getTimeDimension();
List<Identify> identifies = datasourceReq.getIdentifiers();
if (CollectionUtils.isEmpty(dims)) {
throw new RuntimeException("缺少维度信息");
throw new InvalidArgumentException("缺少维度信息");
}
if (CollectionUtils.isEmpty(identifies)) {
throw new RuntimeException("缺少主键信息");
throw new InvalidArgumentException("缺少主键信息");
}
if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) {
throw new RuntimeException("有度量时, 不可缺少时间维度");
throw new InvalidArgumentException("有度量时, 不可缺少时间维度");
}
for (Measure measure : measures) {
if (StringUtils.isNotBlank(measure.getName())
&& NameCheckUtils.containsSpecialCharacters(measure.getName())) {
String message = String.format("度量[%s]包含特殊字符, 请修改", measure.getName());
throw new InvalidArgumentException(message);
}
}
for (Dim dim : dims) {
if (StringUtils.isNotBlank(dim.getName())
&& NameCheckUtils.containsSpecialCharacters(dim.getName())) {
String message = String.format("维度[%s]包含特殊字符, 请修改", dim.getName());
throw new InvalidArgumentException(message);
}
}
for (Identify identify : identifies) {
if (StringUtils.isNotBlank(identify.getName())
&& NameCheckUtils.containsSpecialCharacters(identify.getName())) {
String message = String.format("主键/外键[%s]包含特殊字符, 请修改", identify.getName());
throw new InvalidArgumentException(message);
}
}
}
private void checkExist(DatasourceReq datasourceReq) {
Optional<DatasourceResp> datasourceRespOptional = getDatasource(datasourceReq.getModelId(),
datasourceReq.getBizName());
if (datasourceRespOptional.isPresent()) {
throw new InvalidArgumentException("已存在相同名字的数据源:" + datasourceReq.getBizName());
}
}
@@ -229,11 +268,11 @@ public class DatasourceServiceImpl implements DatasourceService {
@Override
public Map<Long, DatasourceResp> getDatasourceMap() {
Map<Long, DatasourceResp> map = new HashMap<>();
List<DatasourceResp> datasourceDescs = getDatasourceList();
if (CollectionUtils.isEmpty(datasourceDescs)) {
List<DatasourceResp> datasourceResps = getDatasourceList();
if (CollectionUtils.isEmpty(datasourceResps)) {
return map;
}
return datasourceDescs.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1));
return datasourceResps.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1));
}

15
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java
View File

@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.DataAddEvent;
import com.tencent.supersonic.common.pojo.DataDeleteEvent;
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
@@ -37,6 +38,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -84,7 +86,6 @@ public class DimensionServiceImpl implements DimensionService {
log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension));
dimension.createdBy(user.getName());
saveDimension(dimension);
//动态更新字典
String type = DictWordType.DIMENSION.getType();
DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId());
applicationEventPublisher.publishEvent(
@@ -113,6 +114,9 @@ public class DimensionServiceImpl implements DimensionService {
@Override
public void updateDimension(DimensionReq dimensionReq, User user) {
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
Dimension dimension = DimensionConverter.convert(dimensionReq);
dimension.updatedBy(user.getName());
log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
@@ -356,13 +360,16 @@ public class DimensionServiceImpl implements DimensionService {
Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensions(modelId);
for (DimensionReq dimensionReq : dimensionReqs) {
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
for (DimensionResp dimensionResp : dimensionResps) {
if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getBizName())) {
throw new RuntimeException(String.format("exist same dimension name:%s", dimensionReq.getName()));
if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getName())) {
throw new RuntimeException(String.format("存在相同的维度名 :%s", dimensionReq.getName()));
}
if (dimensionResp.getBizName().equalsIgnoreCase(dimensionReq.getBizName())) {
throw new RuntimeException(
String.format("exist same dimension bizName:%s", dimensionReq.getBizName()));
String.format("存在相同的维度名: %s", dimensionReq.getBizName()));
}
}
}

69
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java
View File

@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.DataDeleteEvent;
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
@@ -28,14 +29,13 @@ import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository;
import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.pojo.Metric;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@@ -71,7 +71,7 @@ public class MetricServiceImpl implements MetricService {
@Override
public void creatExprMetric(MetricReq metricReq, User user) {
checkExist(Lists.newArrayList(metricReq));
check(Lists.newArrayList(metricReq));
Metric metric = MetricConverter.convert(metricReq);
metric.createdBy(user.getName());
log.info("[create metric] object:{}", JSONObject.toJSONString(metric));
@@ -90,11 +90,11 @@ public class MetricServiceImpl implements MetricService {
}
List<Metric> metrics = metricReqs.stream().map(MetricConverter::convert).collect(Collectors.toList());
Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricDescs = getMetricByModelId(modelId);
Map<String, MetricResp> metricDescMap = metricDescs.stream()
List<MetricResp> metricResps = getMetricByModelId(modelId);
Map<String, MetricResp> metricRespMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
List<Metric> metricToInsert = metrics.stream()
.filter(metric -> !metricDescMap.containsKey(metric.getBizName())).collect(Collectors.toList());
.filter(metric -> !metricRespMap.containsKey(metric.getBizName())).collect(Collectors.toList());
log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert));
saveMetricBatch(metricToInsert, user);
}
@@ -166,17 +166,17 @@ public class MetricServiceImpl implements MetricService {
@Override
public MetricResp getMetric(Long modelId, String bizName) {
List<MetricResp> metricDescs = getMetricByModelId(modelId);
MetricResp metricDesc = null;
if (CollectionUtils.isEmpty(metricDescs)) {
return metricDesc;
List<MetricResp> metricResps = getMetricByModelId(modelId);
MetricResp metricResp = null;
if (CollectionUtils.isEmpty(metricResps)) {
return metricResp;
}
for (MetricResp metric : metricDescs) {
for (MetricResp metric : metricResps) {
if (metric.getBizName().equalsIgnoreCase(bizName)) {
metricDesc = metric;
metricResp = metric;
}
}
return metricDesc;
return metricResp;
}
@Override
@@ -226,24 +226,24 @@ public class MetricServiceImpl implements MetricService {
@Override
public List<MetricResp> getHighSensitiveMetric(Long modelId) {
List<MetricResp> metricDescs = getMetricByModelId(modelId);
if (CollectionUtils.isEmpty(metricDescs)) {
return metricDescs;
List<MetricResp> metricResps = getMetricByModelId(modelId);
if (CollectionUtils.isEmpty(metricResps)) {
return metricResps;
}
return metricDescs.stream()
.filter(metricDesc -> SensitiveLevelEnum.HIGH.getCode().equals(metricDesc.getSensitiveLevel()))
return metricResps.stream()
.filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
.collect(Collectors.toList());
}
@Override
public List<MetricResp> getAllHighSensitiveMetric() {
List<MetricResp> metricDescs = Lists.newArrayList();
List<MetricResp> metricResps = Lists.newArrayList();
List<MetricDO> metricDOS = metricRepository.getAllMetricList();
if (CollectionUtils.isEmpty(metricDOS)) {
return metricDescs;
return metricResps;
}
return convertList(metricDOS.stream()
.filter(metricDesc -> SensitiveLevelEnum.HIGH.getCode().equals(metricDesc.getSensitiveLevel()))
.filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
.collect(Collectors.toList()));
}
@@ -294,23 +294,26 @@ public class MetricServiceImpl implements MetricService {
MetricTypeParams typeParams = metricReq.getTypeParams();
List<Measure> measures = typeParams.getMeasures();
if (CollectionUtils.isEmpty(measures)) {
throw new RuntimeException("measure can not be none");
throw new InvalidArgumentException("不可缺少度量");
}
if (StringUtils.isBlank(typeParams.getExpr())) {
throw new RuntimeException("expr can not be blank");
throw new InvalidArgumentException("表达式不可为空");
}
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
}
private void checkExist(List<MetricReq> exprMetricReqList) {
private void check(List<MetricReq> exprMetricReqList) {
Long modelId = exprMetricReqList.get(0).getModelId();
List<MetricResp> metricDescs = getMetrics(modelId);
List<MetricResp> metricResps = getMetrics(modelId);
for (MetricReq exprMetricReq : exprMetricReqList) {
for (MetricResp metricDesc : metricDescs) {
if (metricDesc.getName().equalsIgnoreCase(exprMetricReq.getName())) {
throw new RuntimeException(String.format("exist same metric name:%s", metricDesc.getName()));
for (MetricResp metricResp : metricResps) {
if (metricResp.getName().equalsIgnoreCase(exprMetricReq.getName())) {
throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getName()));
}
if (metricDesc.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) {
throw new RuntimeException(String.format("exist same metric en name:%s", metricDesc.getName()));
if (metricResp.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) {
throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getBizName()));
}
preCheckMetric(exprMetricReq);
}
@@ -318,14 +321,14 @@ public class MetricServiceImpl implements MetricService {
}
private List<MetricResp> convertList(List<MetricDO> metricDOS) {
List<MetricResp> metricDescs = Lists.newArrayList();
List<MetricResp> metricResps = Lists.newArrayList();
Map<Long, ModelResp> modelMap = modelService.getModelMap();
if (!CollectionUtils.isEmpty(metricDOS)) {
metricDescs = metricDOS.stream()
metricResps = metricDOS.stream()
.map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap))
.collect(Collectors.toList());
}
return metricDescs;
return metricResps;
}

12
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/NameCheckUtils.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.semantic.model.domain.utils;
import java.util.regex.Pattern;
public class NameCheckUtils {
public static boolean containsSpecialCharacters(String str) {
String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
return Pattern.compile(regex).matcher(str).find();
}
}