mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 19:51:00 +00:00
add metric and dimension name check (#149)
* (improvement)(semantic) add metric and dimension name check * (improvement)(chat) opt QueryResponder recalling history similar solved query --------- Co-authored-by: jolunoluo
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
package com.tencent.supersonic.chat.api.pojo.request;
|
||||
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class SolvedQueryReq {
|
||||
|
||||
private Long queryId;
|
||||
|
||||
private Integer parseId;
|
||||
|
||||
private String queryText;
|
||||
|
||||
private Long modelId;
|
||||
|
||||
private Integer agentId;
|
||||
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
package com.tencent.supersonic.chat.queryresponder;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
|
||||
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
|
||||
@@ -42,12 +44,17 @@ public class DefaultQueryResponder implements QueryResponder {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveSolvedQuery(String queryText, Long queryId, Integer parseId) {
|
||||
public void saveSolvedQuery(SolvedQueryReq solvedQueryReq) {
|
||||
String queryText = solvedQueryReq.getQueryText();
|
||||
try {
|
||||
String uniqueId = generateUniqueId(queryId, parseId);
|
||||
Map<String, String> requestMap = new HashMap<>();
|
||||
String uniqueId = generateUniqueId(solvedQueryReq.getQueryId(), solvedQueryReq.getParseId());
|
||||
Map<String, Object> requestMap = new HashMap<>();
|
||||
requestMap.put("query", queryText);
|
||||
requestMap.put("query_id", uniqueId);
|
||||
Map<String, Object> metaData = new HashMap<>();
|
||||
metaData.put("modelId", String.valueOf(solvedQueryReq.getModelId()));
|
||||
metaData.put("agentId", String.valueOf(solvedQueryReq.getAgentId()));
|
||||
requestMap.put("metadata", metaData);
|
||||
doRequest(embeddingConfig.getSolvedQueryAddPath(),
|
||||
JSONObject.toJSONString(Lists.newArrayList(requestMap)));
|
||||
} catch (Exception e) {
|
||||
@@ -56,7 +63,7 @@ public class DefaultQueryResponder implements QueryResponder {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SolvedQueryRecallResp> recallSolvedQuery(String queryText) {
|
||||
public List<SolvedQueryRecallResp> recallSolvedQuery(String queryText, Integer agentId) {
|
||||
List<SolvedQueryRecallResp> solvedQueryRecallResps = Lists.newArrayList();
|
||||
try {
|
||||
String url = embeddingConfig.getUrl() + embeddingConfig.getSolvedQueryRecallPath() + "?n_results="
|
||||
@@ -66,7 +73,12 @@ public class DefaultQueryResponder implements QueryResponder {
|
||||
headers.setLocation(URI.create(url));
|
||||
URI requestUrl = UriComponentsBuilder
|
||||
.fromHttpUrl(url).build().encode().toUri();
|
||||
String jsonBody = JSONObject.toJSONString(Lists.newArrayList(queryText));
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
map.put("queryTextsList", Lists.newArrayList(queryText));
|
||||
Map<String, Object> filterCondition = new HashMap<>();
|
||||
filterCondition.put("agentId", String.valueOf(agentId));
|
||||
map.put("filterCondition", filterCondition);
|
||||
String jsonBody = JSONObject.toJSONString(map, SerializerFeature.WriteMapNullValue);
|
||||
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
|
||||
log.info("[embedding] request body:{}, url:{}", jsonBody, url);
|
||||
ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity =
|
||||
@@ -80,12 +92,14 @@ public class DefaultQueryResponder implements QueryResponder {
|
||||
for (EmbeddingResp embeddingResp : embeddingResps) {
|
||||
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
|
||||
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
|
||||
if (queryText.equalsIgnoreCase(embeddingRetrieval.getQuery())) {
|
||||
continue;
|
||||
}
|
||||
if (querySet.contains(embeddingRetrieval.getQuery())) {
|
||||
continue;
|
||||
}
|
||||
String id = embeddingRetrieval.getId();
|
||||
SolvedQueryRecallResp solvedQueryRecallResp =
|
||||
SolvedQueryRecallResp.builder()
|
||||
SolvedQueryRecallResp solvedQueryRecallResp = SolvedQueryRecallResp.builder()
|
||||
.queryText(embeddingRetrieval.getQuery())
|
||||
.queryId(getQueryId(id)).parseId(getParseId(id))
|
||||
.build();
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
package com.tencent.supersonic.chat.queryresponder;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
||||
import java.util.List;
|
||||
|
||||
public interface QueryResponder {
|
||||
|
||||
void saveSolvedQuery(String queryText, Long queryId, Integer parseId);
|
||||
void saveSolvedQuery(SolvedQueryReq solvedQueryReq);
|
||||
|
||||
List<SolvedQueryRecallResp> recallSolvedQuery(String queryText);
|
||||
List<SolvedQueryRecallResp> recallSolvedQuery(String queryText, Integer agentId);
|
||||
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
@@ -21,6 +22,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryState;
|
||||
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.CostType;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
|
||||
import com.tencent.supersonic.chat.query.QuerySelector;
|
||||
@@ -150,7 +152,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
.build();
|
||||
}
|
||||
List<SolvedQueryRecallResp> solvedQueryRecallResps =
|
||||
queryResponder.recallSolvedQuery(queryCtx.getRequest().getQueryText());
|
||||
queryResponder.recallSolvedQuery(queryCtx.getRequest().getQueryText(), queryReq.getAgentId());
|
||||
parseResult.setSimilarSolvedQuery(solvedQueryRecallResps);
|
||||
return parseResult;
|
||||
}
|
||||
@@ -172,6 +174,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
|
||||
ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),
|
||||
queryReq.getUser().getName(), queryReq.getParseId());
|
||||
ChatQueryDO chatQueryDO = chatService.getLastQuery(queryReq.getChatId());
|
||||
List<StatisticsDO> timeCostDOList = new ArrayList<>();
|
||||
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
|
||||
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
|
||||
@@ -196,6 +199,11 @@ public class QueryServiceImpl implements QueryService {
|
||||
if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) {
|
||||
chatCtx.setParseInfo(parseInfo);
|
||||
chatService.updateContext(chatCtx);
|
||||
queryResponder.saveSolvedQuery(SolvedQueryReq.builder().parseId(queryReq.getParseId())
|
||||
.queryId(queryReq.getQueryId())
|
||||
.agentId(chatQueryDO.getAgentId())
|
||||
.modelId(parseInfo.getModelId())
|
||||
.queryText(queryReq.getQueryText()).build());
|
||||
}
|
||||
chatCtx.setQueryText(queryReq.getQueryText());
|
||||
chatCtx.setUser(queryReq.getUser().getName());
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.application;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
|
||||
@@ -48,11 +49,14 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
|
||||
@@ -84,25 +88,27 @@ public class DatasourceServiceImpl implements DatasourceService {
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception {
|
||||
preCheck(datasourceReq);
|
||||
checkExist(datasourceReq);
|
||||
Datasource datasource = DatasourceConverter.convert(datasourceReq);
|
||||
log.info("[create datasource] object:{}", JSONObject.toJSONString(datasource));
|
||||
saveDatasource(datasource, user);
|
||||
Optional<DatasourceResp> datasourceDescOptional = getDatasource(datasourceReq.getModelId(),
|
||||
datasourceReq.getBizName());
|
||||
if (!datasourceDescOptional.isPresent()) {
|
||||
throw new RuntimeException("create datasource failed");
|
||||
throw new RuntimeException("创建数据源失败");
|
||||
}
|
||||
DatasourceResp datasourceDesc = datasourceDescOptional.get();
|
||||
datasource.setId(datasourceDesc.getId());
|
||||
DatasourceResp datasourceResp = datasourceDescOptional.get();
|
||||
datasource.setId(datasourceResp.getId());
|
||||
batchCreateDimension(datasource, user);
|
||||
batchCreateMetric(datasource, user);
|
||||
return datasourceDesc;
|
||||
return datasourceResp;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception {
|
||||
preCheck(datasourceReq);
|
||||
Datasource datasource = DatasourceConverter.convert(datasourceReq);
|
||||
@@ -124,20 +130,20 @@ public class DatasourceServiceImpl implements DatasourceService {
|
||||
|
||||
@Override
|
||||
public List<MeasureResp> getMeasureListOfModel(Long modelId) {
|
||||
List<DatasourceResp> datasourceDescs = getDatasourceList(modelId);
|
||||
List<MeasureResp> measureDescs = Lists.newArrayList();
|
||||
if (!CollectionUtils.isEmpty(datasourceDescs)) {
|
||||
for (DatasourceResp datasourceDesc : datasourceDescs) {
|
||||
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
|
||||
List<MeasureResp> measureResps = Lists.newArrayList();
|
||||
if (!CollectionUtils.isEmpty(datasourceResps)) {
|
||||
for (DatasourceResp datasourceDesc : datasourceResps) {
|
||||
DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail();
|
||||
List<Measure> measures = datasourceDetail.getMeasures();
|
||||
if (!CollectionUtils.isEmpty(measures)) {
|
||||
measureDescs.addAll(
|
||||
measureResps.addAll(
|
||||
measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc))
|
||||
.collect(Collectors.toList()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return measureDescs;
|
||||
return measureResps;
|
||||
}
|
||||
|
||||
|
||||
@@ -153,11 +159,11 @@ public class DatasourceServiceImpl implements DatasourceService {
|
||||
|
||||
|
||||
private Optional<DatasourceResp> getDatasource(Long modelId, String bizName) {
|
||||
List<DatasourceResp> datasourceDescs = getDatasourceList(modelId);
|
||||
if (CollectionUtils.isEmpty(datasourceDescs)) {
|
||||
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
|
||||
if (CollectionUtils.isEmpty(datasourceResps)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
for (DatasourceResp datasourceDesc : datasourceDescs) {
|
||||
for (DatasourceResp datasourceDesc : datasourceResps) {
|
||||
if (datasourceDesc.getBizName().equals(bizName)) {
|
||||
return Optional.of(datasourceDesc);
|
||||
}
|
||||
@@ -175,18 +181,51 @@ public class DatasourceServiceImpl implements DatasourceService {
|
||||
|
||||
|
||||
private void preCheck(DatasourceReq datasourceReq) {
|
||||
if (NameCheckUtils.containsSpecialCharacters(datasourceReq.getName())) {
|
||||
String message = String.format("数据源名称[%s]包含特殊字符, 请修改", datasourceReq.getName());
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
List<Dim> dims = datasourceReq.getDimensions();
|
||||
List<Measure> measures = datasourceReq.getMeasures();
|
||||
List<Dim> timeDims = datasourceReq.getTimeDimension();
|
||||
List<Identify> identifies = datasourceReq.getIdentifiers();
|
||||
if (CollectionUtils.isEmpty(dims)) {
|
||||
throw new RuntimeException("缺少维度信息");
|
||||
throw new InvalidArgumentException("缺少维度信息");
|
||||
}
|
||||
if (CollectionUtils.isEmpty(identifies)) {
|
||||
throw new RuntimeException("缺少主键信息");
|
||||
throw new InvalidArgumentException("缺少主键信息");
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) {
|
||||
throw new RuntimeException("有度量时, 不可缺少时间维度");
|
||||
throw new InvalidArgumentException("有度量时, 不可缺少时间维度");
|
||||
}
|
||||
for (Measure measure : measures) {
|
||||
if (StringUtils.isNotBlank(measure.getName())
|
||||
&& NameCheckUtils.containsSpecialCharacters(measure.getName())) {
|
||||
String message = String.format("度量[%s]包含特殊字符, 请修改", measure.getName());
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
}
|
||||
for (Dim dim : dims) {
|
||||
if (StringUtils.isNotBlank(dim.getName())
|
||||
&& NameCheckUtils.containsSpecialCharacters(dim.getName())) {
|
||||
String message = String.format("维度[%s]包含特殊字符, 请修改", dim.getName());
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
}
|
||||
for (Identify identify : identifies) {
|
||||
if (StringUtils.isNotBlank(identify.getName())
|
||||
&& NameCheckUtils.containsSpecialCharacters(identify.getName())) {
|
||||
String message = String.format("主键/外键[%s]包含特殊字符, 请修改", identify.getName());
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void checkExist(DatasourceReq datasourceReq) {
|
||||
Optional<DatasourceResp> datasourceRespOptional = getDatasource(datasourceReq.getModelId(),
|
||||
datasourceReq.getBizName());
|
||||
if (datasourceRespOptional.isPresent()) {
|
||||
throw new InvalidArgumentException("已存在相同名字的数据源:" + datasourceReq.getBizName());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,11 +268,11 @@ public class DatasourceServiceImpl implements DatasourceService {
|
||||
@Override
|
||||
public Map<Long, DatasourceResp> getDatasourceMap() {
|
||||
Map<Long, DatasourceResp> map = new HashMap<>();
|
||||
List<DatasourceResp> datasourceDescs = getDatasourceList();
|
||||
if (CollectionUtils.isEmpty(datasourceDescs)) {
|
||||
List<DatasourceResp> datasourceResps = getDatasourceList();
|
||||
if (CollectionUtils.isEmpty(datasourceResps)) {
|
||||
return map;
|
||||
}
|
||||
return datasourceDescs.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1));
|
||||
return datasourceResps.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.DataAddEvent;
|
||||
import com.tencent.supersonic.common.pojo.DataDeleteEvent;
|
||||
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
|
||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.common.util.ChatGptHelper;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
|
||||
@@ -37,6 +38,7 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -84,7 +86,6 @@ public class DimensionServiceImpl implements DimensionService {
|
||||
log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension));
|
||||
dimension.createdBy(user.getName());
|
||||
saveDimension(dimension);
|
||||
//动态更新字典
|
||||
String type = DictWordType.DIMENSION.getType();
|
||||
DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId());
|
||||
applicationEventPublisher.publishEvent(
|
||||
@@ -113,6 +114,9 @@ public class DimensionServiceImpl implements DimensionService {
|
||||
|
||||
@Override
|
||||
public void updateDimension(DimensionReq dimensionReq, User user) {
|
||||
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
|
||||
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
|
||||
}
|
||||
Dimension dimension = DimensionConverter.convert(dimensionReq);
|
||||
dimension.updatedBy(user.getName());
|
||||
log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
|
||||
@@ -356,13 +360,16 @@ public class DimensionServiceImpl implements DimensionService {
|
||||
Long modelId = dimensionReqs.get(0).getModelId();
|
||||
List<DimensionResp> dimensionResps = getDimensions(modelId);
|
||||
for (DimensionReq dimensionReq : dimensionReqs) {
|
||||
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
|
||||
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
|
||||
}
|
||||
for (DimensionResp dimensionResp : dimensionResps) {
|
||||
if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getBizName())) {
|
||||
throw new RuntimeException(String.format("exist same dimension name:%s", dimensionReq.getName()));
|
||||
if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getName())) {
|
||||
throw new RuntimeException(String.format("存在相同的维度名 :%s", dimensionReq.getName()));
|
||||
}
|
||||
if (dimensionResp.getBizName().equalsIgnoreCase(dimensionReq.getBizName())) {
|
||||
throw new RuntimeException(
|
||||
String.format("exist same dimension bizName:%s", dimensionReq.getBizName()));
|
||||
String.format("存在相同的维度名: %s", dimensionReq.getBizName()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.DataDeleteEvent;
|
||||
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.common.util.ChatGptHelper;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
|
||||
@@ -28,14 +29,13 @@ import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository;
|
||||
import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter;
|
||||
import com.tencent.supersonic.semantic.model.domain.MetricService;
|
||||
import com.tencent.supersonic.semantic.model.domain.pojo.Metric;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
@@ -71,7 +71,7 @@ public class MetricServiceImpl implements MetricService {
|
||||
|
||||
@Override
|
||||
public void creatExprMetric(MetricReq metricReq, User user) {
|
||||
checkExist(Lists.newArrayList(metricReq));
|
||||
check(Lists.newArrayList(metricReq));
|
||||
Metric metric = MetricConverter.convert(metricReq);
|
||||
metric.createdBy(user.getName());
|
||||
log.info("[create metric] object:{}", JSONObject.toJSONString(metric));
|
||||
@@ -90,11 +90,11 @@ public class MetricServiceImpl implements MetricService {
|
||||
}
|
||||
List<Metric> metrics = metricReqs.stream().map(MetricConverter::convert).collect(Collectors.toList());
|
||||
Long modelId = metricReqs.get(0).getModelId();
|
||||
List<MetricResp> metricDescs = getMetricByModelId(modelId);
|
||||
Map<String, MetricResp> metricDescMap = metricDescs.stream()
|
||||
List<MetricResp> metricResps = getMetricByModelId(modelId);
|
||||
Map<String, MetricResp> metricRespMap = metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
List<Metric> metricToInsert = metrics.stream()
|
||||
.filter(metric -> !metricDescMap.containsKey(metric.getBizName())).collect(Collectors.toList());
|
||||
.filter(metric -> !metricRespMap.containsKey(metric.getBizName())).collect(Collectors.toList());
|
||||
log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert));
|
||||
saveMetricBatch(metricToInsert, user);
|
||||
}
|
||||
@@ -166,17 +166,17 @@ public class MetricServiceImpl implements MetricService {
|
||||
|
||||
@Override
|
||||
public MetricResp getMetric(Long modelId, String bizName) {
|
||||
List<MetricResp> metricDescs = getMetricByModelId(modelId);
|
||||
MetricResp metricDesc = null;
|
||||
if (CollectionUtils.isEmpty(metricDescs)) {
|
||||
return metricDesc;
|
||||
List<MetricResp> metricResps = getMetricByModelId(modelId);
|
||||
MetricResp metricResp = null;
|
||||
if (CollectionUtils.isEmpty(metricResps)) {
|
||||
return metricResp;
|
||||
}
|
||||
for (MetricResp metric : metricDescs) {
|
||||
for (MetricResp metric : metricResps) {
|
||||
if (metric.getBizName().equalsIgnoreCase(bizName)) {
|
||||
metricDesc = metric;
|
||||
metricResp = metric;
|
||||
}
|
||||
}
|
||||
return metricDesc;
|
||||
return metricResp;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -226,24 +226,24 @@ public class MetricServiceImpl implements MetricService {
|
||||
|
||||
@Override
|
||||
public List<MetricResp> getHighSensitiveMetric(Long modelId) {
|
||||
List<MetricResp> metricDescs = getMetricByModelId(modelId);
|
||||
if (CollectionUtils.isEmpty(metricDescs)) {
|
||||
return metricDescs;
|
||||
List<MetricResp> metricResps = getMetricByModelId(modelId);
|
||||
if (CollectionUtils.isEmpty(metricResps)) {
|
||||
return metricResps;
|
||||
}
|
||||
return metricDescs.stream()
|
||||
.filter(metricDesc -> SensitiveLevelEnum.HIGH.getCode().equals(metricDesc.getSensitiveLevel()))
|
||||
return metricResps.stream()
|
||||
.filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<MetricResp> getAllHighSensitiveMetric() {
|
||||
List<MetricResp> metricDescs = Lists.newArrayList();
|
||||
List<MetricResp> metricResps = Lists.newArrayList();
|
||||
List<MetricDO> metricDOS = metricRepository.getAllMetricList();
|
||||
if (CollectionUtils.isEmpty(metricDOS)) {
|
||||
return metricDescs;
|
||||
return metricResps;
|
||||
}
|
||||
return convertList(metricDOS.stream()
|
||||
.filter(metricDesc -> SensitiveLevelEnum.HIGH.getCode().equals(metricDesc.getSensitiveLevel()))
|
||||
.filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
|
||||
.collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
@@ -294,23 +294,26 @@ public class MetricServiceImpl implements MetricService {
|
||||
MetricTypeParams typeParams = metricReq.getTypeParams();
|
||||
List<Measure> measures = typeParams.getMeasures();
|
||||
if (CollectionUtils.isEmpty(measures)) {
|
||||
throw new RuntimeException("measure can not be none");
|
||||
throw new InvalidArgumentException("不可缺少度量");
|
||||
}
|
||||
if (StringUtils.isBlank(typeParams.getExpr())) {
|
||||
throw new RuntimeException("expr can not be blank");
|
||||
throw new InvalidArgumentException("表达式不可为空");
|
||||
}
|
||||
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
|
||||
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
|
||||
}
|
||||
}
|
||||
|
||||
private void checkExist(List<MetricReq> exprMetricReqList) {
|
||||
private void check(List<MetricReq> exprMetricReqList) {
|
||||
Long modelId = exprMetricReqList.get(0).getModelId();
|
||||
List<MetricResp> metricDescs = getMetrics(modelId);
|
||||
List<MetricResp> metricResps = getMetrics(modelId);
|
||||
for (MetricReq exprMetricReq : exprMetricReqList) {
|
||||
for (MetricResp metricDesc : metricDescs) {
|
||||
if (metricDesc.getName().equalsIgnoreCase(exprMetricReq.getName())) {
|
||||
throw new RuntimeException(String.format("exist same metric name:%s", metricDesc.getName()));
|
||||
for (MetricResp metricResp : metricResps) {
|
||||
if (metricResp.getName().equalsIgnoreCase(exprMetricReq.getName())) {
|
||||
throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getName()));
|
||||
}
|
||||
if (metricDesc.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) {
|
||||
throw new RuntimeException(String.format("exist same metric en name:%s", metricDesc.getName()));
|
||||
if (metricResp.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) {
|
||||
throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getBizName()));
|
||||
}
|
||||
preCheckMetric(exprMetricReq);
|
||||
}
|
||||
@@ -318,14 +321,14 @@ public class MetricServiceImpl implements MetricService {
|
||||
}
|
||||
|
||||
private List<MetricResp> convertList(List<MetricDO> metricDOS) {
|
||||
List<MetricResp> metricDescs = Lists.newArrayList();
|
||||
List<MetricResp> metricResps = Lists.newArrayList();
|
||||
Map<Long, ModelResp> modelMap = modelService.getModelMap();
|
||||
if (!CollectionUtils.isEmpty(metricDOS)) {
|
||||
metricDescs = metricDOS.stream()
|
||||
metricResps = metricDOS.stream()
|
||||
.map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
return metricDescs;
|
||||
return metricResps;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.tencent.supersonic.semantic.model.domain.utils;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class NameCheckUtils {
|
||||
|
||||
public static boolean containsSpecialCharacters(String str) {
|
||||
String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
|
||||
return Pattern.compile(regex).matcher(str).find();
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user