add metric and dimension name check (#149)

* (improvement)(semantic) add metric and dimension name check

* (improvement)(chat) opt QueryResponder recalling history similar solved query

---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-09-26 20:17:52 +08:00
committed by GitHub
parent 4ad3e1d9cf
commit ff5479f1a2
8 changed files with 176 additions and 67 deletions

View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class SolvedQueryReq {
private Long queryId;
private Integer parseId;
private String queryText;
private Long modelId;
private Integer agentId;
}

View File

@@ -1,7 +1,9 @@
package com.tencent.supersonic.chat.queryresponder; package com.tencent.supersonic.chat.queryresponder;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig; import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp; import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
@@ -42,12 +44,17 @@ public class DefaultQueryResponder implements QueryResponder {
} }
@Override @Override
public void saveSolvedQuery(String queryText, Long queryId, Integer parseId) { public void saveSolvedQuery(SolvedQueryReq solvedQueryReq) {
String queryText = solvedQueryReq.getQueryText();
try { try {
String uniqueId = generateUniqueId(queryId, parseId); String uniqueId = generateUniqueId(solvedQueryReq.getQueryId(), solvedQueryReq.getParseId());
Map<String, String> requestMap = new HashMap<>(); Map<String, Object> requestMap = new HashMap<>();
requestMap.put("query", queryText); requestMap.put("query", queryText);
requestMap.put("query_id", uniqueId); requestMap.put("query_id", uniqueId);
Map<String, Object> metaData = new HashMap<>();
metaData.put("modelId", String.valueOf(solvedQueryReq.getModelId()));
metaData.put("agentId", String.valueOf(solvedQueryReq.getAgentId()));
requestMap.put("metadata", metaData);
doRequest(embeddingConfig.getSolvedQueryAddPath(), doRequest(embeddingConfig.getSolvedQueryAddPath(),
JSONObject.toJSONString(Lists.newArrayList(requestMap))); JSONObject.toJSONString(Lists.newArrayList(requestMap)));
} catch (Exception e) { } catch (Exception e) {
@@ -56,7 +63,7 @@ public class DefaultQueryResponder implements QueryResponder {
} }
@Override @Override
public List<SolvedQueryRecallResp> recallSolvedQuery(String queryText) { public List<SolvedQueryRecallResp> recallSolvedQuery(String queryText, Integer agentId) {
List<SolvedQueryRecallResp> solvedQueryRecallResps = Lists.newArrayList(); List<SolvedQueryRecallResp> solvedQueryRecallResps = Lists.newArrayList();
try { try {
String url = embeddingConfig.getUrl() + embeddingConfig.getSolvedQueryRecallPath() + "?n_results=" String url = embeddingConfig.getUrl() + embeddingConfig.getSolvedQueryRecallPath() + "?n_results="
@@ -66,7 +73,12 @@ public class DefaultQueryResponder implements QueryResponder {
headers.setLocation(URI.create(url)); headers.setLocation(URI.create(url));
URI requestUrl = UriComponentsBuilder URI requestUrl = UriComponentsBuilder
.fromHttpUrl(url).build().encode().toUri(); .fromHttpUrl(url).build().encode().toUri();
String jsonBody = JSONObject.toJSONString(Lists.newArrayList(queryText)); Map<String, Object> map = new HashMap<>();
map.put("queryTextsList", Lists.newArrayList(queryText));
Map<String, Object> filterCondition = new HashMap<>();
filterCondition.put("agentId", String.valueOf(agentId));
map.put("filterCondition", filterCondition);
String jsonBody = JSONObject.toJSONString(map, SerializerFeature.WriteMapNullValue);
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers); HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
log.info("[embedding] request body:{}, url:{}", jsonBody, url); log.info("[embedding] request body:{}, url:{}", jsonBody, url);
ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity = ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity =
@@ -80,12 +92,14 @@ public class DefaultQueryResponder implements QueryResponder {
for (EmbeddingResp embeddingResp : embeddingResps) { for (EmbeddingResp embeddingResp : embeddingResps) {
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval(); List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) { for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
if (queryText.equalsIgnoreCase(embeddingRetrieval.getQuery())) {
continue;
}
if (querySet.contains(embeddingRetrieval.getQuery())) { if (querySet.contains(embeddingRetrieval.getQuery())) {
continue; continue;
} }
String id = embeddingRetrieval.getId(); String id = embeddingRetrieval.getId();
SolvedQueryRecallResp solvedQueryRecallResp = SolvedQueryRecallResp solvedQueryRecallResp = SolvedQueryRecallResp.builder()
SolvedQueryRecallResp.builder()
.queryText(embeddingRetrieval.getQuery()) .queryText(embeddingRetrieval.getQuery())
.queryId(getQueryId(id)).parseId(getParseId(id)) .queryId(getQueryId(id)).parseId(getParseId(id))
.build(); .build();

View File

@@ -1,12 +1,13 @@
package com.tencent.supersonic.chat.queryresponder; package com.tencent.supersonic.chat.queryresponder;
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import java.util.List; import java.util.List;
public interface QueryResponder { public interface QueryResponder {
void saveSolvedQuery(String queryText, Long queryId, Integer parseId); void saveSolvedQuery(SolvedQueryReq solvedQueryReq);
List<SolvedQueryRecallResp> recallSolvedQuery(String queryText); List<SolvedQueryRecallResp> recallSolvedQuery(String queryText, Integer agentId);
} }

View File

@@ -14,6 +14,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
@@ -21,6 +22,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.persistence.dataobject.CostType; import com.tencent.supersonic.chat.persistence.dataobject.CostType;
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO; import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.chat.query.QuerySelector; import com.tencent.supersonic.chat.query.QuerySelector;
@@ -150,7 +152,7 @@ public class QueryServiceImpl implements QueryService {
.build(); .build();
} }
List<SolvedQueryRecallResp> solvedQueryRecallResps = List<SolvedQueryRecallResp> solvedQueryRecallResps =
queryResponder.recallSolvedQuery(queryCtx.getRequest().getQueryText()); queryResponder.recallSolvedQuery(queryCtx.getRequest().getQueryText(), queryReq.getAgentId());
parseResult.setSimilarSolvedQuery(solvedQueryRecallResps); parseResult.setSimilarSolvedQuery(solvedQueryRecallResps);
return parseResult; return parseResult;
} }
@@ -172,6 +174,7 @@ public class QueryServiceImpl implements QueryService {
public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception { public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(), ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),
queryReq.getUser().getName(), queryReq.getParseId()); queryReq.getUser().getName(), queryReq.getParseId());
ChatQueryDO chatQueryDO = chatService.getLastQuery(queryReq.getChatId());
List<StatisticsDO> timeCostDOList = new ArrayList<>(); List<StatisticsDO> timeCostDOList = new ArrayList<>();
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
@@ -196,6 +199,11 @@ public class QueryServiceImpl implements QueryService {
if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) { if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) {
chatCtx.setParseInfo(parseInfo); chatCtx.setParseInfo(parseInfo);
chatService.updateContext(chatCtx); chatService.updateContext(chatCtx);
queryResponder.saveSolvedQuery(SolvedQueryReq.builder().parseId(queryReq.getParseId())
.queryId(queryReq.getQueryId())
.agentId(chatQueryDO.getAgentId())
.modelId(parseInfo.getModelId())
.queryText(queryReq.getQueryText()).build());
} }
chatCtx.setQueryText(queryReq.getQueryText()); chatCtx.setQueryText(queryReq.getQueryText());
chatCtx.setUser(queryReq.getUser().getName()); chatCtx.setUser(queryReq.getUser().getName());

View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.application;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
import com.tencent.supersonic.semantic.api.model.pojo.Dim; import com.tencent.supersonic.semantic.api.model.pojo.Dim;
@@ -48,11 +49,14 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings; import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
@@ -84,25 +88,27 @@ public class DatasourceServiceImpl implements DatasourceService {
} }
@Override @Override
@Transactional
public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception { public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception {
preCheck(datasourceReq); preCheck(datasourceReq);
checkExist(datasourceReq);
Datasource datasource = DatasourceConverter.convert(datasourceReq); Datasource datasource = DatasourceConverter.convert(datasourceReq);
log.info("[create datasource] object:{}", JSONObject.toJSONString(datasource)); log.info("[create datasource] object:{}", JSONObject.toJSONString(datasource));
saveDatasource(datasource, user); saveDatasource(datasource, user);
Optional<DatasourceResp> datasourceDescOptional = getDatasource(datasourceReq.getModelId(), Optional<DatasourceResp> datasourceDescOptional = getDatasource(datasourceReq.getModelId(),
datasourceReq.getBizName()); datasourceReq.getBizName());
if (!datasourceDescOptional.isPresent()) { if (!datasourceDescOptional.isPresent()) {
throw new RuntimeException("create datasource failed"); throw new RuntimeException("创建数据源失败");
} }
DatasourceResp datasourceDesc = datasourceDescOptional.get(); DatasourceResp datasourceResp = datasourceDescOptional.get();
datasource.setId(datasourceDesc.getId()); datasource.setId(datasourceResp.getId());
batchCreateDimension(datasource, user); batchCreateDimension(datasource, user);
batchCreateMetric(datasource, user); batchCreateMetric(datasource, user);
return datasourceDesc; return datasourceResp;
} }
@Override @Override
@Transactional
public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception { public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception {
preCheck(datasourceReq); preCheck(datasourceReq);
Datasource datasource = DatasourceConverter.convert(datasourceReq); Datasource datasource = DatasourceConverter.convert(datasourceReq);
@@ -124,20 +130,20 @@ public class DatasourceServiceImpl implements DatasourceService {
@Override @Override
public List<MeasureResp> getMeasureListOfModel(Long modelId) { public List<MeasureResp> getMeasureListOfModel(Long modelId) {
List<DatasourceResp> datasourceDescs = getDatasourceList(modelId); List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
List<MeasureResp> measureDescs = Lists.newArrayList(); List<MeasureResp> measureResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(datasourceDescs)) { if (!CollectionUtils.isEmpty(datasourceResps)) {
for (DatasourceResp datasourceDesc : datasourceDescs) { for (DatasourceResp datasourceDesc : datasourceResps) {
DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail(); DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail();
List<Measure> measures = datasourceDetail.getMeasures(); List<Measure> measures = datasourceDetail.getMeasures();
if (!CollectionUtils.isEmpty(measures)) { if (!CollectionUtils.isEmpty(measures)) {
measureDescs.addAll( measureResps.addAll(
measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc)) measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
} }
} }
return measureDescs; return measureResps;
} }
@@ -153,11 +159,11 @@ public class DatasourceServiceImpl implements DatasourceService {
private Optional<DatasourceResp> getDatasource(Long modelId, String bizName) { private Optional<DatasourceResp> getDatasource(Long modelId, String bizName) {
List<DatasourceResp> datasourceDescs = getDatasourceList(modelId); List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
if (CollectionUtils.isEmpty(datasourceDescs)) { if (CollectionUtils.isEmpty(datasourceResps)) {
return Optional.empty(); return Optional.empty();
} }
for (DatasourceResp datasourceDesc : datasourceDescs) { for (DatasourceResp datasourceDesc : datasourceResps) {
if (datasourceDesc.getBizName().equals(bizName)) { if (datasourceDesc.getBizName().equals(bizName)) {
return Optional.of(datasourceDesc); return Optional.of(datasourceDesc);
} }
@@ -175,18 +181,51 @@ public class DatasourceServiceImpl implements DatasourceService {
private void preCheck(DatasourceReq datasourceReq) { private void preCheck(DatasourceReq datasourceReq) {
if (NameCheckUtils.containsSpecialCharacters(datasourceReq.getName())) {
String message = String.format("数据源名称[%s]包含特殊字符, 请修改", datasourceReq.getName());
throw new InvalidArgumentException(message);
}
List<Dim> dims = datasourceReq.getDimensions(); List<Dim> dims = datasourceReq.getDimensions();
List<Measure> measures = datasourceReq.getMeasures(); List<Measure> measures = datasourceReq.getMeasures();
List<Dim> timeDims = datasourceReq.getTimeDimension(); List<Dim> timeDims = datasourceReq.getTimeDimension();
List<Identify> identifies = datasourceReq.getIdentifiers(); List<Identify> identifies = datasourceReq.getIdentifiers();
if (CollectionUtils.isEmpty(dims)) { if (CollectionUtils.isEmpty(dims)) {
throw new RuntimeException("缺少维度信息"); throw new InvalidArgumentException("缺少维度信息");
} }
if (CollectionUtils.isEmpty(identifies)) { if (CollectionUtils.isEmpty(identifies)) {
throw new RuntimeException("缺少主键信息"); throw new InvalidArgumentException("缺少主键信息");
} }
if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) { if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) {
throw new RuntimeException("有度量时, 不可缺少时间维度"); throw new InvalidArgumentException("有度量时, 不可缺少时间维度");
}
for (Measure measure : measures) {
if (StringUtils.isNotBlank(measure.getName())
&& NameCheckUtils.containsSpecialCharacters(measure.getName())) {
String message = String.format("度量[%s]包含特殊字符, 请修改", measure.getName());
throw new InvalidArgumentException(message);
}
}
for (Dim dim : dims) {
if (StringUtils.isNotBlank(dim.getName())
&& NameCheckUtils.containsSpecialCharacters(dim.getName())) {
String message = String.format("维度[%s]包含特殊字符, 请修改", dim.getName());
throw new InvalidArgumentException(message);
}
}
for (Identify identify : identifies) {
if (StringUtils.isNotBlank(identify.getName())
&& NameCheckUtils.containsSpecialCharacters(identify.getName())) {
String message = String.format("主键/外键[%s]包含特殊字符, 请修改", identify.getName());
throw new InvalidArgumentException(message);
}
}
}
private void checkExist(DatasourceReq datasourceReq) {
Optional<DatasourceResp> datasourceRespOptional = getDatasource(datasourceReq.getModelId(),
datasourceReq.getBizName());
if (datasourceRespOptional.isPresent()) {
throw new InvalidArgumentException("已存在相同名字的数据源:" + datasourceReq.getBizName());
} }
} }
@@ -229,11 +268,11 @@ public class DatasourceServiceImpl implements DatasourceService {
@Override @Override
public Map<Long, DatasourceResp> getDatasourceMap() { public Map<Long, DatasourceResp> getDatasourceMap() {
Map<Long, DatasourceResp> map = new HashMap<>(); Map<Long, DatasourceResp> map = new HashMap<>();
List<DatasourceResp> datasourceDescs = getDatasourceList(); List<DatasourceResp> datasourceResps = getDatasourceList();
if (CollectionUtils.isEmpty(datasourceDescs)) { if (CollectionUtils.isEmpty(datasourceResps)) {
return map; return map;
} }
return datasourceDescs.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1)); return datasourceResps.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1));
} }

View File

@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.DataAddEvent;
import com.tencent.supersonic.common.pojo.DataDeleteEvent; import com.tencent.supersonic.common.pojo.DataDeleteEvent;
import com.tencent.supersonic.common.pojo.DataUpdateEvent; import com.tencent.supersonic.common.pojo.DataUpdateEvent;
import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail; import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
@@ -37,6 +38,7 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -84,7 +86,6 @@ public class DimensionServiceImpl implements DimensionService {
log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension)); log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension));
dimension.createdBy(user.getName()); dimension.createdBy(user.getName());
saveDimension(dimension); saveDimension(dimension);
//动态更新字典
String type = DictWordType.DIMENSION.getType(); String type = DictWordType.DIMENSION.getType();
DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId()); DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId());
applicationEventPublisher.publishEvent( applicationEventPublisher.publishEvent(
@@ -113,6 +114,9 @@ public class DimensionServiceImpl implements DimensionService {
@Override @Override
public void updateDimension(DimensionReq dimensionReq, User user) { public void updateDimension(DimensionReq dimensionReq, User user) {
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
Dimension dimension = DimensionConverter.convert(dimensionReq); Dimension dimension = DimensionConverter.convert(dimensionReq);
dimension.updatedBy(user.getName()); dimension.updatedBy(user.getName());
log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension)); log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
@@ -356,13 +360,16 @@ public class DimensionServiceImpl implements DimensionService {
Long modelId = dimensionReqs.get(0).getModelId(); Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensions(modelId); List<DimensionResp> dimensionResps = getDimensions(modelId);
for (DimensionReq dimensionReq : dimensionReqs) { for (DimensionReq dimensionReq : dimensionReqs) {
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
for (DimensionResp dimensionResp : dimensionResps) { for (DimensionResp dimensionResp : dimensionResps) {
if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getBizName())) { if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getName())) {
throw new RuntimeException(String.format("exist same dimension name:%s", dimensionReq.getName())); throw new RuntimeException(String.format("存在相同的维度名 :%s", dimensionReq.getName()));
} }
if (dimensionResp.getBizName().equalsIgnoreCase(dimensionReq.getBizName())) { if (dimensionResp.getBizName().equalsIgnoreCase(dimensionReq.getBizName())) {
throw new RuntimeException( throw new RuntimeException(
String.format("exist same dimension bizName:%s", dimensionReq.getBizName())); String.format("存在相同的维度名: %s", dimensionReq.getBizName()));
} }
} }
} }

View File

@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.DataDeleteEvent;
import com.tencent.supersonic.common.pojo.DataUpdateEvent; import com.tencent.supersonic.common.pojo.DataUpdateEvent;
import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.semantic.api.model.pojo.Measure; import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams; import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
@@ -28,14 +29,13 @@ import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository;
import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter; import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter;
import com.tencent.supersonic.semantic.model.domain.MetricService; import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.pojo.Metric; import com.tencent.supersonic.semantic.model.domain.pojo.Metric;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
@@ -71,7 +71,7 @@ public class MetricServiceImpl implements MetricService {
@Override @Override
public void creatExprMetric(MetricReq metricReq, User user) { public void creatExprMetric(MetricReq metricReq, User user) {
checkExist(Lists.newArrayList(metricReq)); check(Lists.newArrayList(metricReq));
Metric metric = MetricConverter.convert(metricReq); Metric metric = MetricConverter.convert(metricReq);
metric.createdBy(user.getName()); metric.createdBy(user.getName());
log.info("[create metric] object:{}", JSONObject.toJSONString(metric)); log.info("[create metric] object:{}", JSONObject.toJSONString(metric));
@@ -90,11 +90,11 @@ public class MetricServiceImpl implements MetricService {
} }
List<Metric> metrics = metricReqs.stream().map(MetricConverter::convert).collect(Collectors.toList()); List<Metric> metrics = metricReqs.stream().map(MetricConverter::convert).collect(Collectors.toList());
Long modelId = metricReqs.get(0).getModelId(); Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricDescs = getMetricByModelId(modelId); List<MetricResp> metricResps = getMetricByModelId(modelId);
Map<String, MetricResp> metricDescMap = metricDescs.stream() Map<String, MetricResp> metricRespMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
List<Metric> metricToInsert = metrics.stream() List<Metric> metricToInsert = metrics.stream()
.filter(metric -> !metricDescMap.containsKey(metric.getBizName())).collect(Collectors.toList()); .filter(metric -> !metricRespMap.containsKey(metric.getBizName())).collect(Collectors.toList());
log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert)); log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert));
saveMetricBatch(metricToInsert, user); saveMetricBatch(metricToInsert, user);
} }
@@ -166,17 +166,17 @@ public class MetricServiceImpl implements MetricService {
@Override @Override
public MetricResp getMetric(Long modelId, String bizName) { public MetricResp getMetric(Long modelId, String bizName) {
List<MetricResp> metricDescs = getMetricByModelId(modelId); List<MetricResp> metricResps = getMetricByModelId(modelId);
MetricResp metricDesc = null; MetricResp metricResp = null;
if (CollectionUtils.isEmpty(metricDescs)) { if (CollectionUtils.isEmpty(metricResps)) {
return metricDesc; return metricResp;
} }
for (MetricResp metric : metricDescs) { for (MetricResp metric : metricResps) {
if (metric.getBizName().equalsIgnoreCase(bizName)) { if (metric.getBizName().equalsIgnoreCase(bizName)) {
metricDesc = metric; metricResp = metric;
} }
} }
return metricDesc; return metricResp;
} }
@Override @Override
@@ -226,24 +226,24 @@ public class MetricServiceImpl implements MetricService {
@Override @Override
public List<MetricResp> getHighSensitiveMetric(Long modelId) { public List<MetricResp> getHighSensitiveMetric(Long modelId) {
List<MetricResp> metricDescs = getMetricByModelId(modelId); List<MetricResp> metricResps = getMetricByModelId(modelId);
if (CollectionUtils.isEmpty(metricDescs)) { if (CollectionUtils.isEmpty(metricResps)) {
return metricDescs; return metricResps;
} }
return metricDescs.stream() return metricResps.stream()
.filter(metricDesc -> SensitiveLevelEnum.HIGH.getCode().equals(metricDesc.getSensitiveLevel())) .filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
@Override @Override
public List<MetricResp> getAllHighSensitiveMetric() { public List<MetricResp> getAllHighSensitiveMetric() {
List<MetricResp> metricDescs = Lists.newArrayList(); List<MetricResp> metricResps = Lists.newArrayList();
List<MetricDO> metricDOS = metricRepository.getAllMetricList(); List<MetricDO> metricDOS = metricRepository.getAllMetricList();
if (CollectionUtils.isEmpty(metricDOS)) { if (CollectionUtils.isEmpty(metricDOS)) {
return metricDescs; return metricResps;
} }
return convertList(metricDOS.stream() return convertList(metricDOS.stream()
.filter(metricDesc -> SensitiveLevelEnum.HIGH.getCode().equals(metricDesc.getSensitiveLevel())) .filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
@@ -294,23 +294,26 @@ public class MetricServiceImpl implements MetricService {
MetricTypeParams typeParams = metricReq.getTypeParams(); MetricTypeParams typeParams = metricReq.getTypeParams();
List<Measure> measures = typeParams.getMeasures(); List<Measure> measures = typeParams.getMeasures();
if (CollectionUtils.isEmpty(measures)) { if (CollectionUtils.isEmpty(measures)) {
throw new RuntimeException("measure can not be none"); throw new InvalidArgumentException("不可缺少度量");
} }
if (StringUtils.isBlank(typeParams.getExpr())) { if (StringUtils.isBlank(typeParams.getExpr())) {
throw new RuntimeException("expr can not be blank"); throw new InvalidArgumentException("表达式不可为空");
}
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
} }
} }
private void checkExist(List<MetricReq> exprMetricReqList) { private void check(List<MetricReq> exprMetricReqList) {
Long modelId = exprMetricReqList.get(0).getModelId(); Long modelId = exprMetricReqList.get(0).getModelId();
List<MetricResp> metricDescs = getMetrics(modelId); List<MetricResp> metricResps = getMetrics(modelId);
for (MetricReq exprMetricReq : exprMetricReqList) { for (MetricReq exprMetricReq : exprMetricReqList) {
for (MetricResp metricDesc : metricDescs) { for (MetricResp metricResp : metricResps) {
if (metricDesc.getName().equalsIgnoreCase(exprMetricReq.getName())) { if (metricResp.getName().equalsIgnoreCase(exprMetricReq.getName())) {
throw new RuntimeException(String.format("exist same metric name:%s", metricDesc.getName())); throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getName()));
} }
if (metricDesc.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) { if (metricResp.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) {
throw new RuntimeException(String.format("exist same metric en name:%s", metricDesc.getName())); throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getBizName()));
} }
preCheckMetric(exprMetricReq); preCheckMetric(exprMetricReq);
} }
@@ -318,14 +321,14 @@ public class MetricServiceImpl implements MetricService {
} }
private List<MetricResp> convertList(List<MetricDO> metricDOS) { private List<MetricResp> convertList(List<MetricDO> metricDOS) {
List<MetricResp> metricDescs = Lists.newArrayList(); List<MetricResp> metricResps = Lists.newArrayList();
Map<Long, ModelResp> modelMap = modelService.getModelMap(); Map<Long, ModelResp> modelMap = modelService.getModelMap();
if (!CollectionUtils.isEmpty(metricDOS)) { if (!CollectionUtils.isEmpty(metricDOS)) {
metricDescs = metricDOS.stream() metricResps = metricDOS.stream()
.map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap)) .map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
return metricDescs; return metricResps;
} }

View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.semantic.model.domain.utils;
import java.util.regex.Pattern;
public class NameCheckUtils {
public static boolean containsSpecialCharacters(String str) {
String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
return Pattern.compile(regex).matcher(str).find();
}
}