mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 19:51:00 +00:00
add metric and dimension name check (#149)
* (improvement)(semantic) add metric and dimension name check * (improvement)(chat) opt QueryResponder recalling history similar solved query --------- Co-authored-by: jolunoluo
This commit is contained in:
@@ -0,0 +1,25 @@
|
|||||||
|
package com.tencent.supersonic.chat.api.pojo.request;
|
||||||
|
|
||||||
|
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Builder
|
||||||
|
public class SolvedQueryReq {
|
||||||
|
|
||||||
|
private Long queryId;
|
||||||
|
|
||||||
|
private Integer parseId;
|
||||||
|
|
||||||
|
private String queryText;
|
||||||
|
|
||||||
|
private Long modelId;
|
||||||
|
|
||||||
|
private Integer agentId;
|
||||||
|
|
||||||
|
}
|
||||||
@@ -1,7 +1,9 @@
|
|||||||
package com.tencent.supersonic.chat.queryresponder;
|
package com.tencent.supersonic.chat.queryresponder;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
|
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
|
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
|
||||||
@@ -42,12 +44,17 @@ public class DefaultQueryResponder implements QueryResponder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void saveSolvedQuery(String queryText, Long queryId, Integer parseId) {
|
public void saveSolvedQuery(SolvedQueryReq solvedQueryReq) {
|
||||||
|
String queryText = solvedQueryReq.getQueryText();
|
||||||
try {
|
try {
|
||||||
String uniqueId = generateUniqueId(queryId, parseId);
|
String uniqueId = generateUniqueId(solvedQueryReq.getQueryId(), solvedQueryReq.getParseId());
|
||||||
Map<String, String> requestMap = new HashMap<>();
|
Map<String, Object> requestMap = new HashMap<>();
|
||||||
requestMap.put("query", queryText);
|
requestMap.put("query", queryText);
|
||||||
requestMap.put("query_id", uniqueId);
|
requestMap.put("query_id", uniqueId);
|
||||||
|
Map<String, Object> metaData = new HashMap<>();
|
||||||
|
metaData.put("modelId", String.valueOf(solvedQueryReq.getModelId()));
|
||||||
|
metaData.put("agentId", String.valueOf(solvedQueryReq.getAgentId()));
|
||||||
|
requestMap.put("metadata", metaData);
|
||||||
doRequest(embeddingConfig.getSolvedQueryAddPath(),
|
doRequest(embeddingConfig.getSolvedQueryAddPath(),
|
||||||
JSONObject.toJSONString(Lists.newArrayList(requestMap)));
|
JSONObject.toJSONString(Lists.newArrayList(requestMap)));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@@ -56,7 +63,7 @@ public class DefaultQueryResponder implements QueryResponder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<SolvedQueryRecallResp> recallSolvedQuery(String queryText) {
|
public List<SolvedQueryRecallResp> recallSolvedQuery(String queryText, Integer agentId) {
|
||||||
List<SolvedQueryRecallResp> solvedQueryRecallResps = Lists.newArrayList();
|
List<SolvedQueryRecallResp> solvedQueryRecallResps = Lists.newArrayList();
|
||||||
try {
|
try {
|
||||||
String url = embeddingConfig.getUrl() + embeddingConfig.getSolvedQueryRecallPath() + "?n_results="
|
String url = embeddingConfig.getUrl() + embeddingConfig.getSolvedQueryRecallPath() + "?n_results="
|
||||||
@@ -66,7 +73,12 @@ public class DefaultQueryResponder implements QueryResponder {
|
|||||||
headers.setLocation(URI.create(url));
|
headers.setLocation(URI.create(url));
|
||||||
URI requestUrl = UriComponentsBuilder
|
URI requestUrl = UriComponentsBuilder
|
||||||
.fromHttpUrl(url).build().encode().toUri();
|
.fromHttpUrl(url).build().encode().toUri();
|
||||||
String jsonBody = JSONObject.toJSONString(Lists.newArrayList(queryText));
|
Map<String, Object> map = new HashMap<>();
|
||||||
|
map.put("queryTextsList", Lists.newArrayList(queryText));
|
||||||
|
Map<String, Object> filterCondition = new HashMap<>();
|
||||||
|
filterCondition.put("agentId", String.valueOf(agentId));
|
||||||
|
map.put("filterCondition", filterCondition);
|
||||||
|
String jsonBody = JSONObject.toJSONString(map, SerializerFeature.WriteMapNullValue);
|
||||||
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
|
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
|
||||||
log.info("[embedding] request body:{}, url:{}", jsonBody, url);
|
log.info("[embedding] request body:{}, url:{}", jsonBody, url);
|
||||||
ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity =
|
ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity =
|
||||||
@@ -80,12 +92,14 @@ public class DefaultQueryResponder implements QueryResponder {
|
|||||||
for (EmbeddingResp embeddingResp : embeddingResps) {
|
for (EmbeddingResp embeddingResp : embeddingResps) {
|
||||||
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
|
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
|
||||||
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
|
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
|
||||||
|
if (queryText.equalsIgnoreCase(embeddingRetrieval.getQuery())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (querySet.contains(embeddingRetrieval.getQuery())) {
|
if (querySet.contains(embeddingRetrieval.getQuery())) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
String id = embeddingRetrieval.getId();
|
String id = embeddingRetrieval.getId();
|
||||||
SolvedQueryRecallResp solvedQueryRecallResp =
|
SolvedQueryRecallResp solvedQueryRecallResp = SolvedQueryRecallResp.builder()
|
||||||
SolvedQueryRecallResp.builder()
|
|
||||||
.queryText(embeddingRetrieval.getQuery())
|
.queryText(embeddingRetrieval.getQuery())
|
||||||
.queryId(getQueryId(id)).parseId(getParseId(id))
|
.queryId(getQueryId(id)).parseId(getParseId(id))
|
||||||
.build();
|
.build();
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
package com.tencent.supersonic.chat.queryresponder;
|
package com.tencent.supersonic.chat.queryresponder;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public interface QueryResponder {
|
public interface QueryResponder {
|
||||||
|
|
||||||
void saveSolvedQuery(String queryText, Long queryId, Integer parseId);
|
void saveSolvedQuery(SolvedQueryReq solvedQueryReq);
|
||||||
|
|
||||||
List<SolvedQueryRecallResp> recallSolvedQuery(String queryText);
|
List<SolvedQueryRecallResp> recallSolvedQuery(String queryText, Integer agentId);
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -14,6 +14,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
|||||||
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
|
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||||
@@ -21,6 +22,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryState;
|
|||||||
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
|
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
|
||||||
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
|
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
|
||||||
|
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
|
||||||
import com.tencent.supersonic.chat.persistence.dataobject.CostType;
|
import com.tencent.supersonic.chat.persistence.dataobject.CostType;
|
||||||
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
|
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
|
||||||
import com.tencent.supersonic.chat.query.QuerySelector;
|
import com.tencent.supersonic.chat.query.QuerySelector;
|
||||||
@@ -150,7 +152,7 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
List<SolvedQueryRecallResp> solvedQueryRecallResps =
|
List<SolvedQueryRecallResp> solvedQueryRecallResps =
|
||||||
queryResponder.recallSolvedQuery(queryCtx.getRequest().getQueryText());
|
queryResponder.recallSolvedQuery(queryCtx.getRequest().getQueryText(), queryReq.getAgentId());
|
||||||
parseResult.setSimilarSolvedQuery(solvedQueryRecallResps);
|
parseResult.setSimilarSolvedQuery(solvedQueryRecallResps);
|
||||||
return parseResult;
|
return parseResult;
|
||||||
}
|
}
|
||||||
@@ -172,6 +174,7 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
|
public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
|
||||||
ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),
|
ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),
|
||||||
queryReq.getUser().getName(), queryReq.getParseId());
|
queryReq.getUser().getName(), queryReq.getParseId());
|
||||||
|
ChatQueryDO chatQueryDO = chatService.getLastQuery(queryReq.getChatId());
|
||||||
List<StatisticsDO> timeCostDOList = new ArrayList<>();
|
List<StatisticsDO> timeCostDOList = new ArrayList<>();
|
||||||
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
|
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
|
||||||
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
|
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
|
||||||
@@ -196,6 +199,11 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) {
|
if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) {
|
||||||
chatCtx.setParseInfo(parseInfo);
|
chatCtx.setParseInfo(parseInfo);
|
||||||
chatService.updateContext(chatCtx);
|
chatService.updateContext(chatCtx);
|
||||||
|
queryResponder.saveSolvedQuery(SolvedQueryReq.builder().parseId(queryReq.getParseId())
|
||||||
|
.queryId(queryReq.getQueryId())
|
||||||
|
.agentId(chatQueryDO.getAgentId())
|
||||||
|
.modelId(parseInfo.getModelId())
|
||||||
|
.queryText(queryReq.getQueryText()).build());
|
||||||
}
|
}
|
||||||
chatCtx.setQueryText(queryReq.getQueryText());
|
chatCtx.setQueryText(queryReq.getQueryText());
|
||||||
chatCtx.setUser(queryReq.getUser().getName());
|
chatCtx.setUser(queryReq.getUser().getName());
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.application;
|
|||||||
import com.alibaba.fastjson.JSONObject;
|
import com.alibaba.fastjson.JSONObject;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
|
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
|
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
|
||||||
@@ -48,11 +49,14 @@ import java.util.Map;
|
|||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.context.annotation.Lazy;
|
import org.springframework.context.annotation.Lazy;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
|
||||||
@@ -84,25 +88,27 @@ public class DatasourceServiceImpl implements DatasourceService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@Transactional
|
||||||
public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception {
|
public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception {
|
||||||
preCheck(datasourceReq);
|
preCheck(datasourceReq);
|
||||||
|
checkExist(datasourceReq);
|
||||||
Datasource datasource = DatasourceConverter.convert(datasourceReq);
|
Datasource datasource = DatasourceConverter.convert(datasourceReq);
|
||||||
log.info("[create datasource] object:{}", JSONObject.toJSONString(datasource));
|
log.info("[create datasource] object:{}", JSONObject.toJSONString(datasource));
|
||||||
saveDatasource(datasource, user);
|
saveDatasource(datasource, user);
|
||||||
Optional<DatasourceResp> datasourceDescOptional = getDatasource(datasourceReq.getModelId(),
|
Optional<DatasourceResp> datasourceDescOptional = getDatasource(datasourceReq.getModelId(),
|
||||||
datasourceReq.getBizName());
|
datasourceReq.getBizName());
|
||||||
if (!datasourceDescOptional.isPresent()) {
|
if (!datasourceDescOptional.isPresent()) {
|
||||||
throw new RuntimeException("create datasource failed");
|
throw new RuntimeException("创建数据源失败");
|
||||||
}
|
}
|
||||||
DatasourceResp datasourceDesc = datasourceDescOptional.get();
|
DatasourceResp datasourceResp = datasourceDescOptional.get();
|
||||||
datasource.setId(datasourceDesc.getId());
|
datasource.setId(datasourceResp.getId());
|
||||||
batchCreateDimension(datasource, user);
|
batchCreateDimension(datasource, user);
|
||||||
batchCreateMetric(datasource, user);
|
batchCreateMetric(datasource, user);
|
||||||
return datasourceDesc;
|
return datasourceResp;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@Transactional
|
||||||
public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception {
|
public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception {
|
||||||
preCheck(datasourceReq);
|
preCheck(datasourceReq);
|
||||||
Datasource datasource = DatasourceConverter.convert(datasourceReq);
|
Datasource datasource = DatasourceConverter.convert(datasourceReq);
|
||||||
@@ -124,20 +130,20 @@ public class DatasourceServiceImpl implements DatasourceService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<MeasureResp> getMeasureListOfModel(Long modelId) {
|
public List<MeasureResp> getMeasureListOfModel(Long modelId) {
|
||||||
List<DatasourceResp> datasourceDescs = getDatasourceList(modelId);
|
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
|
||||||
List<MeasureResp> measureDescs = Lists.newArrayList();
|
List<MeasureResp> measureResps = Lists.newArrayList();
|
||||||
if (!CollectionUtils.isEmpty(datasourceDescs)) {
|
if (!CollectionUtils.isEmpty(datasourceResps)) {
|
||||||
for (DatasourceResp datasourceDesc : datasourceDescs) {
|
for (DatasourceResp datasourceDesc : datasourceResps) {
|
||||||
DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail();
|
DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail();
|
||||||
List<Measure> measures = datasourceDetail.getMeasures();
|
List<Measure> measures = datasourceDetail.getMeasures();
|
||||||
if (!CollectionUtils.isEmpty(measures)) {
|
if (!CollectionUtils.isEmpty(measures)) {
|
||||||
measureDescs.addAll(
|
measureResps.addAll(
|
||||||
measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc))
|
measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc))
|
||||||
.collect(Collectors.toList()));
|
.collect(Collectors.toList()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return measureDescs;
|
return measureResps;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -153,11 +159,11 @@ public class DatasourceServiceImpl implements DatasourceService {
|
|||||||
|
|
||||||
|
|
||||||
private Optional<DatasourceResp> getDatasource(Long modelId, String bizName) {
|
private Optional<DatasourceResp> getDatasource(Long modelId, String bizName) {
|
||||||
List<DatasourceResp> datasourceDescs = getDatasourceList(modelId);
|
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
|
||||||
if (CollectionUtils.isEmpty(datasourceDescs)) {
|
if (CollectionUtils.isEmpty(datasourceResps)) {
|
||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
}
|
}
|
||||||
for (DatasourceResp datasourceDesc : datasourceDescs) {
|
for (DatasourceResp datasourceDesc : datasourceResps) {
|
||||||
if (datasourceDesc.getBizName().equals(bizName)) {
|
if (datasourceDesc.getBizName().equals(bizName)) {
|
||||||
return Optional.of(datasourceDesc);
|
return Optional.of(datasourceDesc);
|
||||||
}
|
}
|
||||||
@@ -175,18 +181,51 @@ public class DatasourceServiceImpl implements DatasourceService {
|
|||||||
|
|
||||||
|
|
||||||
private void preCheck(DatasourceReq datasourceReq) {
|
private void preCheck(DatasourceReq datasourceReq) {
|
||||||
|
if (NameCheckUtils.containsSpecialCharacters(datasourceReq.getName())) {
|
||||||
|
String message = String.format("数据源名称[%s]包含特殊字符, 请修改", datasourceReq.getName());
|
||||||
|
throw new InvalidArgumentException(message);
|
||||||
|
}
|
||||||
List<Dim> dims = datasourceReq.getDimensions();
|
List<Dim> dims = datasourceReq.getDimensions();
|
||||||
List<Measure> measures = datasourceReq.getMeasures();
|
List<Measure> measures = datasourceReq.getMeasures();
|
||||||
List<Dim> timeDims = datasourceReq.getTimeDimension();
|
List<Dim> timeDims = datasourceReq.getTimeDimension();
|
||||||
List<Identify> identifies = datasourceReq.getIdentifiers();
|
List<Identify> identifies = datasourceReq.getIdentifiers();
|
||||||
if (CollectionUtils.isEmpty(dims)) {
|
if (CollectionUtils.isEmpty(dims)) {
|
||||||
throw new RuntimeException("缺少维度信息");
|
throw new InvalidArgumentException("缺少维度信息");
|
||||||
}
|
}
|
||||||
if (CollectionUtils.isEmpty(identifies)) {
|
if (CollectionUtils.isEmpty(identifies)) {
|
||||||
throw new RuntimeException("缺少主键信息");
|
throw new InvalidArgumentException("缺少主键信息");
|
||||||
}
|
}
|
||||||
if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) {
|
if (!CollectionUtils.isEmpty(measures) && CollectionUtils.isEmpty(timeDims)) {
|
||||||
throw new RuntimeException("有度量时, 不可缺少时间维度");
|
throw new InvalidArgumentException("有度量时, 不可缺少时间维度");
|
||||||
|
}
|
||||||
|
for (Measure measure : measures) {
|
||||||
|
if (StringUtils.isNotBlank(measure.getName())
|
||||||
|
&& NameCheckUtils.containsSpecialCharacters(measure.getName())) {
|
||||||
|
String message = String.format("度量[%s]包含特殊字符, 请修改", measure.getName());
|
||||||
|
throw new InvalidArgumentException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (Dim dim : dims) {
|
||||||
|
if (StringUtils.isNotBlank(dim.getName())
|
||||||
|
&& NameCheckUtils.containsSpecialCharacters(dim.getName())) {
|
||||||
|
String message = String.format("维度[%s]包含特殊字符, 请修改", dim.getName());
|
||||||
|
throw new InvalidArgumentException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (Identify identify : identifies) {
|
||||||
|
if (StringUtils.isNotBlank(identify.getName())
|
||||||
|
&& NameCheckUtils.containsSpecialCharacters(identify.getName())) {
|
||||||
|
String message = String.format("主键/外键[%s]包含特殊字符, 请修改", identify.getName());
|
||||||
|
throw new InvalidArgumentException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkExist(DatasourceReq datasourceReq) {
|
||||||
|
Optional<DatasourceResp> datasourceRespOptional = getDatasource(datasourceReq.getModelId(),
|
||||||
|
datasourceReq.getBizName());
|
||||||
|
if (datasourceRespOptional.isPresent()) {
|
||||||
|
throw new InvalidArgumentException("已存在相同名字的数据源:" + datasourceReq.getBizName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,11 +268,11 @@ public class DatasourceServiceImpl implements DatasourceService {
|
|||||||
@Override
|
@Override
|
||||||
public Map<Long, DatasourceResp> getDatasourceMap() {
|
public Map<Long, DatasourceResp> getDatasourceMap() {
|
||||||
Map<Long, DatasourceResp> map = new HashMap<>();
|
Map<Long, DatasourceResp> map = new HashMap<>();
|
||||||
List<DatasourceResp> datasourceDescs = getDatasourceList();
|
List<DatasourceResp> datasourceResps = getDatasourceList();
|
||||||
if (CollectionUtils.isEmpty(datasourceDescs)) {
|
if (CollectionUtils.isEmpty(datasourceResps)) {
|
||||||
return map;
|
return map;
|
||||||
}
|
}
|
||||||
return datasourceDescs.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1));
|
return datasourceResps.stream().collect(Collectors.toMap(DatasourceResp::getId, a -> a, (k1, k2) -> k1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.DataAddEvent;
|
|||||||
import com.tencent.supersonic.common.pojo.DataDeleteEvent;
|
import com.tencent.supersonic.common.pojo.DataDeleteEvent;
|
||||||
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
|
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
|
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||||
import com.tencent.supersonic.common.util.ChatGptHelper;
|
import com.tencent.supersonic.common.util.ChatGptHelper;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
|
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
|
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
|
||||||
@@ -37,6 +38,7 @@ import java.util.ArrayList;
|
|||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
@@ -84,7 +86,6 @@ public class DimensionServiceImpl implements DimensionService {
|
|||||||
log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension));
|
log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension));
|
||||||
dimension.createdBy(user.getName());
|
dimension.createdBy(user.getName());
|
||||||
saveDimension(dimension);
|
saveDimension(dimension);
|
||||||
//动态更新字典
|
|
||||||
String type = DictWordType.DIMENSION.getType();
|
String type = DictWordType.DIMENSION.getType();
|
||||||
DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId());
|
DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId());
|
||||||
applicationEventPublisher.publishEvent(
|
applicationEventPublisher.publishEvent(
|
||||||
@@ -113,6 +114,9 @@ public class DimensionServiceImpl implements DimensionService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void updateDimension(DimensionReq dimensionReq, User user) {
|
public void updateDimension(DimensionReq dimensionReq, User user) {
|
||||||
|
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
|
||||||
|
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
|
||||||
|
}
|
||||||
Dimension dimension = DimensionConverter.convert(dimensionReq);
|
Dimension dimension = DimensionConverter.convert(dimensionReq);
|
||||||
dimension.updatedBy(user.getName());
|
dimension.updatedBy(user.getName());
|
||||||
log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
|
log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
|
||||||
@@ -356,13 +360,16 @@ public class DimensionServiceImpl implements DimensionService {
|
|||||||
Long modelId = dimensionReqs.get(0).getModelId();
|
Long modelId = dimensionReqs.get(0).getModelId();
|
||||||
List<DimensionResp> dimensionResps = getDimensions(modelId);
|
List<DimensionResp> dimensionResps = getDimensions(modelId);
|
||||||
for (DimensionReq dimensionReq : dimensionReqs) {
|
for (DimensionReq dimensionReq : dimensionReqs) {
|
||||||
|
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
|
||||||
|
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
|
||||||
|
}
|
||||||
for (DimensionResp dimensionResp : dimensionResps) {
|
for (DimensionResp dimensionResp : dimensionResps) {
|
||||||
if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getBizName())) {
|
if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getName())) {
|
||||||
throw new RuntimeException(String.format("exist same dimension name:%s", dimensionReq.getName()));
|
throw new RuntimeException(String.format("存在相同的维度名 :%s", dimensionReq.getName()));
|
||||||
}
|
}
|
||||||
if (dimensionResp.getBizName().equalsIgnoreCase(dimensionReq.getBizName())) {
|
if (dimensionResp.getBizName().equalsIgnoreCase(dimensionReq.getBizName())) {
|
||||||
throw new RuntimeException(
|
throw new RuntimeException(
|
||||||
String.format("exist same dimension bizName:%s", dimensionReq.getBizName()));
|
String.format("存在相同的维度名: %s", dimensionReq.getBizName()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import com.tencent.supersonic.common.pojo.DataDeleteEvent;
|
|||||||
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
|
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
|
||||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
|
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||||
import com.tencent.supersonic.common.util.ChatGptHelper;
|
import com.tencent.supersonic.common.util.ChatGptHelper;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
|
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
|
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
|
||||||
@@ -28,14 +29,13 @@ import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository;
|
|||||||
import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter;
|
import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter;
|
||||||
import com.tencent.supersonic.semantic.model.domain.MetricService;
|
import com.tencent.supersonic.semantic.model.domain.MetricService;
|
||||||
import com.tencent.supersonic.semantic.model.domain.pojo.Metric;
|
import com.tencent.supersonic.semantic.model.domain.pojo.Metric;
|
||||||
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
@@ -71,7 +71,7 @@ public class MetricServiceImpl implements MetricService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void creatExprMetric(MetricReq metricReq, User user) {
|
public void creatExprMetric(MetricReq metricReq, User user) {
|
||||||
checkExist(Lists.newArrayList(metricReq));
|
check(Lists.newArrayList(metricReq));
|
||||||
Metric metric = MetricConverter.convert(metricReq);
|
Metric metric = MetricConverter.convert(metricReq);
|
||||||
metric.createdBy(user.getName());
|
metric.createdBy(user.getName());
|
||||||
log.info("[create metric] object:{}", JSONObject.toJSONString(metric));
|
log.info("[create metric] object:{}", JSONObject.toJSONString(metric));
|
||||||
@@ -90,11 +90,11 @@ public class MetricServiceImpl implements MetricService {
|
|||||||
}
|
}
|
||||||
List<Metric> metrics = metricReqs.stream().map(MetricConverter::convert).collect(Collectors.toList());
|
List<Metric> metrics = metricReqs.stream().map(MetricConverter::convert).collect(Collectors.toList());
|
||||||
Long modelId = metricReqs.get(0).getModelId();
|
Long modelId = metricReqs.get(0).getModelId();
|
||||||
List<MetricResp> metricDescs = getMetricByModelId(modelId);
|
List<MetricResp> metricResps = getMetricByModelId(modelId);
|
||||||
Map<String, MetricResp> metricDescMap = metricDescs.stream()
|
Map<String, MetricResp> metricRespMap = metricResps.stream()
|
||||||
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||||
List<Metric> metricToInsert = metrics.stream()
|
List<Metric> metricToInsert = metrics.stream()
|
||||||
.filter(metric -> !metricDescMap.containsKey(metric.getBizName())).collect(Collectors.toList());
|
.filter(metric -> !metricRespMap.containsKey(metric.getBizName())).collect(Collectors.toList());
|
||||||
log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert));
|
log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert));
|
||||||
saveMetricBatch(metricToInsert, user);
|
saveMetricBatch(metricToInsert, user);
|
||||||
}
|
}
|
||||||
@@ -166,17 +166,17 @@ public class MetricServiceImpl implements MetricService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public MetricResp getMetric(Long modelId, String bizName) {
|
public MetricResp getMetric(Long modelId, String bizName) {
|
||||||
List<MetricResp> metricDescs = getMetricByModelId(modelId);
|
List<MetricResp> metricResps = getMetricByModelId(modelId);
|
||||||
MetricResp metricDesc = null;
|
MetricResp metricResp = null;
|
||||||
if (CollectionUtils.isEmpty(metricDescs)) {
|
if (CollectionUtils.isEmpty(metricResps)) {
|
||||||
return metricDesc;
|
return metricResp;
|
||||||
}
|
}
|
||||||
for (MetricResp metric : metricDescs) {
|
for (MetricResp metric : metricResps) {
|
||||||
if (metric.getBizName().equalsIgnoreCase(bizName)) {
|
if (metric.getBizName().equalsIgnoreCase(bizName)) {
|
||||||
metricDesc = metric;
|
metricResp = metric;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return metricDesc;
|
return metricResp;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -226,24 +226,24 @@ public class MetricServiceImpl implements MetricService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<MetricResp> getHighSensitiveMetric(Long modelId) {
|
public List<MetricResp> getHighSensitiveMetric(Long modelId) {
|
||||||
List<MetricResp> metricDescs = getMetricByModelId(modelId);
|
List<MetricResp> metricResps = getMetricByModelId(modelId);
|
||||||
if (CollectionUtils.isEmpty(metricDescs)) {
|
if (CollectionUtils.isEmpty(metricResps)) {
|
||||||
return metricDescs;
|
return metricResps;
|
||||||
}
|
}
|
||||||
return metricDescs.stream()
|
return metricResps.stream()
|
||||||
.filter(metricDesc -> SensitiveLevelEnum.HIGH.getCode().equals(metricDesc.getSensitiveLevel()))
|
.filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<MetricResp> getAllHighSensitiveMetric() {
|
public List<MetricResp> getAllHighSensitiveMetric() {
|
||||||
List<MetricResp> metricDescs = Lists.newArrayList();
|
List<MetricResp> metricResps = Lists.newArrayList();
|
||||||
List<MetricDO> metricDOS = metricRepository.getAllMetricList();
|
List<MetricDO> metricDOS = metricRepository.getAllMetricList();
|
||||||
if (CollectionUtils.isEmpty(metricDOS)) {
|
if (CollectionUtils.isEmpty(metricDOS)) {
|
||||||
return metricDescs;
|
return metricResps;
|
||||||
}
|
}
|
||||||
return convertList(metricDOS.stream()
|
return convertList(metricDOS.stream()
|
||||||
.filter(metricDesc -> SensitiveLevelEnum.HIGH.getCode().equals(metricDesc.getSensitiveLevel()))
|
.filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
|
||||||
.collect(Collectors.toList()));
|
.collect(Collectors.toList()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -294,23 +294,26 @@ public class MetricServiceImpl implements MetricService {
|
|||||||
MetricTypeParams typeParams = metricReq.getTypeParams();
|
MetricTypeParams typeParams = metricReq.getTypeParams();
|
||||||
List<Measure> measures = typeParams.getMeasures();
|
List<Measure> measures = typeParams.getMeasures();
|
||||||
if (CollectionUtils.isEmpty(measures)) {
|
if (CollectionUtils.isEmpty(measures)) {
|
||||||
throw new RuntimeException("measure can not be none");
|
throw new InvalidArgumentException("不可缺少度量");
|
||||||
}
|
}
|
||||||
if (StringUtils.isBlank(typeParams.getExpr())) {
|
if (StringUtils.isBlank(typeParams.getExpr())) {
|
||||||
throw new RuntimeException("expr can not be blank");
|
throw new InvalidArgumentException("表达式不可为空");
|
||||||
|
}
|
||||||
|
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
|
||||||
|
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkExist(List<MetricReq> exprMetricReqList) {
|
private void check(List<MetricReq> exprMetricReqList) {
|
||||||
Long modelId = exprMetricReqList.get(0).getModelId();
|
Long modelId = exprMetricReqList.get(0).getModelId();
|
||||||
List<MetricResp> metricDescs = getMetrics(modelId);
|
List<MetricResp> metricResps = getMetrics(modelId);
|
||||||
for (MetricReq exprMetricReq : exprMetricReqList) {
|
for (MetricReq exprMetricReq : exprMetricReqList) {
|
||||||
for (MetricResp metricDesc : metricDescs) {
|
for (MetricResp metricResp : metricResps) {
|
||||||
if (metricDesc.getName().equalsIgnoreCase(exprMetricReq.getName())) {
|
if (metricResp.getName().equalsIgnoreCase(exprMetricReq.getName())) {
|
||||||
throw new RuntimeException(String.format("exist same metric name:%s", metricDesc.getName()));
|
throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getName()));
|
||||||
}
|
}
|
||||||
if (metricDesc.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) {
|
if (metricResp.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) {
|
||||||
throw new RuntimeException(String.format("exist same metric en name:%s", metricDesc.getName()));
|
throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getBizName()));
|
||||||
}
|
}
|
||||||
preCheckMetric(exprMetricReq);
|
preCheckMetric(exprMetricReq);
|
||||||
}
|
}
|
||||||
@@ -318,14 +321,14 @@ public class MetricServiceImpl implements MetricService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private List<MetricResp> convertList(List<MetricDO> metricDOS) {
|
private List<MetricResp> convertList(List<MetricDO> metricDOS) {
|
||||||
List<MetricResp> metricDescs = Lists.newArrayList();
|
List<MetricResp> metricResps = Lists.newArrayList();
|
||||||
Map<Long, ModelResp> modelMap = modelService.getModelMap();
|
Map<Long, ModelResp> modelMap = modelService.getModelMap();
|
||||||
if (!CollectionUtils.isEmpty(metricDOS)) {
|
if (!CollectionUtils.isEmpty(metricDOS)) {
|
||||||
metricDescs = metricDOS.stream()
|
metricResps = metricDOS.stream()
|
||||||
.map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap))
|
.map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
return metricDescs;
|
return metricResps;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
package com.tencent.supersonic.semantic.model.domain.utils;
|
||||||
|
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
public class NameCheckUtils {
|
||||||
|
|
||||||
|
public static boolean containsSpecialCharacters(String str) {
|
||||||
|
String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
|
||||||
|
return Pattern.compile(regex).matcher(str).find();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user