diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java index ad073c30e..fc8646a9f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java @@ -26,7 +26,8 @@ public class EntityInfoParseResponder implements ParseResponder { SemanticService semanticService = ContextUtils.getBean(SemanticService.class); EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryReq.getUser()); - if (QueryManager.isEntityQuery(parseInfo.getQueryMode())) { + if (QueryManager.isEntityQuery(parseInfo.getQueryMode()) + || QueryManager.isMetricQuery(parseInfo.getQueryMode())) { parseInfo.setEntityInfo(entityInfo); } //2. set native value diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SolvedQueryParseResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SolvedQueryParseResponder.java deleted file mode 100644 index 538a3d73d..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SolvedQueryParseResponder.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.tencent.supersonic.chat.responder.parse; - -import com.tencent.supersonic.chat.api.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.response.ParseResp; -import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; -import com.tencent.supersonic.chat.utils.SolvedQueryManager; -import com.tencent.supersonic.common.util.ContextUtils; -import lombok.extern.slf4j.Slf4j; -import java.util.List; - -@Slf4j -public class SolvedQueryParseResponder implements ParseResponder { - - @Override - public void fillResponse(ParseResp parseResp, QueryContext queryContext) { - SolvedQueryManager solvedQueryManager = ContextUtils.getBean(SolvedQueryManager.class); - List solvedQueryRecallResps = - solvedQueryManager.recallSolvedQuery(queryContext.getRequest().getQueryText(), - queryContext.getRequest().getAgentId()); - parseResp.setSimilarSolvedQuery(solvedQueryRecallResps); - } - -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java index c07f64a4a..41dd227f0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java @@ -4,6 +4,7 @@ package com.tencent.supersonic.chat.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; +import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; @@ -85,4 +86,10 @@ public class ChatController { return chatService.queryShowCase(pageQueryInfoCommand, agentId); } + @RequestMapping("/getSolvedQuery") + public List getSolvedQuery(@RequestParam(value = "queryText") String queryText, + @RequestParam(value = "agentId") Integer agentId) { + return chatService.getSolvedQuery(queryText, agentId); + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java index e22b372a1..d310ea260 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; +import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; @@ -22,23 +23,23 @@ public interface ChatService { * @param chatId * @return */ - public Long getContextModel(Integer chatId); + Long getContextModel(Integer chatId); - public ChatContext getOrCreateContext(int chatId); + ChatContext getOrCreateContext(int chatId); - public void updateContext(ChatContext chatCtx); + void updateContext(ChatContext chatCtx); - public void switchContext(ChatContext chatCtx); + void switchContext(ChatContext chatCtx); - public Boolean addChat(User user, String chatName, Integer agentId); + Boolean addChat(User user, String chatName, Integer agentId); - public List getAll(String userName, Integer agentId); + List getAll(String userName, Integer agentId); - public boolean updateChatName(Long chatId, String chatName, String userName); + boolean updateChatName(Long chatId, String chatName, String userName); - public boolean updateFeedback(Integer id, Integer score, String feedback); + boolean updateFeedback(Integer id, Integer score, String feedback); - public boolean updateChatIsTop(Long chatId, int isTop); + boolean updateChatIsTop(Long chatId, int isTop); Boolean deleteChat(Long chatId, String userName); @@ -46,20 +47,22 @@ public interface ChatService { ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoCommend, int agentId); - public void addQuery(QueryResult queryResult, ChatContext chatCtx); + void addQuery(QueryResult queryResult, ChatContext chatCtx); - public void batchAddParse(ChatContext chatCtx, QueryReq queryReq, + void batchAddParse(ChatContext chatCtx, QueryReq queryReq, ParseResp parseResult, List candidateParses, List selectedParses); - public ChatQueryDO getLastQuery(long chatId); + ChatQueryDO getLastQuery(long chatId); - public int updateQuery(ChatQueryDO chatQueryDO); + int updateQuery(ChatQueryDO chatQueryDO); - public Boolean updateQuery(Long questionId, QueryResult queryResult, ChatContext chatCtx); + Boolean updateQuery(Long questionId, QueryResult queryResult, ChatContext chatCtx); - public ChatParseDO getParseInfo(Long questionId, String userName, int parseId); + ChatParseDO getParseInfo(Long questionId, String userName, int parseId); - public Boolean deleteChatQuery(Long questionId); + Boolean deleteChatQuery(Long questionId); + + List getSolvedQuery(String queryText, Integer agentId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java index 0d5f5c051..353ab42c6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; +import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; @@ -25,6 +26,7 @@ import java.util.Objects; import java.util.stream.Collectors; import com.tencent.supersonic.chat.service.ChatService; +import com.tencent.supersonic.chat.utils.SolvedQueryManager; import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Primary; @@ -38,12 +40,14 @@ public class ChatServiceImpl implements ChatService { private ChatContextRepository chatContextRepository; private ChatRepository chatRepository; private ChatQueryRepository chatQueryRepository; + private SolvedQueryManager solvedQueryManager; public ChatServiceImpl(ChatContextRepository chatContextRepository, ChatRepository chatRepository, - ChatQueryRepository chatQueryRepository) { + ChatQueryRepository chatQueryRepository, SolvedQueryManager solvedQueryManager) { this.chatContextRepository = chatContextRepository; this.chatRepository = chatRepository; this.chatQueryRepository = chatQueryRepository; + this.solvedQueryManager = solvedQueryManager; } @Override @@ -192,4 +196,9 @@ public class ChatServiceImpl implements ChatService { return chatQueryRepository.deleteChatQuery(questionId); } + @Override + public List getSolvedQuery(String queryText, Integer agentId) { + return solvedQueryManager.recallSolvedQuery(queryText, agentId); + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index e8bc8bf43..2c28aa7e3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -96,7 +96,7 @@ public class QueryServiceImpl implements QueryService { // in order to support multi-turn conversation, chat context is needed ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId()); List timeCostDOList = new ArrayList<>(); - schemaMappers.stream().forEach(mapper -> { + for (SchemaMapper mapper : schemaMappers) { Long startTime = System.currentTimeMillis(); mapper.map(queryCtx); Long endTime = System.currentTimeMillis(); @@ -104,8 +104,8 @@ public class QueryServiceImpl implements QueryService { timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime)) .interfaceName(className).type(CostType.MAPPER.getType()).build()); log.info("{} result:{}", className, JsonUtil.toString(queryCtx)); - }); - semanticParsers.stream().forEach(parser -> { + } + for (SemanticParser parser : semanticParsers) { Long startTime = System.currentTimeMillis(); parser.parse(queryCtx, chatCtx); Long endTime = System.currentTimeMillis(); @@ -113,7 +113,7 @@ public class QueryServiceImpl implements QueryService { timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime)) .interfaceName(className).type(CostType.PARSER.getType()).build()); log.info("{} result:{}", className, JsonUtil.toString(queryCtx)); - }); + } ParseResp parseResult; if (queryCtx.getCandidateQueries().size() > 0) { log.debug("pick before [{}]", queryCtx.getCandidateQueries().stream().collect( @@ -122,12 +122,8 @@ public class QueryServiceImpl implements QueryService { log.debug("pick after [{}]", selectedQueries.stream().collect( Collectors.toList())); - List selectedParses = selectedQueries.stream() - .map(SemanticQuery::getParseInfo) - .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed()) - .collect(Collectors.toList()); - List candidateParses = queryCtx.getCandidateQueries().stream() - .map(SemanticQuery::getParseInfo).collect(Collectors.toList()); + List selectedParses = convertParseInfo(selectedQueries); + List candidateParses = convertParseInfo(queryCtx.getCandidateQueries()); parseResult = ParseResp.builder() .chatId(queryReq.getChatId()) .queryText(queryReq.getQueryText()) @@ -151,6 +147,13 @@ public class QueryServiceImpl implements QueryService { return parseResult; } + private List convertParseInfo(List semanticQueries) { + return semanticQueries.stream() + .map(SemanticQuery::getParseInfo) + .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed()) + .collect(Collectors.toList()); + } + @Override public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception { ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(), diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index 7802fe784..5938c14b0 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -40,8 +40,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\ com.tencent.supersonic.chat.responder.parse.ParseResponder=\ com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \ - com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder, \ - com.tencent.supersonic.chat.responder.parse.SolvedQueryParseResponder + com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\ com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 4f81060f3..8c590cd4e 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -40,8 +40,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\ com.tencent.supersonic.chat.responder.parse.ParseResponder=\ com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \ - com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder, \ - com.tencent.supersonic.chat.responder.parse.SolvedQueryParseResponder + com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\ com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder \ No newline at end of file diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MeasureResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MeasureResp.java index d7947bc3b..72811b6f8 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MeasureResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MeasureResp.java @@ -25,4 +25,6 @@ public class MeasureResp { private String bizName; + private Long modelId; + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java index 2612a2698..c24dc070f 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java @@ -103,15 +103,13 @@ public class CatalogImpl implements Catalog { } @Override - public String getAgg(Long modelId, String metricBizName) { + public String getAgg(List metricResps, List measureRespList, String metricBizName) { try { - List metricResps = getMetrics(modelId); if (!CollectionUtils.isEmpty(metricResps)) { Optional metric = metricResps.stream() .filter(m -> m.getBizName().equalsIgnoreCase(metricBizName)).findFirst(); if (metric.isPresent() && Objects.nonNull(metric.get().getTypeParams()) && !CollectionUtils.isEmpty( metric.get().getTypeParams().getMeasures())) { - List measureRespList = datasourceService.getMeasureListOfModel(modelId); if (!CollectionUtils.isEmpty(measureRespList)) { String measureName = metric.get().getTypeParams().getMeasures().get(0).getBizName(); Optional measure = measureRespList.stream() diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java index 81f60a21f..cbeb769a4 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java @@ -146,6 +146,28 @@ public class DatasourceServiceImpl implements DatasourceService { return measureResps; } + @Override + public List getMeasureListOfModel(List modelIds) { + if (CollectionUtils.isEmpty(modelIds)) { + return Lists.newArrayList(); + } + List datasourceResps = getDatasourceList().stream().filter(datasourceResp -> + modelIds.contains(datasourceResp.getModelId())).collect(Collectors.toList()); + List measureResps = Lists.newArrayList(); + if (!CollectionUtils.isEmpty(datasourceResps)) { + for (DatasourceResp datasourceDesc : datasourceResps) { + DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail(); + List measures = datasourceDetail.getMeasures(); + if (!CollectionUtils.isEmpty(measures)) { + measureResps.addAll( + measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc)) + .collect(Collectors.toList())); + } + } + } + return measureResps; + } + private void batchCreateDimension(Datasource datasource, User user) throws Exception { List dimensionReqs = DatasourceConverter.convertDimensionList(datasource); @@ -239,13 +261,6 @@ public class DatasourceServiceImpl implements DatasourceService { return DatasourceConverter.convertList(datasourceRepository.getDatasourceList()); } - @Override - public List getDatasourceListByDatabaseId(Long databaseId) { - return getDatasourceList().stream() - .filter(datasourceResp -> datasourceResp.getDatabaseId().equals(databaseId)) - .collect(Collectors.toList()); - } - @Override public List getDatasourceListNoMeasurePrefix(Long modelId) { List datasourceResps = getDatasourceList(modelId); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java index f4a11a8d1..944ccae67 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java @@ -120,19 +120,16 @@ public class DimensionServiceImpl implements DimensionService { Dimension dimension = DimensionConverter.convert(dimensionReq); dimension.updatedBy(user.getName()); log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension)); - List dimensionRespList = getDimensions(dimensionReq.getModelId()).stream().filter( - o -> o.getId().equals(dimensionReq.getId())).collect(Collectors.toList()); updateDimension(dimension); + DimensionResp dimensionResp = getDimension(dimensionReq.getId()); //动态更新字典 String type = DictWordType.DIMENSION.getType(); - if (!CollectionUtils.isEmpty(dimensionRespList)) { - log.info("dimensionRespList size:{}", dimensionRespList.size()); - log.info("name:{}", dimensionRespList.get(0).getName()); + if (dimensionResp != null) { applicationEventPublisher.publishEvent( - new DataUpdateEvent(this, dimensionRespList.get(0).getName(), + new DataUpdateEvent(this, dimensionResp.getName(), dimensionReq.getName(), dimension.getModelId(), - dimensionRespList.get(0).getId(), type)); + dimensionResp.getId(), type)); } } @@ -156,6 +153,12 @@ public class DimensionServiceImpl implements DimensionService { return null; } + @Override + public DimensionResp getDimension(Long id) { + DimensionDO dimensionDO = dimensionRepository.getDimensionById(id); + return DimensionConverter.convert2DimensionResp(dimensionDO, new HashMap<>(), new HashMap<>()); + } + @Override public PageInfo queryDimension(PageDimensionReq pageDimensionReq) { DimensionFilter dimensionFilter = new DimensionFilter(); @@ -199,6 +202,12 @@ public class DimensionServiceImpl implements DimensionService { return convertList(getDimensionDOS(), datasourceService.getDatasourceMap()); } + @Override + public List getDimensionsByModelIds(List modelIds) { + List dimensionDOS = dimensionRepository.getDimensionListOfmodelIds(modelIds); + return convertList(dimensionDOS, datasourceService.getDatasourceMap()); + } + @Override public List getDimensionsByDatasource(Long datasourceId) { List dimensionResps = Lists.newArrayList(); @@ -239,7 +248,7 @@ public class DimensionServiceImpl implements DimensionService { protected List getDimensionDOS(Long modelId) { - return dimensionRepository.getDimensionListOfDomain(modelId); + return dimensionRepository.getDimensionListOfmodel(modelId); } protected List getDimensionDOS() { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java index 69d5101ac..ddaeec94a 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java @@ -121,7 +121,14 @@ public class MetricServiceImpl implements MetricService { } public List getMetrics(List ids) { - return convertList(metricRepository.getMetricListByIds(ids)); + List metricDOS = metricRepository.getMetricListByIds(ids); + return convertList(metricDOS); + } + + @Override + public List getMetricsByModelIds(List modelIds) { + List metricDOS = metricRepository.getMetricList(modelIds); + return convertList(metricDOS); } @Override diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java index 847793046..3a26ec9c8 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java @@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; +import com.tencent.supersonic.semantic.api.model.response.MeasureResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; @@ -226,7 +227,7 @@ public class ModelServiceImpl implements ModelService { @Override public Map getModelFullPathMap() { - return getModelList().stream().filter(m -> m != null).collect(Collectors.toMap(ModelResp::getId, + return getModelList().stream().filter(Objects::nonNull).collect(Collectors.toMap(ModelResp::getId, ModelResp::getFullPath, (k1, k2) -> k1)); } @@ -277,12 +278,29 @@ public class ModelServiceImpl implements ModelService { if (CollectionUtils.isEmpty(modelIds)) { modelIds = generateModelIdsReq(modelSchemaFilterReq); } - modelIds.stream().forEach(modelId -> { - ModelSchemaResp modelSchemaResp = fetchSingleModelSchema(modelId); - if (Objects.nonNull(modelSchemaResp)) { - modelSchemaRespList.add(modelSchemaResp); + Map> metricRespMap = metricService.getMetricsByModelIds(modelIds) + .stream().collect(Collectors.groupingBy(MetricResp::getModelId)); + Map> dimensionRespsMap = dimensionService.getDimensionsByModelIds(modelIds) + .stream().collect(Collectors.groupingBy(DimensionResp::getModelId)); + Map> measureRespsMap = datasourceService.getMeasureListOfModel(modelIds) + .stream().collect(Collectors.groupingBy(MeasureResp::getModelId)); + for (Long modelId : modelIds) { + ModelResp modelResp = getModelMap().get(modelId); + if (modelResp == null) { + continue; } - }); + List measureResps = measureRespsMap.getOrDefault(modelId, Lists.newArrayList()); + List metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList()); + List metricSchemaResps = metricResps.stream().map(metricResp -> + convert(metricResp, metricResps, measureResps)).collect(Collectors.toList()); + List dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList()) + .stream().map(this::convert).collect(Collectors.toList()); + ModelSchemaResp modelSchemaResp = new ModelSchemaResp(); + BeanUtils.copyProperties(modelResp, modelSchemaResp); + modelSchemaResp.setDimensions(dimensionResps); + modelSchemaResp.setMetrics(metricSchemaResps); + modelSchemaRespList.add(modelSchemaResp); + } return modelSchemaRespList; } @@ -298,30 +316,33 @@ public class ModelServiceImpl implements ModelService { private List generateMetricSchema(Long modelId) { List metricSchemaDescList = new ArrayList<>(); - List metricDescList = metricService.getMetrics(modelId); - metricDescList.stream().forEach(metricDesc -> { - MetricSchemaResp metricSchemaDesc = new MetricSchemaResp(); - BeanUtils.copyProperties(metricDesc, metricSchemaDesc); - metricSchemaDesc.setUseCnt(0L); - String agg = catalog.getAgg(modelId, metricSchemaDesc.getBizName()); - metricSchemaDesc.setDefaultAgg(agg); - metricSchemaDescList.add(metricSchemaDesc); - } - ); + List metricResps = metricService.getMetrics(modelId); + List measureResps = datasourceService.getMeasureListOfModel(modelId); + metricResps.stream().forEach(metricResp -> + metricSchemaDescList.add(convert(metricResp, metricResps, measureResps))); return metricSchemaDescList; } private List generateDimSchema(Long modelId) { - List dimSchemaDescList = new ArrayList<>(); List dimDescList = dimensionService.getDimensions(modelId); - dimDescList.stream().forEach(dimDesc -> { - DimSchemaResp dimSchemaDesc = new DimSchemaResp(); - BeanUtils.copyProperties(dimDesc, dimSchemaDesc); - dimSchemaDesc.setUseCnt(0L); - dimSchemaDescList.add(dimSchemaDesc); - } - ); - return dimSchemaDescList; + return dimDescList.stream().map(this::convert).collect(Collectors.toList()); + } + + private DimSchemaResp convert(DimensionResp dimensionResp) { + DimSchemaResp dimSchemaResp = new DimSchemaResp(); + BeanUtils.copyProperties(dimensionResp, dimSchemaResp); + dimSchemaResp.setUseCnt(0L); + return dimSchemaResp; + } + + private MetricSchemaResp convert(MetricResp metricResp, List metricResps, + List measureResps) { + MetricSchemaResp metricSchemaResp = new MetricSchemaResp(); + BeanUtils.copyProperties(metricResp, metricSchemaResp); + metricSchemaResp.setUseCnt(0L); + String agg = catalog.getAgg(metricResps, measureResps, metricSchemaResp.getBizName()); + metricSchemaResp.setDefaultAgg(agg); + return metricSchemaResp; } private List generateModelIdsReq(ModelSchemaFilterReq filter) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java index ab4c60127..811047556 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatabaseResp; import com.tencent.supersonic.semantic.api.model.response.DatasourceResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ItemDateResp; +import com.tencent.supersonic.semantic.api.model.response.MeasureResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl; @@ -37,6 +38,6 @@ public interface Catalog { ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric); - String getAgg(Long modelId, String metricBizName); + String getAgg(List metricResps, List measureRespList, String metricBizName); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java index e9fbd11e3..d321a6d94 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java @@ -24,8 +24,6 @@ public interface DatasourceService { List getDatasourceListNoMeasurePrefix(Long modelId); - List getDatasourceListByDatabaseId(Long databaseId); - List getDatasourceList(); List getDatasourceList(Long modelId); @@ -44,6 +42,8 @@ public interface DatasourceService { List getMeasureListOfModel(Long modelId); + List getMeasureListOfModel(List modelIds); + void getModelYamlTplByModelIds(Set modelIds, Map> dimensionYamlMap, List datasourceYamlTplList, List metricYamlTplList); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java index 3e80331c0..4264fed9b 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java @@ -16,7 +16,11 @@ public interface DimensionService { List getDimensions(); - DimensionResp getDimension(String bizName, Long domainId); + DimensionResp getDimension(Long id); + + DimensionResp getDimension(String bizName, Long modelId); + + List getDimensionsByModelIds(List modelIds); void createDimension(DimensionReq dimensionReq, User user) throws Exception; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java index 969b935e5..cd2dc4c15 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java @@ -23,6 +23,8 @@ public interface MetricService { void createMetricBatch(List metricReqs, User user) throws Exception; + List getMetricsByModelIds(List modelIds); + PageInfo queryMetric(PageMetricReq pageMetricReq, User user); MetricResp getMetric(Long modelId, String bizName); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java index fd6d0c850..9bbae5d5f 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java @@ -15,7 +15,9 @@ public interface DimensionRepository { List getDimensionListOfDatasource(Long datasourceId); - List getDimensionListOfDomain(Long domainId); + List getDimensionListOfmodel(Long domainId); + + List getDimensionListOfmodelIds(List modelIds); List getDimensionList(); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java index 891233dd0..630a72f8c 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java @@ -17,6 +17,8 @@ public interface MetricRepository { List getMetricList(Long domainId); + List getMetricList(List modelIds); + List getMetricList(); List getMetricListByIds(List ids); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java index 53359290a..1a0b77bef 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java @@ -88,13 +88,14 @@ public class DatasourceConverter { return datasourceDesc; } - public static MeasureResp convert(Measure measure, DatasourceResp datasourceDesc) { - MeasureResp measureDesc = new MeasureResp(); - BeanUtils.copyProperties(measure, measureDesc); - measureDesc.setDatasourceId(datasourceDesc.getId()); - measureDesc.setDatasourceName(datasourceDesc.getName()); - measureDesc.setDatasourceBizName(datasourceDesc.getBizName()); - return measureDesc; + public static MeasureResp convert(Measure measure, DatasourceResp datasourceResp) { + MeasureResp measureResp = new MeasureResp(); + BeanUtils.copyProperties(measure, measureResp); + measureResp.setDatasourceId(datasourceResp.getId()); + measureResp.setDatasourceName(datasourceResp.getName()); + measureResp.setDatasourceBizName(datasourceResp.getBizName()); + measureResp.setModelId(datasourceResp.getModelId()); + return measureResp; } public static DimensionReq convert(Dim dim, Datasource datasource) { diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/NameCheckUtils.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/NameCheckUtils.java index 81f1d876c..0af05cf5b 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/NameCheckUtils.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/NameCheckUtils.java @@ -5,6 +5,9 @@ import java.util.regex.Pattern; public class NameCheckUtils { public static boolean containsSpecialCharacters(String str) { + if (str == null) { + return false; + } String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*"; return Pattern.compile(regex).matcher(str).find(); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java index 16ad69922..23c04eaef 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java @@ -48,9 +48,16 @@ public class DimensionRepositoryImpl implements DimensionRepository { } @Override - public List getDimensionListOfDomain(Long domainId) { + public List getDimensionListOfmodel(Long modelId) { DimensionDOExample dimensionDOExample = new DimensionDOExample(); - dimensionDOExample.createCriteria().andModelIdEqualTo(domainId); + dimensionDOExample.createCriteria().andModelIdEqualTo(modelId); + return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); + } + + @Override + public List getDimensionListOfmodelIds(List modelIds) { + DimensionDOExample dimensionDOExample = new DimensionDOExample(); + dimensionDOExample.createCriteria().andModelIdIn(modelIds); return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java index 175a0e4e3..9f23e542d 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java @@ -42,9 +42,16 @@ public class MetricRepositoryImpl implements MetricRepository { } @Override - public List getMetricList(Long domainId) { + public List getMetricList(Long modelId) { MetricDOExample metricDOExample = new MetricDOExample(); - metricDOExample.createCriteria().andModelIdEqualTo(domainId); + metricDOExample.createCriteria().andModelIdEqualTo(modelId); + return metricDOMapper.selectByExampleWithBLOBs(metricDOExample); + } + + @Override + public List getMetricList(List modelIds) { + MetricDOExample metricDOExample = new MetricDOExample(); + metricDOExample.createCriteria().andModelIdIn(modelIds); return metricDOMapper.selectByExampleWithBLOBs(metricDOExample); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepositoryImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepositoryImpl.java index 7b6cd0601..f236c9421 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepositoryImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepositoryImpl.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.semantic.query.persistence.repository; import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; - import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.tencent.supersonic.semantic.api.model.pojo.QueryStat; @@ -16,6 +15,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.util.Strings; import org.springframework.stereotype.Repository; @@ -37,9 +37,10 @@ public class StatRepositoryImpl implements StatRepository { } @Override - public List getStatInfo(ItemUseReq itemUseCommend) { + @SneakyThrows + public List getStatInfo(ItemUseReq itemUseReq) { List result = new ArrayList<>(); - List statInfos = statMapper.getStatInfo(itemUseCommend); + List statInfos = statMapper.getStatInfo(itemUseReq); Map map = new ConcurrentHashMap<>(); statInfos.stream().forEach(stat -> { String dimensions = stat.getDimensions(); @@ -55,9 +56,8 @@ public class StatRepositoryImpl implements StatRepository { result.add(new ItemUseResp(classId, type, nameEn, v)); }); - List itemUseResps = result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed()) + return result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed()) .collect(Collectors.toList()); - return itemUseResps; } @Override diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index f7061a188..00fbaded2 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -1,9 +1,11 @@ package com.tencent.supersonic.semantic.query.service; +import com.google.common.cache.CacheBuilder; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; @@ -31,6 +33,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @@ -42,6 +45,8 @@ import org.springframework.stereotype.Service; @Slf4j public class QueryServiceImpl implements QueryService { + protected final com.google.common.cache.Cache> itemUseCache = + CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build(); private final StatUtils statUtils; private final CacheUtils cacheUtils; @@ -206,9 +211,16 @@ public class QueryServiceImpl implements QueryService { } @Override - public List getStatInfo(ItemUseReq itemUseCommend) { - List statInfos = statUtils.getStatInfo(itemUseCommend); - return statInfos; + @SneakyThrows + public List getStatInfo(ItemUseReq itemUseReq) { + if (itemUseReq.getCacheEnable()) { + return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> { + List data = statUtils.getStatInfo(itemUseReq); + itemUseCache.put(JsonUtil.toString(itemUseReq), data); + return data; + }); + } + return statUtils.getStatInfo(itemUseReq); } @Override