(improvment)(chat) optimize parse performance (#197)

* (improvment)(chat) optimize parse performance
---------

Co-authored-by: jolunoluo

chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java

@@ -26,7 +26,8 @@ public class EntityInfoParseResponder implements ParseResponder {
             SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
             EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryReq.getUser());
 
-            if (QueryManager.isEntityQuery(parseInfo.getQueryMode())) {
+            if (QueryManager.isEntityQuery(parseInfo.getQueryMode())
+                    || QueryManager.isMetricQuery(parseInfo.getQueryMode())) {
                 parseInfo.setEntityInfo(entityInfo);
             }
             //2. set native value

chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SolvedQueryParseResponder.java

@@ -1,23 +0,0 @@
-package com.tencent.supersonic.chat.responder.parse;
-
-import com.tencent.supersonic.chat.api.pojo.QueryContext;
-import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
-import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
-import com.tencent.supersonic.chat.utils.SolvedQueryManager;
-import com.tencent.supersonic.common.util.ContextUtils;
-import lombok.extern.slf4j.Slf4j;
-import java.util.List;
-
-@Slf4j
-public class SolvedQueryParseResponder implements ParseResponder {
-
-    @Override
-    public void fillResponse(ParseResp parseResp, QueryContext queryContext) {
-        SolvedQueryManager solvedQueryManager = ContextUtils.getBean(SolvedQueryManager.class);
-        List<SolvedQueryRecallResp> solvedQueryRecallResps =
-                solvedQueryManager.recallSolvedQuery(queryContext.getRequest().getQueryText(),
-                        queryContext.getRequest().getAgentId());
-        parseResp.setSimilarSolvedQuery(solvedQueryRecallResps);
-    }
-
-}

chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java

@@ -4,6 +4,7 @@ package com.tencent.supersonic.chat.rest;
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
 import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
+import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
 import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
@@ -85,4 +86,10 @@ public class ChatController {
         return chatService.queryShowCase(pageQueryInfoCommand, agentId);
     }
 
+    @RequestMapping("/getSolvedQuery")
+    public List<SolvedQueryRecallResp> getSolvedQuery(@RequestParam(value = "queryText") String queryText,
+                                                      @RequestParam(value = "agentId") Integer agentId) {
+        return chatService.getSolvedQuery(queryText, agentId);
+    }
+
 }

chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
+import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
@@ -22,23 +23,23 @@ public interface ChatService {
      * @param chatId
      * @return
      */
-    public Long getContextModel(Integer chatId);
+    Long getContextModel(Integer chatId);
 
-    public ChatContext getOrCreateContext(int chatId);
+    ChatContext getOrCreateContext(int chatId);
 
-    public void updateContext(ChatContext chatCtx);
+    void updateContext(ChatContext chatCtx);
 
-    public void switchContext(ChatContext chatCtx);
+    void switchContext(ChatContext chatCtx);
 
-    public Boolean addChat(User user, String chatName, Integer agentId);
+    Boolean addChat(User user, String chatName, Integer agentId);
 
-    public List<ChatDO> getAll(String userName, Integer agentId);
+    List<ChatDO> getAll(String userName, Integer agentId);
 
-    public boolean updateChatName(Long chatId, String chatName, String userName);
+    boolean updateChatName(Long chatId, String chatName, String userName);
 
-    public boolean updateFeedback(Integer id, Integer score, String feedback);
+    boolean updateFeedback(Integer id, Integer score, String feedback);
 
-    public boolean updateChatIsTop(Long chatId, int isTop);
+    boolean updateChatIsTop(Long chatId, int isTop);
 
     Boolean deleteChat(Long chatId, String userName);
 
@@ -46,20 +47,22 @@ public interface ChatService {
 
     ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoCommend, int agentId);
 
-    public void addQuery(QueryResult queryResult, ChatContext chatCtx);
+    void addQuery(QueryResult queryResult, ChatContext chatCtx);
 
-    public void batchAddParse(ChatContext chatCtx, QueryReq queryReq,
+    void batchAddParse(ChatContext chatCtx, QueryReq queryReq,
                          ParseResp parseResult,
                          List<SemanticParseInfo> candidateParses,
                          List<SemanticParseInfo> selectedParses);
 
-    public ChatQueryDO getLastQuery(long chatId);
+    ChatQueryDO getLastQuery(long chatId);
 
-    public int updateQuery(ChatQueryDO chatQueryDO);
+    int updateQuery(ChatQueryDO chatQueryDO);
 
-    public Boolean updateQuery(Long questionId, QueryResult queryResult, ChatContext chatCtx);
+    Boolean updateQuery(Long questionId, QueryResult queryResult, ChatContext chatCtx);
 
-    public ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
+    ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
 
-    public Boolean deleteChatQuery(Long questionId);
+    Boolean deleteChatQuery(Long questionId);
+
+    List<SolvedQueryRecallResp> getSolvedQuery(String queryText, Integer agentId);
 }

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
+import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
 import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
@@ -25,6 +26,7 @@ import java.util.Objects;
 import java.util.stream.Collectors;
 
 import com.tencent.supersonic.chat.service.ChatService;
+import com.tencent.supersonic.chat.utils.SolvedQueryManager;
 import com.tencent.supersonic.common.util.JsonUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.annotation.Primary;
@@ -38,12 +40,14 @@ public class ChatServiceImpl implements ChatService {
     private ChatContextRepository chatContextRepository;
     private ChatRepository chatRepository;
     private ChatQueryRepository chatQueryRepository;
+    private SolvedQueryManager solvedQueryManager;
 
     public ChatServiceImpl(ChatContextRepository chatContextRepository, ChatRepository chatRepository,
-                           ChatQueryRepository chatQueryRepository) {
+                           ChatQueryRepository chatQueryRepository, SolvedQueryManager solvedQueryManager) {
         this.chatContextRepository = chatContextRepository;
         this.chatRepository = chatRepository;
         this.chatQueryRepository = chatQueryRepository;
+        this.solvedQueryManager = solvedQueryManager;
     }
 
     @Override
@@ -192,4 +196,9 @@ public class ChatServiceImpl implements ChatService {
         return chatQueryRepository.deleteChatQuery(questionId);
     }
 
+    @Override
+    public List<SolvedQueryRecallResp> getSolvedQuery(String queryText, Integer agentId) {
+        return solvedQueryManager.recallSolvedQuery(queryText, agentId);
+    }
+
 }

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java

@@ -96,7 +96,7 @@ public class QueryServiceImpl implements QueryService {
         // in order to support multi-turn conversation, chat context is needed
         ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId());
         List<StatisticsDO> timeCostDOList = new ArrayList<>();
-        schemaMappers.stream().forEach(mapper -> {
+        for (SchemaMapper mapper : schemaMappers) {
             Long startTime = System.currentTimeMillis();
             mapper.map(queryCtx);
             Long endTime = System.currentTimeMillis();
@@ -104,8 +104,8 @@ public class QueryServiceImpl implements QueryService {
             timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime))
                     .interfaceName(className).type(CostType.MAPPER.getType()).build());
             log.info("{} result:{}", className, JsonUtil.toString(queryCtx));
-        });
+        }
-        semanticParsers.stream().forEach(parser -> {
+        for (SemanticParser parser : semanticParsers) {
             Long startTime = System.currentTimeMillis();
             parser.parse(queryCtx, chatCtx);
             Long endTime = System.currentTimeMillis();
@@ -113,7 +113,7 @@ public class QueryServiceImpl implements QueryService {
             timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime))
                     .interfaceName(className).type(CostType.PARSER.getType()).build());
             log.info("{} result:{}", className, JsonUtil.toString(queryCtx));
-        });
+        }
         ParseResp parseResult;
         if (queryCtx.getCandidateQueries().size() > 0) {
             log.debug("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
@@ -122,12 +122,8 @@ public class QueryServiceImpl implements QueryService {
             log.debug("pick after [{}]", selectedQueries.stream().collect(
                     Collectors.toList()));
 
-            List<SemanticParseInfo> selectedParses = selectedQueries.stream()
+            List<SemanticParseInfo> selectedParses = convertParseInfo(selectedQueries);
-                    .map(SemanticQuery::getParseInfo)
+            List<SemanticParseInfo> candidateParses = convertParseInfo(queryCtx.getCandidateQueries());
-                    .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
-                    .collect(Collectors.toList());
-            List<SemanticParseInfo> candidateParses = queryCtx.getCandidateQueries().stream()
-                    .map(SemanticQuery::getParseInfo).collect(Collectors.toList());
             parseResult = ParseResp.builder()
                     .chatId(queryReq.getChatId())
                     .queryText(queryReq.getQueryText())
@@ -151,6 +147,13 @@ public class QueryServiceImpl implements QueryService {
         return parseResult;
     }
 
+    private List<SemanticParseInfo> convertParseInfo(List<SemanticQuery> semanticQueries) {
+        return semanticQueries.stream()
+                .map(SemanticQuery::getParseInfo)
+                .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
+                .collect(Collectors.toList());
+    }
+
     @Override
     public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
         ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),

launchers/chat/src/main/resources/META-INF/spring.factories

@@ -40,8 +40,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
 
 com.tencent.supersonic.chat.responder.parse.ParseResponder=\
     com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
-    com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder, \
+    com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder
-    com.tencent.supersonic.chat.responder.parse.SolvedQueryParseResponder
 
 com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
     com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder

launchers/standalone/src/main/resources/META-INF/spring.factories

@@ -40,8 +40,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
 
 com.tencent.supersonic.chat.responder.parse.ParseResponder=\
     com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
-    com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder, \
+    com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder
-    com.tencent.supersonic.chat.responder.parse.SolvedQueryParseResponder
 
 com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
     com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MeasureResp.java

@@ -25,4 +25,6 @@ public class MeasureResp {
 
     private String bizName;
 
+    private Long modelId;
+
 }

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java

@@ -103,15 +103,13 @@ public class CatalogImpl implements Catalog {
     }
 
     @Override
-    public String getAgg(Long modelId, String metricBizName) {
+    public String getAgg(List<MetricResp> metricResps, List<MeasureResp> measureRespList, String metricBizName) {
         try {
-            List<MetricResp> metricResps = getMetrics(modelId);
             if (!CollectionUtils.isEmpty(metricResps)) {
                 Optional<MetricResp> metric = metricResps.stream()
                         .filter(m -> m.getBizName().equalsIgnoreCase(metricBizName)).findFirst();
                 if (metric.isPresent() && Objects.nonNull(metric.get().getTypeParams()) && !CollectionUtils.isEmpty(
                         metric.get().getTypeParams().getMeasures())) {
-                    List<MeasureResp> measureRespList = datasourceService.getMeasureListOfModel(modelId);
                     if (!CollectionUtils.isEmpty(measureRespList)) {
                         String measureName = metric.get().getTypeParams().getMeasures().get(0).getBizName();
                         Optional<MeasureResp> measure = measureRespList.stream()

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java

@@ -146,6 +146,28 @@ public class DatasourceServiceImpl implements DatasourceService {
         return measureResps;
     }
 
+    @Override
+    public List<MeasureResp> getMeasureListOfModel(List<Long> modelIds) {
+        if (CollectionUtils.isEmpty(modelIds)) {
+            return Lists.newArrayList();
+        }
+        List<DatasourceResp> datasourceResps = getDatasourceList().stream().filter(datasourceResp ->
+                modelIds.contains(datasourceResp.getModelId())).collect(Collectors.toList());
+        List<MeasureResp> measureResps = Lists.newArrayList();
+        if (!CollectionUtils.isEmpty(datasourceResps)) {
+            for (DatasourceResp datasourceDesc : datasourceResps) {
+                DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail();
+                List<Measure> measures = datasourceDetail.getMeasures();
+                if (!CollectionUtils.isEmpty(measures)) {
+                    measureResps.addAll(
+                            measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc))
+                                    .collect(Collectors.toList()));
+                }
+            }
+        }
+        return measureResps;
+    }
+
 
     private void batchCreateDimension(Datasource datasource, User user) throws Exception {
         List<DimensionReq> dimensionReqs = DatasourceConverter.convertDimensionList(datasource);
@@ -239,13 +261,6 @@ public class DatasourceServiceImpl implements DatasourceService {
         return DatasourceConverter.convertList(datasourceRepository.getDatasourceList());
     }
 
-    @Override
-    public List<DatasourceResp> getDatasourceListByDatabaseId(Long databaseId) {
-        return getDatasourceList().stream()
-                .filter(datasourceResp -> datasourceResp.getDatabaseId().equals(databaseId))
-                .collect(Collectors.toList());
-    }
-
     @Override
     public List<DatasourceResp> getDatasourceListNoMeasurePrefix(Long modelId) {
         List<DatasourceResp> datasourceResps = getDatasourceList(modelId);

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java

@@ -120,19 +120,16 @@ public class DimensionServiceImpl implements DimensionService {
         Dimension dimension = DimensionConverter.convert(dimensionReq);
         dimension.updatedBy(user.getName());
         log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
-        List<DimensionResp> dimensionRespList = getDimensions(dimensionReq.getModelId()).stream().filter(
-                o -> o.getId().equals(dimensionReq.getId())).collect(Collectors.toList());
         updateDimension(dimension);
+        DimensionResp dimensionResp = getDimension(dimensionReq.getId());
         //动态更新字典
         String type = DictWordType.DIMENSION.getType();
-        if (!CollectionUtils.isEmpty(dimensionRespList)) {
+        if (dimensionResp != null) {
-            log.info("dimensionRespList size:{}", dimensionRespList.size());
-            log.info("name:{}", dimensionRespList.get(0).getName());
             applicationEventPublisher.publishEvent(
-                    new DataUpdateEvent(this, dimensionRespList.get(0).getName(),
+                    new DataUpdateEvent(this, dimensionResp.getName(),
                             dimensionReq.getName(),
                             dimension.getModelId(),
-                            dimensionRespList.get(0).getId(), type));
+                            dimensionResp.getId(), type));
         }
     }
 
@@ -156,6 +153,12 @@ public class DimensionServiceImpl implements DimensionService {
         return null;
     }
 
+    @Override
+    public DimensionResp getDimension(Long id) {
+        DimensionDO dimensionDO = dimensionRepository.getDimensionById(id);
+        return DimensionConverter.convert2DimensionResp(dimensionDO, new HashMap<>(), new HashMap<>());
+    }
+
     @Override
     public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq) {
         DimensionFilter dimensionFilter = new DimensionFilter();
@@ -199,6 +202,12 @@ public class DimensionServiceImpl implements DimensionService {
         return convertList(getDimensionDOS(), datasourceService.getDatasourceMap());
     }
 
+    @Override
+    public List<DimensionResp> getDimensionsByModelIds(List<Long> modelIds) {
+        List<DimensionDO> dimensionDOS = dimensionRepository.getDimensionListOfmodelIds(modelIds);
+        return convertList(dimensionDOS, datasourceService.getDatasourceMap());
+    }
+
     @Override
     public List<DimensionResp> getDimensionsByDatasource(Long datasourceId) {
         List<DimensionResp> dimensionResps = Lists.newArrayList();
@@ -239,7 +248,7 @@ public class DimensionServiceImpl implements DimensionService {
 
 
     protected List<DimensionDO> getDimensionDOS(Long modelId) {
-        return dimensionRepository.getDimensionListOfDomain(modelId);
+        return dimensionRepository.getDimensionListOfmodel(modelId);
     }
 
     protected List<DimensionDO> getDimensionDOS() {

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java

@@ -121,7 +121,14 @@ public class MetricServiceImpl implements MetricService {
     }
 
     public List<MetricResp> getMetrics(List<Long> ids) {
-        return convertList(metricRepository.getMetricListByIds(ids));
+        List<MetricDO> metricDOS = metricRepository.getMetricListByIds(ids);
+        return convertList(metricDOS);
+    }
+
+    @Override
+    public List<MetricResp> getMetricsByModelIds(List<Long> modelIds) {
+        List<MetricDO> metricDOS = metricRepository.getMetricList(modelIds);
+        return convertList(metricDOS);
     }
 
     @Override

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java

@@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
 import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.api.model.response.DomainResp;
+import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelResp;
@@ -226,7 +227,7 @@ public class ModelServiceImpl implements ModelService {
 
     @Override
     public Map<Long, String> getModelFullPathMap() {
-        return getModelList().stream().filter(m -> m != null).collect(Collectors.toMap(ModelResp::getId,
+        return getModelList().stream().filter(Objects::nonNull).collect(Collectors.toMap(ModelResp::getId,
                 ModelResp::getFullPath, (k1, k2) -> k1));
     }
 
@@ -277,12 +278,29 @@ public class ModelServiceImpl implements ModelService {
         if (CollectionUtils.isEmpty(modelIds)) {
             modelIds = generateModelIdsReq(modelSchemaFilterReq);
         }
-        modelIds.stream().forEach(modelId -> {
+        Map<Long, List<MetricResp>> metricRespMap = metricService.getMetricsByModelIds(modelIds)
-            ModelSchemaResp modelSchemaResp = fetchSingleModelSchema(modelId);
+                .stream().collect(Collectors.groupingBy(MetricResp::getModelId));
-            if (Objects.nonNull(modelSchemaResp)) {
+        Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensionsByModelIds(modelIds)
-                modelSchemaRespList.add(modelSchemaResp);
+                .stream().collect(Collectors.groupingBy(DimensionResp::getModelId));
+        Map<Long, List<MeasureResp>> measureRespsMap = datasourceService.getMeasureListOfModel(modelIds)
+                .stream().collect(Collectors.groupingBy(MeasureResp::getModelId));
+        for (Long modelId : modelIds) {
+            ModelResp modelResp = getModelMap().get(modelId);
+            if (modelResp == null) {
+                continue;
             }
-        });
+            List<MeasureResp> measureResps = measureRespsMap.getOrDefault(modelId, Lists.newArrayList());
+            List<MetricResp> metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList());
+            List<MetricSchemaResp> metricSchemaResps = metricResps.stream().map(metricResp ->
+                    convert(metricResp, metricResps, measureResps)).collect(Collectors.toList());
+            List<DimSchemaResp> dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList())
+                    .stream().map(this::convert).collect(Collectors.toList());
+            ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
+            BeanUtils.copyProperties(modelResp, modelSchemaResp);
+            modelSchemaResp.setDimensions(dimensionResps);
+            modelSchemaResp.setMetrics(metricSchemaResps);
+            modelSchemaRespList.add(modelSchemaResp);
+        }
         return modelSchemaRespList;
     }
 
@@ -298,30 +316,33 @@ public class ModelServiceImpl implements ModelService {
 
     private List<MetricSchemaResp> generateMetricSchema(Long modelId) {
         List<MetricSchemaResp> metricSchemaDescList = new ArrayList<>();
-        List<MetricResp> metricDescList = metricService.getMetrics(modelId);
+        List<MetricResp> metricResps = metricService.getMetrics(modelId);
-        metricDescList.stream().forEach(metricDesc -> {
+        List<MeasureResp> measureResps = datasourceService.getMeasureListOfModel(modelId);
-                    MetricSchemaResp metricSchemaDesc = new MetricSchemaResp();
+        metricResps.stream().forEach(metricResp ->
-                    BeanUtils.copyProperties(metricDesc, metricSchemaDesc);
+                metricSchemaDescList.add(convert(metricResp, metricResps, measureResps)));
-                    metricSchemaDesc.setUseCnt(0L);
-                    String agg = catalog.getAgg(modelId, metricSchemaDesc.getBizName());
-                    metricSchemaDesc.setDefaultAgg(agg);
-                    metricSchemaDescList.add(metricSchemaDesc);
-                }
-        );
         return metricSchemaDescList;
     }
 
     private List<DimSchemaResp> generateDimSchema(Long modelId) {
-        List<DimSchemaResp> dimSchemaDescList = new ArrayList<>();
         List<DimensionResp> dimDescList = dimensionService.getDimensions(modelId);
-        dimDescList.stream().forEach(dimDesc -> {
+        return dimDescList.stream().map(this::convert).collect(Collectors.toList());
-                    DimSchemaResp dimSchemaDesc = new DimSchemaResp();
+    }
-                    BeanUtils.copyProperties(dimDesc, dimSchemaDesc);
+
-                    dimSchemaDesc.setUseCnt(0L);
+    private DimSchemaResp convert(DimensionResp dimensionResp) {
-                    dimSchemaDescList.add(dimSchemaDesc);
+        DimSchemaResp dimSchemaResp = new DimSchemaResp();
-                }
+        BeanUtils.copyProperties(dimensionResp, dimSchemaResp);
-        );
+        dimSchemaResp.setUseCnt(0L);
-        return dimSchemaDescList;
+        return dimSchemaResp;
+    }
+
+    private MetricSchemaResp convert(MetricResp metricResp, List<MetricResp> metricResps,
+                                     List<MeasureResp> measureResps) {
+        MetricSchemaResp metricSchemaResp = new MetricSchemaResp();
+        BeanUtils.copyProperties(metricResp, metricSchemaResp);
+        metricSchemaResp.setUseCnt(0L);
+        String agg = catalog.getAgg(metricResps, measureResps, metricSchemaResp.getBizName());
+        metricSchemaResp.setDefaultAgg(agg);
+        return metricSchemaResp;
     }
 
     private List<Long> generateModelIdsReq(ModelSchemaFilterReq filter) {

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java

@@ -5,6 +5,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
 import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
+import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
 import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
 import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
@@ -37,6 +38,6 @@ public interface Catalog {
 
     ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
 
-    String getAgg(Long modelId, String metricBizName);
+    String getAgg(List<MetricResp> metricResps, List<MeasureResp> measureRespList, String metricBizName);
 
 }

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java

@@ -24,8 +24,6 @@ public interface DatasourceService {
 
     List<DatasourceResp> getDatasourceListNoMeasurePrefix(Long modelId);
 
-    List<DatasourceResp> getDatasourceListByDatabaseId(Long databaseId);
-
     List<DatasourceResp> getDatasourceList();
 
     List<DatasourceResp> getDatasourceList(Long modelId);
@@ -44,6 +42,8 @@ public interface DatasourceService {
 
     List<MeasureResp> getMeasureListOfModel(Long modelId);
 
+    List<MeasureResp> getMeasureListOfModel(List<Long> modelIds);
+
     void getModelYamlTplByModelIds(Set<Long> modelIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
             List<DatasourceYamlTpl> datasourceYamlTplList, List<MetricYamlTpl> metricYamlTplList);
 

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java

@@ -16,7 +16,11 @@ public interface DimensionService {
 
     List<DimensionResp> getDimensions();
 
-    DimensionResp getDimension(String bizName, Long domainId);
+    DimensionResp getDimension(Long id);
+
+    DimensionResp getDimension(String bizName, Long modelId);
+
+    List<DimensionResp> getDimensionsByModelIds(List<Long> modelIds);
 
     void createDimension(DimensionReq dimensionReq, User user) throws Exception;
 

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java

@@ -23,6 +23,8 @@ public interface MetricService {
 
     void createMetricBatch(List<MetricReq> metricReqs, User user) throws Exception;
 
+    List<MetricResp> getMetricsByModelIds(List<Long> modelIds);
+
     PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
 
     MetricResp getMetric(Long modelId, String bizName);

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java

@@ -15,7 +15,9 @@ public interface DimensionRepository {
 
     List<DimensionDO> getDimensionListOfDatasource(Long datasourceId);
 
-    List<DimensionDO> getDimensionListOfDomain(Long domainId);
+    List<DimensionDO> getDimensionListOfmodel(Long domainId);
+
+    List<DimensionDO> getDimensionListOfmodelIds(List<Long> modelIds);
 
     List<DimensionDO> getDimensionList();
 

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java

@@ -17,6 +17,8 @@ public interface MetricRepository {
 
     List<MetricDO> getMetricList(Long domainId);
 
+    List<MetricDO> getMetricList(List<Long> modelIds);
+
     List<MetricDO> getMetricList();
 
     List<MetricDO> getMetricListByIds(List<Long> ids);

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java

@@ -88,13 +88,14 @@ public class DatasourceConverter {
         return datasourceDesc;
     }
 
-    public static MeasureResp convert(Measure measure, DatasourceResp datasourceDesc) {
+    public static MeasureResp convert(Measure measure, DatasourceResp datasourceResp) {
-        MeasureResp measureDesc = new MeasureResp();
+        MeasureResp measureResp = new MeasureResp();
-        BeanUtils.copyProperties(measure, measureDesc);
+        BeanUtils.copyProperties(measure, measureResp);
-        measureDesc.setDatasourceId(datasourceDesc.getId());
+        measureResp.setDatasourceId(datasourceResp.getId());
-        measureDesc.setDatasourceName(datasourceDesc.getName());
+        measureResp.setDatasourceName(datasourceResp.getName());
-        measureDesc.setDatasourceBizName(datasourceDesc.getBizName());
+        measureResp.setDatasourceBizName(datasourceResp.getBizName());
-        return measureDesc;
+        measureResp.setModelId(datasourceResp.getModelId());
+        return measureResp;
     }
 
     public static DimensionReq convert(Dim dim, Datasource datasource) {

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/NameCheckUtils.java

@@ -5,6 +5,9 @@ import java.util.regex.Pattern;
 public class NameCheckUtils {
 
     public static boolean containsSpecialCharacters(String str) {
+        if (str == null) {
+            return false;
+        }
         String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
         return Pattern.compile(regex).matcher(str).find();
     }

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java

@@ -48,9 +48,16 @@ public class DimensionRepositoryImpl implements DimensionRepository {
     }
 
     @Override
-    public List<DimensionDO> getDimensionListOfDomain(Long domainId) {
+    public List<DimensionDO> getDimensionListOfmodel(Long modelId) {
         DimensionDOExample dimensionDOExample = new DimensionDOExample();
-        dimensionDOExample.createCriteria().andModelIdEqualTo(domainId);
+        dimensionDOExample.createCriteria().andModelIdEqualTo(modelId);
+        return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
+    }
+
+    @Override
+    public List<DimensionDO> getDimensionListOfmodelIds(List<Long> modelIds) {
+        DimensionDOExample dimensionDOExample = new DimensionDOExample();
+        dimensionDOExample.createCriteria().andModelIdIn(modelIds);
         return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
     }
 

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java

@@ -42,9 +42,16 @@ public class MetricRepositoryImpl implements MetricRepository {
     }
 
     @Override
-    public List<MetricDO> getMetricList(Long domainId) {
+    public List<MetricDO> getMetricList(Long modelId) {
         MetricDOExample metricDOExample = new MetricDOExample();
-        metricDOExample.createCriteria().andModelIdEqualTo(domainId);
+        metricDOExample.createCriteria().andModelIdEqualTo(modelId);
+        return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
+    }
+
+    @Override
+    public List<MetricDO> getMetricList(List<Long> modelIds) {
+        MetricDOExample metricDOExample = new MetricDOExample();
+        metricDOExample.createCriteria().andModelIdIn(modelIds);
         return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
     }
 

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepositoryImpl.java

@@ -1,7 +1,6 @@
 package com.tencent.supersonic.semantic.query.persistence.repository;
 
 import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
-
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
@@ -16,6 +15,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.stereotype.Repository;
@@ -37,9 +37,10 @@ public class StatRepositoryImpl implements StatRepository {
     }
 
     @Override
-    public List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend) {
+    @SneakyThrows
+    public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
         List<ItemUseResp> result = new ArrayList<>();
-        List<QueryStat> statInfos = statMapper.getStatInfo(itemUseCommend);
+        List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
         Map<String, Long> map = new ConcurrentHashMap<>();
         statInfos.stream().forEach(stat -> {
             String dimensions = stat.getDimensions();
@@ -55,9 +56,8 @@ public class StatRepositoryImpl implements StatRepository {
             result.add(new ItemUseResp(classId, type, nameEn, v));
         });
 
-        List<ItemUseResp> itemUseResps = result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
+        return result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
                 .collect(Collectors.toList());
-        return itemUseResps;
     }
 
     @Override

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java

@@ -1,9 +1,11 @@
 package com.tencent.supersonic.semantic.query.service;
 
+import com.google.common.cache.CacheBuilder;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.common.util.cache.CacheUtils;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
@@ -31,6 +33,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.TimeUnit;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
@@ -42,6 +45,8 @@ import org.springframework.stereotype.Service;
 @Slf4j
 public class QueryServiceImpl implements QueryService {
 
+    protected final com.google.common.cache.Cache<String, List<ItemUseResp>> itemUseCache =
+            CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
 
     private final StatUtils statUtils;
     private final CacheUtils cacheUtils;
@@ -206,9 +211,16 @@ public class QueryServiceImpl implements QueryService {
     }
 
     @Override
-    public List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend) {
+    @SneakyThrows
-        List<ItemUseResp> statInfos = statUtils.getStatInfo(itemUseCommend);
+    public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
-        return statInfos;
+        if (itemUseReq.getCacheEnable()) {
+            return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> {
+                List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
+                itemUseCache.put(JsonUtil.toString(itemUseReq), data);
+                return data;
+            });
+        }
+        return statUtils.getStatInfo(itemUseReq);
     }
 
     @Override