From 6466999dbc542ce273c3b371735311f05d94c1fc Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Sat, 22 Jun 2024 11:15:59 +0800 Subject: [PATCH] (improvement)(headless)Remove redundant SemanticService and SearchService. --- .../execute/DimensionRecommendProcessor.java | 4 +- .../processor/parse/EntityInfoProcessor.java | 4 +- .../server/service/impl/ChatServiceImpl.java | 4 +- .../service/impl/ConfigServiceImpl.java | 14 +- .../server/processor/ParseInfoProcessor.java | 6 +- .../rest/api/ChatQueryApiController.java | 5 +- .../rest/api/MetaDiscoveryApiController.java | 6 +- .../server/service/MetaDiscoveryService.java | 10 - ...earchService.java => RetrieveService.java} | 10 +- .../server/service/SemanticLayerService.java | 11 +- .../service/impl/ChatQueryServiceImpl.java | 22 +- .../server/service/impl/DictWordService.java | 6 +- .../impl/MetaDiscoveryServiceImpl.java | 192 --------------- .../service/impl/MetricServiceImpl.java | 8 +- ...viceImpl.java => RetrieveServiceImpl.java} | 224 ++++++++++++++--- ...eImpl.java => S2SemanticLayerService.java} | 202 ++++++++++++++- .../server/service/impl/SemanticService.java | 229 ------------------ .../server/service/MetricServiceImplTest.java | 2 +- .../supersonic/evaluation/Text2SQLEval.java | 2 +- .../headless/MetaDiscoveryTest.java | 10 +- 20 files changed, 453 insertions(+), 518 deletions(-) delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaDiscoveryService.java rename headless/server/src/main/java/com/tencent/supersonic/headless/server/service/{SearchService.java => RetrieveService.java} (55%) delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java rename headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/{SearchServiceImpl.java => RetrieveServiceImpl.java} (58%) rename headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/{SemanticLayerServiceImpl.java => S2SemanticLayerService.java} (56%) delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index d4e4e80e2..2ad316fc8 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; -import com.tencent.supersonic.headless.server.service.impl.SemanticService; +import com.tencent.supersonic.headless.server.service.SemanticLayerService; import org.springframework.util.CollectionUtils; import java.util.Comparator; @@ -40,7 +40,7 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { } private List getDimensions(Long metricId, Long dataSetId) { - SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class); DataSetSchema dataSetSchema = semanticService.getDataSetSchema(dataSetId); List drillDownDimensions = Lists.newArrayList(); Set metricElements = dataSetSchema.getMetrics(); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java index a7ca095d4..0178397fc 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.EntityInfo; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import com.tencent.supersonic.headless.chat.query.QueryManager; -import com.tencent.supersonic.headless.server.service.impl.SemanticService; +import com.tencent.supersonic.headless.server.service.SemanticLayerService; import org.springframework.util.CollectionUtils; import java.util.List; @@ -30,7 +30,7 @@ public class EntityInfoProcessor implements ParseResultProcessor { return; } //1. set entity info - SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class); DataSetSchema dataSetSchema = semanticService.getDataSetSchema(parseInfo.getDataSetId()); EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, chatParseContext.getUser()); if (QueryManager.isTagQuery(queryMode) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java index 095723da4..a5e37677d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java @@ -30,7 +30,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import com.tencent.supersonic.headless.api.pojo.response.QueryResult; import com.tencent.supersonic.headless.api.pojo.response.SearchResult; import com.tencent.supersonic.headless.server.service.ChatQueryService; -import com.tencent.supersonic.headless.server.service.SearchService; +import com.tencent.supersonic.headless.server.service.RetrieveService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -47,7 +47,7 @@ public class ChatServiceImpl implements ChatService { @Autowired private ChatQueryService chatQueryService; @Autowired - private SearchService searchService; + private RetrieveService searchService; @Autowired private SimilarQueryManager similarQueryManager; private List chatParsers = ComponentFactory.getChatParsers(); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index 63a004d10..34e6f4e01 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -33,10 +33,9 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; -import com.tencent.supersonic.headless.server.service.impl.SemanticService; +import com.tencent.supersonic.headless.server.service.SemanticLayerService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -56,18 +55,17 @@ public class ConfigServiceImpl implements ConfigService { private final ChatConfigHelper chatConfigHelper; private final DimensionService dimensionService; private final MetricService metricService; - @Autowired - private SemanticService semanticService; + private final SemanticLayerService semanticLayerService; public ConfigServiceImpl(ChatConfigRepository chatConfigRepository, - ChatConfigHelper chatConfigHelper, - DimensionService dimensionService, - MetricService metricService) { + ChatConfigHelper chatConfigHelper, DimensionService dimensionService, + MetricService metricService, SemanticLayerService semanticLayerService) { this.chatConfigRepository = chatConfigRepository; this.chatConfigHelper = chatConfigHelper; this.dimensionService = dimensionService; this.metricService = metricService; + this.semanticLayerService = semanticLayerService; } @Override @@ -208,7 +206,7 @@ public class ConfigServiceImpl implements ConfigService { } BeanUtils.copyProperties(chatConfigResp, chatConfigRich); - DataSetSchema dataSetSchema = semanticService.getDataSetSchema(modelId); + DataSetSchema dataSetSchema = semanticLayerService.getDataSetSchema(modelId); if (dataSetSchema == null) { return chatConfigRich; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java index c92e37b61..ff2ee9cab 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java @@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import com.tencent.supersonic.headless.chat.ChatContext; import com.tencent.supersonic.headless.chat.QueryContext; import com.tencent.supersonic.headless.chat.query.SemanticQuery; -import com.tencent.supersonic.headless.server.service.impl.SemanticService; +import com.tencent.supersonic.headless.server.service.SemanticLayerService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -83,7 +83,7 @@ public class ParseInfoProcessor implements ResultProcessor { log.error("set dimensionFilter error :", e); } - SemanticSchema semanticSchema = ContextUtils.getBean(SemanticService.class).getSemanticSchema(); + SemanticSchema semanticSchema = ContextUtils.getBean(SemanticLayerService.class).getSemanticSchema(); if (Objects.isNull(semanticSchema)) { return; } @@ -195,7 +195,7 @@ public class ParseInfoProcessor implements ResultProcessor { } protected Map getNameToElement(Long dataSetId) { - SemanticSchema semanticSchema = ContextUtils.getBean(SemanticService.class).getSemanticSchema(); + SemanticSchema semanticSchema = ContextUtils.getBean(SemanticLayerService.class).getSemanticSchema(); List dimensions = semanticSchema.getDimensions(dataSetId); List metrics = semanticSchema.getMetrics(dataSetId); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ChatQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ChatQueryApiController.java index 33dd8b303..fdc3e6255 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ChatQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ChatQueryApiController.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.headless.api.pojo.request.QueryReq; import com.tencent.supersonic.headless.api.pojo.response.MapResp; import com.tencent.supersonic.headless.server.service.ChatQueryService; -import com.tencent.supersonic.headless.server.service.SearchService; +import com.tencent.supersonic.headless.server.service.RetrieveService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -22,8 +22,9 @@ public class ChatQueryApiController { @Autowired private ChatQueryService chatQueryService; + @Autowired - private SearchService searchService; + private RetrieveService searchService; @PostMapping("/chat/search") public Object search(@RequestBody QueryReq queryReq, diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetaDiscoveryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetaDiscoveryApiController.java index 01e944486..2fd1c7f80 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetaDiscoveryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetaDiscoveryApiController.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.rest.api; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; -import com.tencent.supersonic.headless.server.service.MetaDiscoveryService; +import com.tencent.supersonic.headless.server.service.RetrieveService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -20,14 +20,14 @@ import javax.servlet.http.HttpServletResponse; public class MetaDiscoveryApiController { @Autowired - private MetaDiscoveryService metaDiscovery; + private RetrieveService metaDiscovery; @PostMapping("map") public Object map(@RequestBody QueryMapReq queryMapReq, HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); queryMapReq.setUser(user); - return metaDiscovery.getMapMeta(queryMapReq); + return metaDiscovery.map(queryMapReq); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaDiscoveryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaDiscoveryService.java deleted file mode 100644 index 596c55b7f..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaDiscoveryService.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.tencent.supersonic.headless.server.service; - -import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; -import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp; - -public interface MetaDiscoveryService { - - MapInfoResp getMapMeta(QueryMapReq queryMapReq); - -} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SearchService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/RetrieveService.java similarity index 55% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SearchService.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/RetrieveService.java index df8d618aa..ea78350ed 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SearchService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/RetrieveService.java @@ -1,16 +1,16 @@ package com.tencent.supersonic.headless.server.service; +import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; import com.tencent.supersonic.headless.api.pojo.request.QueryReq; +import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp; import com.tencent.supersonic.headless.api.pojo.response.SearchResult; import java.util.List; +public interface RetrieveService { -/** - * search service - */ -public interface SearchService { + MapInfoResp map(QueryMapReq queryMapReq); List search(QueryReq queryCtx); -} +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemanticLayerService.java index d5b9c3d86..64e896fef 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemanticLayerService.java @@ -2,6 +2,8 @@ package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.EntityInfo; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; @@ -13,19 +15,24 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import java.util.List; +/** + * This interface abstracts functionalities provided by a semantic layer. + */ public interface SemanticLayerService { SemanticSchema getSemanticSchema(); DataSetSchema getDataSetSchema(Long id); - SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception; + List getDataSetSchema(); - //List queryByReqs(List queryReqs, User user) throws Exception; + SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception; SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); List getStatInfo(ItemUseReq itemUseCommend); ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception; + + EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java index a92843cda..a7b2485f2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java @@ -93,14 +93,12 @@ import java.util.stream.Collectors; public class ChatQueryServiceImpl implements ChatQueryService { @Autowired - private SemanticService semanticService; + private SemanticLayerService semanticLayerService; @Autowired private ChatContextService chatContextService; @Autowired private KnowledgeBaseService knowledgeBaseService; @Autowired - private SemanticLayerService queryService; - @Autowired private DataSetService dataSetService; @Autowired private WorkflowService workflowService; @@ -137,7 +135,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { public QueryContext buildQueryContext(QueryReq queryReq) { - SemanticSchema semanticSchema = semanticService.getSemanticSchema(); + SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema(); Map> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(); QueryContext queryCtx = QueryContext.builder() .queryFilters(queryReq.getQueryFilters()) @@ -184,7 +182,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { private QueryResult doExecution(SemanticQueryReq semanticQueryReq, SemanticParseInfo parseInfo, User user) throws Exception { - SemanticQueryResp queryResp = queryService.queryByReq(semanticQueryReq, user); + SemanticQueryResp queryResp = semanticLayerService.queryByReq(semanticQueryReq, user); QueryResult queryResult = new QueryResult(); if (queryResp != null) { queryResult.setQueryAuthorization(queryResp.getQueryAuthorization()); @@ -214,7 +212,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { @Override public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws Exception { SemanticParseInfo parseInfo = getSemanticParseInfo(queryData); - SemanticSchema semanticSchema = semanticService.getSemanticSchema(); + SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema(); SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); semanticQuery.setParseInfo(parseInfo); @@ -239,7 +237,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); ExplainSqlReq explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq) .queryTypeEnum(QueryMethod.SQL).build(); - ExplainResp explain = queryService.explain(explainSqlReq, user); + ExplainResp explain = semanticLayerService.explain(explainSqlReq, user); if (StringUtils.isNotBlank(explain.getSql())) { parseInfo.getSqlInfo().setQuerySQL(explain.getSql()); } @@ -258,7 +256,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { QueryResult queryResult = doExecution(semanticQueryReq, semanticQuery.getParseInfo(), user); queryResult.setChatContext(semanticQuery.getParseInfo()); DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(parseInfo.getDataSetId()); - SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class); EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user); queryResult.setEntityInfo(entityInfo); return queryResult; @@ -324,7 +322,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { @Override public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) { - SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class); DataSetSchema dataSetSchema = semanticService.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); return semanticService.getEntityInfo(parseInfo, dataSetSchema, user); @@ -520,7 +518,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { @Override public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { SemanticQueryResp semanticQueryResp = new SemanticQueryResp(); - SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class); SemanticSchema semanticSchema = semanticService.getSemanticSchema(); SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID()); Set detectDataSetIds = new HashSet<>(); @@ -589,7 +587,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { List groups = new ArrayList<>(); groups.add(dimensionValueReq.getBizName()); queryStructReq.setGroups(groups); - return queryService.queryByReq(queryStructReq, user); + return semanticLayerService.queryByReq(queryStructReq, user); } public void correct(QuerySqlReq querySqlReq, User user) { @@ -605,7 +603,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { private SemanticParseInfo correctSqlReq(QuerySqlReq querySqlReq, User user) { QueryContext queryCtx = new QueryContext(); - SemanticSchema semanticSchema = semanticService.getSemanticSchema(); + SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema(); queryCtx.setSemanticSchema(semanticSchema); SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); SqlInfo sqlInfo = new SqlInfo(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java index fdc5429ac..a6e569c00 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.chat.knowledge.DictWord; import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService; import com.tencent.supersonic.headless.chat.knowledge.builder.WordBuilderFactory; -import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.service.SemanticLayerService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -24,7 +24,7 @@ import java.util.stream.Collectors; public class DictWordService { @Autowired - private SchemaService schemaService; + private SemanticLayerService semanticLayerService; @Autowired private KnowledgeBaseService knowledgeBaseService; @@ -49,7 +49,7 @@ public class DictWordService { } public List getAllDictWords() { - SemanticSchema semanticSchema = new SemanticSchema(schemaService.getDataSetSchema()); + SemanticSchema semanticSchema = new SemanticSchema(semanticLayerService.getDataSetSchema()); List words = new ArrayList<>(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java deleted file mode 100644 index 7069a0641..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java +++ /dev/null @@ -1,192 +0,0 @@ -package com.tencent.supersonic.headless.server.service.impl; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.headless.api.pojo.SemanticSchema; -import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryReq; -import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp; -import com.tencent.supersonic.headless.api.pojo.response.MapResp; -import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder; -import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.service.ChatQueryService; -import com.tencent.supersonic.headless.server.service.DataSetService; -import com.tencent.supersonic.headless.server.service.MetaDiscoveryService; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Service -public class MetaDiscoveryServiceImpl implements MetaDiscoveryService { - - @Autowired - private DataSetService dataSetService; - - @Autowired - private ChatQueryService chatQueryService; - - @Autowired - private SemanticService semanticService; - - @Override - public MapInfoResp getMapMeta(QueryMapReq queryMapReq) { - - QueryReq queryReq = new QueryReq(); - BeanUtils.copyProperties(queryMapReq, queryReq); - List dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser()); - - Set dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet()); - queryReq.setDataSetIds(dataSetIds); - MapResp mapResp = chatQueryService.performMapping(queryReq); - dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet()); - return convert(mapResp, queryMapReq.getTopN(), dataSetIds); - } - - private MapInfoResp convert(MapResp mapResp, Integer topN, Set dataSetIds) { - MapInfoResp mapInfoResp = new MapInfoResp(); - if (Objects.isNull(mapResp)) { - return mapInfoResp; - } - BeanUtils.copyProperties(mapResp, mapInfoResp); - MetaFilter metaFilter = new MetaFilter(); - metaFilter.setIds(new ArrayList<>(dataSetIds)); - List dataSetList = dataSetService.getDataSetList(metaFilter); - Map dataSetMap = dataSetList.stream() - .collect(Collectors.toMap(DataSetResp::getId, d -> d)); - mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN)); - mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap)); - return mapInfoResp; - } - - private Map getDataSetInfo(SchemaMapInfo mapInfo, - Map dataSetMap, - Integer topN) { - Map map = new HashMap<>(); - Map> mapFields = getMapFields(mapInfo, dataSetMap); - Map> topFields = getTopFields(topN, mapInfo, dataSetMap); - for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) { - DataSetResp dataSetResp = dataSetMap.get(dataSetId); - if (dataSetResp == null) { - continue; - } - if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) { - continue; - } - DataSetMapInfo dataSetMapInfo = new DataSetMapInfo(); - dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList())); - dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList())); - dataSetMapInfo.setName(dataSetResp.getName()); - dataSetMapInfo.setDescription(dataSetResp.getDescription()); - map.put(dataSetMapInfo.getName(), dataSetMapInfo); - } - return map; - } - - private Map> getMapFields(SchemaMapInfo mapInfo, - Map dataSetMap) { - Map> result = new HashMap<>(); - for (Map.Entry> entry : mapInfo.getDataSetElementMatches().entrySet()) { - List values = entry.getValue().stream() - .filter(schemaElementMatch -> - !SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) { - result.put(entry.getKey(), values); - } - } - return result; - } - - private Map> getTopFields(Integer topN, - SchemaMapInfo mapInfo, - Map dataSetMap) { - Map> result = new HashMap<>(); - if (0 == topN) { - return result; - } - SemanticSchema semanticSchema = semanticService.getSemanticSchema(); - for (Map.Entry> entry : mapInfo.getDataSetElementMatches().entrySet()) { - Long dataSetId = entry.getKey(); - List values = entry.getValue(); - DataSetResp dataSetResp = dataSetMap.get(dataSetId); - if (dataSetResp == null || CollectionUtils.isEmpty(values)) { - continue; - } - String dataSetName = dataSetResp.getName(); - //topN dimensions - Set dimensions = semanticSchema.getDimensions(dataSetId) - .stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) - .limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet()); - - SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName); - dimensions.add(timeDimensionMatch); - - //topN metrics - Set metrics = semanticSchema.getMetrics(dataSetId) - .stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) - .limit(topN).map(mergeFunction()).collect(Collectors.toSet()); - - dimensions.addAll(metrics); - result.put(dataSetId, new ArrayList<>(dimensions)); - } - return result; - } - - private Map> getTerms(SchemaMapInfo mapInfo, - Map dataSetNameMap) { - Map> termMap = new HashMap<>(); - Map> dataSetElementMatches = mapInfo.getDataSetElementMatches(); - for (Map.Entry> entry : dataSetElementMatches.entrySet()) { - DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey()); - if (dataSetResp == null) { - continue; - } - List terms = entry.getValue().stream().filter(schemaElementMatch - -> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); - termMap.put(dataSetResp.getName(), terms); - } - return termMap; - } - - /*** - * get time dimension SchemaElementMatch - * @param dataSetId - * @param dataSetName - * @return - */ - private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) { - SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName) - .type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build(); - - SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element) - .detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName()) - .similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build(); - - return timeDimensionMatch; - } - - private Function mergeFunction() { - return schemaElement -> SchemaElementMatch.builder().element(schemaElement) - .frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1) - .detectWord(schemaElement.getName()).build(); - } -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index 613445d11..467bfc670 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -61,7 +61,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.service.CollectService; import com.tencent.supersonic.headless.server.service.DataSetService; import com.tencent.supersonic.headless.server.service.DimensionService; -import com.tencent.supersonic.headless.server.service.MetaDiscoveryService; +import com.tencent.supersonic.headless.server.service.RetrieveService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.TagMetaService; @@ -110,7 +110,7 @@ public class MetricServiceImpl extends ServiceImpl private TagMetaService tagMetaService; - private MetaDiscoveryService metaDiscoveryService; + private RetrieveService metaDiscoveryService; public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, @@ -120,7 +120,7 @@ public class MetricServiceImpl extends ServiceImpl ApplicationEventPublisher eventPublisher, DimensionService dimensionService, TagMetaService tagMetaService, - @Lazy MetaDiscoveryService metaDiscoveryService) { + @Lazy RetrieveService metaDiscoveryService) { this.metricRepository = metricRepository; this.modelService = modelService; this.aliasGenerateHelper = aliasGenerateHelper; @@ -298,7 +298,7 @@ public class MetricServiceImpl extends ServiceImpl queryMapReq.setQueryText(pageMetricReq.getKey()); queryMapReq.setUser(user); queryMapReq.setMapModeEnum(MapModeEnum.LOOSE); - MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq); + MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq); Map dataSetMapInfoMap = mapMeta.getDataSetMapInfo(); if (CollectionUtils.isEmpty(dataSetMapInfoMap)) { return metricRespPageInfo; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java similarity index 58% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java index 5e8e919b8..0fa80f08e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java @@ -3,71 +3,104 @@ package com.tencent.supersonic.headless.server.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; +import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; import com.tencent.supersonic.headless.api.pojo.request.QueryReq; +import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo; +import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; +import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp; +import com.tencent.supersonic.headless.api.pojo.response.MapResp; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.api.pojo.response.SearchResult; -import com.tencent.supersonic.headless.chat.mapper.MatchText; -import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType; -import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy; import com.tencent.supersonic.headless.chat.QueryContext; import com.tencent.supersonic.headless.chat.knowledge.DataSetInfoStat; import com.tencent.supersonic.headless.chat.knowledge.DictWord; import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult; import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService; +import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper; +import com.tencent.supersonic.headless.chat.mapper.MatchText; +import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType; +import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.ChatContextService; +import com.tencent.supersonic.headless.server.service.ChatQueryService; import com.tencent.supersonic.headless.server.service.DataSetService; -import com.tencent.supersonic.headless.server.service.SearchService; +import com.tencent.supersonic.headless.server.service.RetrieveService; +import com.tencent.supersonic.headless.server.service.SemanticLayerService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; + import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; - -/** - * search service impl - */ @Service @Slf4j -public class SearchServiceImpl implements SearchService { +public class RetrieveServiceImpl implements RetrieveService { private static final int RESULT_SIZE = 10; - @Autowired - private SemanticService semanticService; - @Autowired - private SearchMatchStrategy searchMatchStrategy; - @Autowired - private ChatContextService chatContextService; - @Autowired - private KnowledgeBaseService knowledgeBaseService; + @Autowired private DataSetService dataSetService; + @Autowired + private ChatQueryService chatQueryService; + + @Autowired + private ChatContextService chatContextService; + + @Autowired + private SemanticLayerService semanticLayerService; + + @Autowired + private KnowledgeBaseService knowledgeBaseService; + + @Autowired + private SearchMatchStrategy searchMatchStrategy; + + @Override + public MapInfoResp map(QueryMapReq queryMapReq) { + + QueryReq queryReq = new QueryReq(); + BeanUtils.copyProperties(queryMapReq, queryReq); + List dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser()); + + Set dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet()); + queryReq.setDataSetIds(dataSetIds); + MapResp mapResp = chatQueryService.performMapping(queryReq); + dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet()); + return convert(mapResp, queryMapReq.getTopN(), dataSetIds); + } + @Override public List search(QueryReq queryReq) { String queryText = queryReq.getQueryText(); // 1.get meta info - SemanticSchema semanticSchemaDb = semanticService.getSemanticSchema(); + SemanticSchema semanticSchemaDb = semanticLayerService.getSemanticSchema(); List metricsDb = semanticSchemaDb.getMetrics(); final Map dataSetIdToName = semanticSchemaDb.getDataSetIdToName(); Map> modelIdToDataSetIds = @@ -87,7 +120,7 @@ public class SearchServiceImpl implements SearchService { regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue())); // 3.get the most matching data - Optional>> mostSimilarSearchResult = regTextMap.entrySet() + Optional>> mostSimilarSearchResult = regTextMap.entrySet() .stream() .filter(entry -> CollectionUtils.isNotEmpty(entry.getValue())) .reduce((entry1, entry2) -> @@ -127,7 +160,7 @@ public class SearchServiceImpl implements SearchService { } private List getPossibleDataSets(QueryReq queryCtx, List originals, - DataSetInfoStat dataSetInfoStat, Set dataSetIds) { + DataSetInfoStat dataSetInfoStat, Set dataSetIds) { if (CollectionUtils.isNotEmpty(dataSetIds)) { return new ArrayList<>(dataSetIds); } @@ -152,13 +185,13 @@ public class SearchServiceImpl implements SearchService { } private Set searchDimensionValue(List metricsDb, - Map modelToName, - long metricModelCount, - boolean existMetricAndDimension, - MatchText matchText, - Map natureToNameMap, - Map.Entry natureToNameEntry, - QueryFilters queryFilters) { + Map modelToName, + long metricModelCount, + boolean existMetricAndDimension, + MatchText matchText, + Map natureToNameMap, + Map.Entry natureToNameEntry, + QueryFilters queryFilters) { Set searchResults = new LinkedHashSet(); String nature = natureToNameEntry.getKey(); @@ -247,7 +280,7 @@ public class SearchServiceImpl implements SearchService { * @return */ private Map getNatureToNameMap(Map.Entry> recommendTextListEntry, - Set possibleModels) { + Set possibleModels) { List recommendValues = recommendTextListEntry.getValue(); return recommendValues.stream() .flatMap(entry -> entry.getNatures().stream() @@ -269,7 +302,8 @@ public class SearchServiceImpl implements SearchService { } private boolean searchMetricAndDimension(Set possibleDataSets, Map modelToName, - Map.Entry> searchTextEntry, Set searchResults) { + Map.Entry> searchTextEntry, + Set searchResults) { boolean existMetric = false; log.info("searchMetricAndDimension searchTextEntry:{}", searchTextEntry); MatchText matchText = searchTextEntry.getKey(); @@ -317,4 +351,134 @@ public class SearchServiceImpl implements SearchService { } return possibleDataSets.contains(entry.getModel()); } + + private MapInfoResp convert(MapResp mapResp, Integer topN, Set dataSetIds) { + MapInfoResp mapInfoResp = new MapInfoResp(); + if (Objects.isNull(mapResp)) { + return mapInfoResp; + } + BeanUtils.copyProperties(mapResp, mapInfoResp); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setIds(new ArrayList<>(dataSetIds)); + List dataSetList = dataSetService.getDataSetList(metaFilter); + Map dataSetMap = dataSetList.stream() + .collect(Collectors.toMap(DataSetResp::getId, d -> d)); + mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN)); + mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap)); + return mapInfoResp; + } + + private Map getDataSetInfo(SchemaMapInfo mapInfo, + Map dataSetMap, + Integer topN) { + Map map = new HashMap<>(); + Map> mapFields = getMapFields(mapInfo, dataSetMap); + Map> topFields = getTopFields(topN, mapInfo, dataSetMap); + for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) { + DataSetResp dataSetResp = dataSetMap.get(dataSetId); + if (dataSetResp == null) { + continue; + } + if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) { + continue; + } + DataSetMapInfo dataSetMapInfo = new DataSetMapInfo(); + dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList())); + dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList())); + dataSetMapInfo.setName(dataSetResp.getName()); + dataSetMapInfo.setDescription(dataSetResp.getDescription()); + map.put(dataSetMapInfo.getName(), dataSetMapInfo); + } + return map; + } + + private Map> getMapFields(SchemaMapInfo mapInfo, + Map dataSetMap) { + Map> result = new HashMap<>(); + for (Map.Entry> entry : mapInfo.getDataSetElementMatches().entrySet()) { + List values = entry.getValue().stream() + .filter(schemaElementMatch -> + !SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) { + result.put(entry.getKey(), values); + } + } + return result; + } + + private Map> getTopFields(Integer topN, + SchemaMapInfo mapInfo, + Map dataSetMap) { + Map> result = new HashMap<>(); + if (0 == topN) { + return result; + } + SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema(); + for (Map.Entry> entry : mapInfo.getDataSetElementMatches().entrySet()) { + Long dataSetId = entry.getKey(); + List values = entry.getValue(); + DataSetResp dataSetResp = dataSetMap.get(dataSetId); + if (dataSetResp == null || CollectionUtils.isEmpty(values)) { + continue; + } + String dataSetName = dataSetResp.getName(); + //topN dimensions + Set dimensions = semanticSchema.getDimensions(dataSetId) + .stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) + .limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet()); + + SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName); + dimensions.add(timeDimensionMatch); + + //topN metrics + Set metrics = semanticSchema.getMetrics(dataSetId) + .stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) + .limit(topN).map(mergeFunction()).collect(Collectors.toSet()); + + dimensions.addAll(metrics); + result.put(dataSetId, new ArrayList<>(dimensions)); + } + return result; + } + + private Map> getTerms(SchemaMapInfo mapInfo, + Map dataSetNameMap) { + Map> termMap = new HashMap<>(); + Map> dataSetElementMatches = mapInfo.getDataSetElementMatches(); + for (Map.Entry> entry : dataSetElementMatches.entrySet()) { + DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey()); + if (dataSetResp == null) { + continue; + } + List terms = entry.getValue().stream().filter(schemaElementMatch + -> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); + termMap.put(dataSetResp.getName(), terms); + } + return termMap; + } + + /*** + * get time dimension SchemaElementMatch + * @param dataSetId + * @param dataSetName + * @return + */ + private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) { + SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName) + .type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build(); + + SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element) + .detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName()) + .similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build(); + + return timeDimensionMatch; + } + + private Function mergeFunction() { + return schemaElement -> SchemaElementMatch.builder().element(schemaElement) + .frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1) + .detectWord(schemaElement.getName()).build(); + } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticLayerServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/S2SemanticLayerService.java similarity index 56% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticLayerServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/S2SemanticLayerService.java index 286c1cdbd..8b1a9e285 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticLayerServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/S2SemanticLayerService.java @@ -3,15 +3,27 @@ package com.tencent.supersonic.headless.server.service.impl; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.headless.api.pojo.DataInfo; +import com.tencent.supersonic.headless.api.pojo.DataSetInfo; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.EntityInfo; import com.tencent.supersonic.headless.api.pojo.QueryParam; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; @@ -23,6 +35,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder; import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.translator.SemanticTranslator; @@ -41,16 +54,23 @@ import com.tencent.supersonic.headless.server.utils.StatUtils; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; +import java.time.LocalDate; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; @Service @Slf4j -public class SemanticLayerServiceImpl implements SemanticLayerService { +public class S2SemanticLayerService implements SemanticLayerService { private StatUtils statUtils; private final QueryUtils queryUtils; @@ -61,7 +81,7 @@ public class SemanticLayerServiceImpl implements SemanticLayerService { private final SchemaService schemaService; private final SemanticTranslator semanticTranslator; - public SemanticLayerServiceImpl( + public S2SemanticLayerService( StatUtils statUtils, QueryUtils queryUtils, QueryReqConverter queryReqConverter, @@ -88,6 +108,11 @@ public class SemanticLayerServiceImpl implements SemanticLayerService { return schemaService.getDataSetSchema(id); } + @Override + public List getDataSetSchema() { + return schemaService.getDataSetSchema(); + } + @Override @S2DataPermission @SneakyThrows @@ -261,4 +286,177 @@ public class SemanticLayerServiceImpl implements SemanticLayerService { throw e; } } + + public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) { + if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) { + EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema); + if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) { + entityInfo.setMetrics(null); + entityInfo.setDimensions(null); + return entityInfo; + } + String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey(); + if (StringUtils.isNotBlank(primaryKey)) { + String entityId = ""; + for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) { + if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName() + .equals(primaryKey)) { + if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) { + entityId = chatFilter.getValue().toString(); + } + } + } + entityInfo.setEntityId(entityId); + try { + fillEntityInfoValue(entityInfo, dataSetSchema, user); + return entityInfo; + } catch (Exception e) { + log.error("setMainModel error", e); + } + } + } + return null; + } + + private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) { + + EntityInfo entityInfo = new EntityInfo(); + if (dataSetSchema == null) { + return entityInfo; + } + Long dataSetId = dataSetSchema.getDataSet().getDataSet(); + DataSetInfo dataSetInfo = new DataSetInfo(); + dataSetInfo.setItemId(dataSetId.intValue()); + dataSetInfo.setName(dataSetSchema.getDataSet().getName()); + dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias()); + dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName()); + if (Objects.nonNull(dataSetSchema.getEntity())) { + dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName()); + } + entityInfo.setDataSetInfo(dataSetInfo); + TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig(); + if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) { + return entityInfo; + } + List dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() + .map(id -> { + SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id); + if (element == null) { + return null; + } + return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null); + }).filter(Objects::nonNull).collect(Collectors.toList()); + List metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() + .map(id -> { + SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id); + if (element == null) { + return null; + } + return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null); + }).filter(Objects::nonNull).collect(Collectors.toList()); + entityInfo.setDimensions(dimensions); + entityInfo.setMetrics(metrics); + return entityInfo; + } + + private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) { + SemanticQueryResp queryResultWithColumns = + getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user); + if (queryResultWithColumns != null) { + if (!org.springframework.util.CollectionUtils.isEmpty(queryResultWithColumns.getResultList()) + && queryResultWithColumns.getResultList().size() > 0) { + Map result = queryResultWithColumns.getResultList().get(0); + for (Map.Entry entry : result.entrySet()) { + String entryKey = getEntryKey(entry); + if (entry.getValue() == null || entryKey == null) { + continue; + } + entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName())) + .forEach(i -> i.setValue(entry.getValue().toString())); + entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName())) + .forEach(i -> i.setValue(entry.getValue().toString())); + } + } + } + } + + private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, + DataSetSchema dataSetSchema, User user) { + SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); + semanticParseInfo.setDataSet(dataSetSchema.getDataSet()); + semanticParseInfo.setQueryType(QueryType.DETAIL); + semanticParseInfo.setMetrics(getMetrics(entityInfo)); + semanticParseInfo.setDimensions(getDimensions(entityInfo)); + DateConf dateInfo = new DateConf(); + int unit = 1; + TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); + if (Objects.nonNull(timeDefaultConfig)) { + unit = timeDefaultConfig.getUnit(); + String date = LocalDate.now().plusDays(-unit).toString(); + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); + dateInfo.setStartDate(date); + dateInfo.setEndDate(date); + } else { + dateInfo.setUnit(unit); + dateInfo.setDateMode(DateConf.DateMode.RECENT); + } + semanticParseInfo.setDateInfo(dateInfo); + + //add filter + QueryFilter chatFilter = getQueryFilter(entityInfo); + Set chatFilters = new LinkedHashSet(); + chatFilters.add(chatFilter); + semanticParseInfo.setDimensionFilters(chatFilters); + + SemanticQueryResp queryResultWithColumns = null; + try { + QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo); + queryResultWithColumns = queryByReq(queryStructReq, user); + } catch (Exception e) { + log.warn("setMainModel queryByStruct error, e:", e); + } + return queryResultWithColumns; + } + + private QueryFilter getQueryFilter(EntityInfo entityInfo) { + QueryFilter chatFilter = new QueryFilter(); + chatFilter.setValue(entityInfo.getEntityId()); + chatFilter.setOperator(FilterOperatorEnum.EQUALS); + chatFilter.setBizName(getEntityPrimaryName(entityInfo)); + return chatFilter; + } + + private Set getDimensions(EntityInfo modelInfo) { + Set dimensions = new LinkedHashSet(); + for (DataInfo mainEntityDimension : modelInfo.getDimensions()) { + SchemaElement dimension = new SchemaElement(); + dimension.setBizName(mainEntityDimension.getBizName()); + dimensions.add(dimension); + } + return dimensions; + } + + private String getEntryKey(Map.Entry entry) { + // metric parser special handle, TODO delete + String entryKey = entry.getKey(); + if (entryKey.contains("__")) { + entryKey = entryKey.split("__")[1]; + } + return entryKey; + } + + private Set getMetrics(EntityInfo modelInfo) { + Set metrics = new LinkedHashSet(); + for (DataInfo metricValue : modelInfo.getMetrics()) { + SchemaElement metric = new SchemaElement(); + BeanUtils.copyProperties(metricValue, metric); + metrics.add(metric); + } + return metrics; + } + + private String getEntityPrimaryName(EntityInfo entityInfo) { + return entityInfo.getDataSetInfo().getPrimaryKey(); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java deleted file mode 100644 index 0f0ea636e..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java +++ /dev/null @@ -1,229 +0,0 @@ -package com.tencent.supersonic.headless.server.service.impl; - - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.headless.api.pojo.DataInfo; -import com.tencent.supersonic.headless.api.pojo.DataSetInfo; -import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.EntityInfo; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.SemanticSchema; -import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder; -import com.tencent.supersonic.headless.server.service.SemanticLayerService; -import com.tencent.supersonic.headless.server.service.SchemaService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - -import java.time.LocalDate; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -@Service -@Slf4j -public class SemanticService { - - @Autowired - private SchemaService schemaService; - - @Autowired - private SemanticLayerService queryService; - - public SemanticSchema getSemanticSchema() { - return new SemanticSchema(schemaService.getDataSetSchema()); - } - - public DataSetSchema getDataSetSchema(Long id) { - return schemaService.getDataSetSchema(id); - } - - public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) { - if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) { - EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema); - if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) { - entityInfo.setMetrics(null); - entityInfo.setDimensions(null); - return entityInfo; - } - String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey(); - if (StringUtils.isNotBlank(primaryKey)) { - String entityId = ""; - for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) { - if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName() - .equals(primaryKey)) { - if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) { - entityId = chatFilter.getValue().toString(); - } - } - } - entityInfo.setEntityId(entityId); - try { - fillEntityInfoValue(entityInfo, dataSetSchema, user); - return entityInfo; - } catch (Exception e) { - log.error("setMainModel error", e); - } - } - } - return null; - } - - private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) { - - EntityInfo entityInfo = new EntityInfo(); - if (dataSetSchema == null) { - return entityInfo; - } - Long dataSetId = dataSetSchema.getDataSet().getDataSet(); - DataSetInfo dataSetInfo = new DataSetInfo(); - dataSetInfo.setItemId(dataSetId.intValue()); - dataSetInfo.setName(dataSetSchema.getDataSet().getName()); - dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias()); - dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName()); - if (Objects.nonNull(dataSetSchema.getEntity())) { - dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName()); - } - entityInfo.setDataSetInfo(dataSetInfo); - TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig(); - if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) { - return entityInfo; - } - List dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() - .map(id -> { - SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id); - if (element == null) { - return null; - } - return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null); - }).filter(Objects::nonNull).collect(Collectors.toList()); - List metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() - .map(id -> { - SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id); - if (element == null) { - return null; - } - return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null); - }).filter(Objects::nonNull).collect(Collectors.toList()); - entityInfo.setDimensions(dimensions); - entityInfo.setMetrics(metrics); - return entityInfo; - } - - public void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) { - SemanticQueryResp queryResultWithColumns = - getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user); - if (queryResultWithColumns != null) { - if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList()) - && queryResultWithColumns.getResultList().size() > 0) { - Map result = queryResultWithColumns.getResultList().get(0); - for (Map.Entry entry : result.entrySet()) { - String entryKey = getEntryKey(entry); - if (entry.getValue() == null || entryKey == null) { - continue; - } - entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName())) - .forEach(i -> i.setValue(entry.getValue().toString())); - entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName())) - .forEach(i -> i.setValue(entry.getValue().toString())); - } - } - } - } - - public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, - DataSetSchema dataSetSchema, User user) { - SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); - semanticParseInfo.setDataSet(dataSetSchema.getDataSet()); - semanticParseInfo.setQueryType(QueryType.DETAIL); - semanticParseInfo.setMetrics(getMetrics(entityInfo)); - semanticParseInfo.setDimensions(getDimensions(entityInfo)); - DateConf dateInfo = new DateConf(); - int unit = 1; - TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); - if (Objects.nonNull(timeDefaultConfig)) { - unit = timeDefaultConfig.getUnit(); - String date = LocalDate.now().plusDays(-unit).toString(); - dateInfo.setDateMode(DateConf.DateMode.BETWEEN); - dateInfo.setStartDate(date); - dateInfo.setEndDate(date); - } else { - dateInfo.setUnit(unit); - dateInfo.setDateMode(DateConf.DateMode.RECENT); - } - semanticParseInfo.setDateInfo(dateInfo); - - //add filter - QueryFilter chatFilter = getQueryFilter(entityInfo); - Set chatFilters = new LinkedHashSet(); - chatFilters.add(chatFilter); - semanticParseInfo.setDimensionFilters(chatFilters); - - SemanticQueryResp queryResultWithColumns = null; - try { - QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo); - queryResultWithColumns = queryService.queryByReq(queryStructReq, user); - } catch (Exception e) { - log.warn("setMainModel queryByStruct error, e:", e); - } - return queryResultWithColumns; - } - - private QueryFilter getQueryFilter(EntityInfo entityInfo) { - QueryFilter chatFilter = new QueryFilter(); - chatFilter.setValue(entityInfo.getEntityId()); - chatFilter.setOperator(FilterOperatorEnum.EQUALS); - chatFilter.setBizName(getEntityPrimaryName(entityInfo)); - return chatFilter; - } - - private Set getDimensions(EntityInfo modelInfo) { - Set dimensions = new LinkedHashSet(); - for (DataInfo mainEntityDimension : modelInfo.getDimensions()) { - SchemaElement dimension = new SchemaElement(); - dimension.setBizName(mainEntityDimension.getBizName()); - dimensions.add(dimension); - } - return dimensions; - } - - private String getEntryKey(Map.Entry entry) { - // metric parser special handle, TODO delete - String entryKey = entry.getKey(); - if (entryKey.contains("__")) { - entryKey = entryKey.split("__")[1]; - } - return entryKey; - } - - private Set getMetrics(EntityInfo modelInfo) { - Set metrics = new LinkedHashSet(); - for (DataInfo metricValue : modelInfo.getMetrics()) { - SchemaElement metric = new SchemaElement(); - BeanUtils.copyProperties(metricValue, metric); - metrics.add(metric); - } - return metrics; - } - - private String getEntityPrimaryName(EntityInfo entityInfo) { - return entityInfo.getDataSetInfo().getPrimaryKey(); - } - -} diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java index e996a1c71..19585b139 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java @@ -70,7 +70,7 @@ public class MetricServiceImplTest { DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class); DimensionService dimensionService = Mockito.mock(DimensionService.class); TagMetaService tagMetaService = Mockito.mock(TagMetaService.class); - MetaDiscoveryService metaDiscoveryService = Mockito.mock(MetaDiscoveryService.class); + RetrieveService metaDiscoveryService = Mockito.mock(RetrieveService.class); return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper, collectService, dataSetService, eventPublisher, dimensionService, tagMetaService, metaDiscoveryService); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java b/launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java index baf8c7c00..2f0e39e2d 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java @@ -84,7 +84,7 @@ public class Text2SQLEval extends BaseTest { AgentConfig agentConfig = new AgentConfig(); agentConfig.getTools().add(getLLMQueryTool()); agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); - agent.setLlmConfig(getLLMConfig(LLMType.DEEPSEEK)); + agent.setLlmConfig(getLLMConfig(LLMType.GPT)); MultiTurnConfig multiTurnConfig = new MultiTurnConfig(); multiTurnConfig.setEnableMultiTurn(enableMultiturn); agent.setMultiTurnConfig(multiTurnConfig); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java index e309a757b..f239880b2 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java @@ -4,7 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.QueryDataType; import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp; -import com.tencent.supersonic.headless.server.service.MetaDiscoveryService; +import com.tencent.supersonic.headless.server.service.RetrieveService; import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -15,7 +15,7 @@ import java.util.Collections; public class MetaDiscoveryTest extends BaseTest { @Autowired - protected MetaDiscoveryService metaDiscoveryService; + protected RetrieveService metaDiscoveryService; @Test public void testGetMapMeta() throws Exception { @@ -24,7 +24,7 @@ public class MetaDiscoveryTest extends BaseTest { queryMapReq.setTopN(10); queryMapReq.setUser(User.getFakeUser()); queryMapReq.setDataSetNames(Collections.singletonList("超音数数据集")); - MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq); + MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq); Assertions.assertNotNull(mapMeta); Assertions.assertNotEquals(0, mapMeta.getDataSetMapInfo().get("超音数数据集").getMapFields()); @@ -39,7 +39,7 @@ public class MetaDiscoveryTest extends BaseTest { queryMapReq.setUser(User.getFakeUser()); queryMapReq.setDataSetNames(Collections.singletonList("艺人库")); queryMapReq.setQueryDataType(QueryDataType.TAG); - MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq); + MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq); Assert.assertNotNull(mapMeta); } @@ -51,7 +51,7 @@ public class MetaDiscoveryTest extends BaseTest { queryMapReq.setUser(User.getFakeUser()); queryMapReq.setDataSetNames(Collections.singletonList("超音数")); queryMapReq.setQueryDataType(QueryDataType.METRIC); - MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq); + MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq); Assert.assertNotNull(mapMeta); } }