(improvement)(Headless) Migrate the map interface to headless, and change agentId to dataSetIds. (#899)

2025-12-10 11:07:06 +00:00 · 2024-04-09 10:58:40 +08:00
parent a29ba2257a
commit 43b9d5545d
14 changed files with 176 additions and 88 deletions
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java
@@ -41,13 +41,6 @@ public class ChatQueryController {
        return chatService.search(chatParseReq);
    }

-    @PostMapping("map")
-    public Object map(@RequestBody ChatParseReq chatParseReq,
-                      HttpServletRequest request, HttpServletResponse response) throws Exception {
-        chatParseReq.setUser(UserHolder.findUser(request, response));
-        return chatService.performMapping(chatParseReq);
-    }
-
    @PostMapping("parse")
    public Object parse(@RequestBody ChatParseReq chatParseReq,
                        HttpServletRequest request, HttpServletResponse response) throws Exception {
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java
@@ -6,7 +6,6 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
-import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
 import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
 import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
 import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
@@ -17,8 +16,6 @@ public interface ChatService {

    List<SearchResult> search(ChatParseReq chatParseReq);

-    MapInfoResp performMapping(ChatParseReq chatParseReq);
-
    ParseResp performParsing(ChatParseReq chatParseReq);

    QueryResult performExecution(ChatExecuteReq chatExecuteReq) throws Exception;
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java
@@ -17,7 +17,6 @@ import com.tencent.supersonic.chat.server.service.AgentService;
 import com.tencent.supersonic.chat.server.service.ChatManageService;
 import com.tencent.supersonic.chat.server.service.ChatService;
 import com.tencent.supersonic.chat.server.util.ComponentFactory;
-import com.tencent.supersonic.chat.server.util.MapInfoConverter;
 import com.tencent.supersonic.chat.server.util.QueryReqConverter;
 import com.tencent.supersonic.chat.server.util.SimilarQueryManager;
 import com.tencent.supersonic.common.util.BeanMapper;
@@ -26,7 +25,6 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
-import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
 import com.tencent.supersonic.headless.api.pojo.response.MapResp;
 import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
 import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
@@ -64,11 +62,6 @@ public class ChatServiceImpl implements ChatService {
        return searchService.search(queryReq);
    }

-    @Override
-    public MapInfoResp performMapping(ChatParseReq chatParseReq) {
-        return getMapResp(chatParseReq);
-    }
-
    @Override
    public ParseResp performParsing(ChatParseReq chatParseReq) {
        ParseResp parseResp = new ParseResp(chatParseReq.getChatId(), chatParseReq.getQueryText());
@@ -113,17 +106,6 @@ public class ChatServiceImpl implements ChatService {
        return chatParseContext;
    }

-    private MapInfoResp getMapResp(ChatParseReq chatParseReq) {
-        ChatParseContext chatParseContext = new ChatParseContext();
-        BeanMapper.mapper(chatParseReq, chatParseContext);
-        AgentService agentService = ContextUtils.getBean(AgentService.class);
-        Agent agent = agentService.getAgent(chatParseReq.getAgentId());
-        chatParseContext.setAgent(agent);
-        QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
-        MapResp mapResp = chatQueryService.performMapping(queryReq);
-        return MapInfoConverter.convert(mapResp, chatParseReq.getTopN());
-    }
-
    private ChatExecuteContext buildExecuteContext(ChatExecuteReq chatExecuteReq) {
        ChatExecuteContext chatExecuteContext = new ChatExecuteContext();
        BeanMapper.mapper(chatExecuteReq, chatExecuteContext);
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MapModeEnum.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MapModeEnum.java
@@ -0,0 +1,7 @@
+package com.tencent.supersonic.headless.api.pojo.enums;
+
+public enum MapModeEnum {
+    STRICT,
+    MODERATE,
+    LOOSE;
+}
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMapReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMapReq.java
@@ -0,0 +1,18 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class QueryMapReq {
+    private String queryText;
+    private List<String> dataSetNames;
+    private User user;
+    private Integer topN = 10;
+    private MapModeEnum mapModeEnum;
+}
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
@@ -33,6 +33,8 @@ public class MetaFilter {

    private List<Long> ids;

+    private List<String> names;
+
    private List<String> fieldsDepend;

    private Integer isTag;
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java
@@ -2,59 +2,20 @@ package com.tencent.supersonic.headless.server.pojo;

 import lombok.Data;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.util.CollectionUtils;

-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.LinkedHashSet;
-import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;

@Data
 public class ModelCluster {

    private static final String split = "_";
-
    private Set<Long> modelIds = new LinkedHashSet<>();
-
-    private Set<String> modelNames = new LinkedHashSet<>();
-
    private String key;
-
-    private String name;
-
    public static ModelCluster build(Set<Long> modelIds) {
        ModelCluster modelCluster = new ModelCluster();
        modelCluster.setModelIds(modelIds);
        modelCluster.setKey(StringUtils.join(modelIds, split));
        return modelCluster;
    }
-
-    public static ModelCluster build(String key) {
-        ModelCluster modelCluster = new ModelCluster();
-        modelCluster.setModelIds(getModelIdFromKey(key));
-        modelCluster.setKey(key);
-        return modelCluster;
-    }
-
-    public void buildName(Map<Long, String> modelNameMap) {
-        modelNames = modelNameMap.entrySet().stream().filter(entry ->
-                        modelIds.contains(entry.getKey())).map(Map.Entry::getValue)
-                .collect(Collectors.toSet());
-        name = String.join(split, modelNames);
-    }
-
-    public static Set<Long> getModelIdFromKey(String key) {
-        return Arrays.stream(key.split(split))
-                .map(Long::parseLong).collect(Collectors.toSet());
-    }
-
-    public Long getFirstModel() {
-        if (CollectionUtils.isEmpty(modelIds)) {
-            return -1L;
-        }
-        return new ArrayList<>(modelIds).get(0);
-    }
-
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetaDiscoveryApiController.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetaDiscoveryApiController.java
@@ -0,0 +1,33 @@
+package com.tencent.supersonic.headless.server.rest.api;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
+import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@RestController
+@RequestMapping("/api/semantic/query")
+@Slf4j
+public class MetaDiscoveryApiController {
+
+    @Autowired
+    private MetaDiscoveryService metaDiscovery;
+
+    @PostMapping("map")
+    public Object map(@RequestBody QueryMapReq queryMapReq,
+                      HttpServletRequest request, HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        queryMapReq.setUser(user);
+        return metaDiscovery.getMapMeta(queryMapReq);
+    }
+
+}
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java
@@ -30,6 +30,10 @@ public interface DataSetService {

    List<DataSetResp> getDataSets(String dataSetName, User user);

+    List<DataSetResp> getDataSets(List<String> dataSetNames, User user);
+
+    Map<Long, String> getDataSetIdToNameMap(List<Long> dataSetIds);
+
    List<DataSetResp> getDataSetsInheritAuth(User user, Long domainId);

    SemanticQueryReq convert(QueryDataSetReq queryDataSetReq);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaDiscoveryService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaDiscoveryService.java
@@ -0,0 +1,10 @@
+package com.tencent.supersonic.headless.server.service;
+
+import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
+import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
+
+public interface MetaDiscoveryService {
+
+    MapInfoResp getMapMeta(QueryMapReq queryMapReq);
+
+}
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java
@@ -38,17 +38,17 @@ import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
 import com.tencent.supersonic.headless.api.pojo.response.QueryState;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import com.tencent.supersonic.headless.core.chat.corrector.SemanticCorrector;
+import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
+import com.tencent.supersonic.headless.core.chat.knowledge.SearchService;
+import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
+import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.headless.core.chat.mapper.SchemaMapper;
 import com.tencent.supersonic.headless.core.chat.parser.SemanticParser;
 import com.tencent.supersonic.headless.core.chat.query.QueryManager;
 import com.tencent.supersonic.headless.core.chat.query.SemanticQuery;
 import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlQuery;
 import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery;
-import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
-import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
-import com.tencent.supersonic.headless.core.chat.knowledge.SearchService;
-import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
-import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.headless.core.pojo.ChatContext;
 import com.tencent.supersonic.headless.core.pojo.QueryContext;
 import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO;
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java
@@ -131,18 +131,38 @@ public class DataSetServiceImpl

    @Override
    public List<DataSetResp> getDataSets(User user) {
-        List<DataSetResp> dataSetResps = getDataSetList(new MetaFilter(), user);
-        return getDataSetFilterByAuth(dataSetResps, user);
+        MetaFilter metaFilter = new MetaFilter();
+        return getDataSetsByAuth(user, metaFilter);
    }

    @Override
    public List<DataSetResp> getDataSets(String dataSetName, User user) {
        MetaFilter metaFilter = new MetaFilter();
        metaFilter.setName(dataSetName);
+        return getDataSetsByAuth(user, metaFilter);
+    }
+
+    @Override
+    public List<DataSetResp> getDataSets(List<String> dataSetNames, User user) {
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setNames(dataSetNames);
+        return getDataSetsByAuth(user, metaFilter);
+    }
+
+    private List<DataSetResp> getDataSetsByAuth(User user, MetaFilter metaFilter) {
        List<DataSetResp> dataSetResps = getDataSetList(metaFilter, user);
        return getDataSetFilterByAuth(dataSetResps, user);
    }

+    @Override
+    public Map<Long, String> getDataSetIdToNameMap(List<Long> dataSetIds) {
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setIds(dataSetIds);
+        List<DataSetResp> dataSetResps = getDataSetList(metaFilter, User.getFakeUser());
+        return dataSetResps.stream().collect(
+                Collectors.toMap(DataSetResp::getId, DataSetResp::getName, (k1, k2) -> k1));
+    }
+
    @Override
    public List<DataSetResp> getDataSetsInheritAuth(User user, Long domainId) {
        List<DataSetResp> dataSetResps = getDataSetList(new MetaFilter(), user);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java
@@ -1,19 +1,25 @@
-package com.tencent.supersonic.chat.server.util;
+package com.tencent.supersonic.headless.server.service.impl;

 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
+import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
 import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
 import com.tencent.supersonic.headless.api.pojo.response.MapResp;
 import com.tencent.supersonic.headless.core.chat.knowledge.builder.BaseWordBuilder;
-import com.tencent.supersonic.headless.server.service.impl.SemanticService;
+import com.tencent.supersonic.headless.server.service.ChatQueryService;
+import com.tencent.supersonic.headless.server.service.DataSetService;
+import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;

 import java.util.ArrayList;
 import java.util.Comparator;
@@ -25,22 +31,47 @@ import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;

-public class MapInfoConverter {
+@Service
+public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {

-    public static MapInfoResp convert(MapResp mapResp, Integer topN) {
+    @Autowired
+    private DataSetService dataSetService;
+
+    @Autowired
+    private ChatQueryService chatQueryService;
+
+    @Autowired
+    private SemanticService semanticService;
+
+    @Override
+    public MapInfoResp getMapMeta(QueryMapReq queryMapReq) {
+
+        QueryReq queryReq = new QueryReq();
+        BeanUtils.copyProperties(queryMapReq, queryReq);
+        List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
+        Set<Long> dataSetIds = dataSets.stream().map(dataSetResp -> dataSetResp.getId()).collect(Collectors.toSet());
+        queryReq.setDataSetIds(dataSetIds);
+
+        MapResp mapResp = chatQueryService.performMapping(queryReq);
+        return convert(mapResp, queryMapReq.getTopN());
+    }
+
+    public MapInfoResp convert(MapResp mapResp, Integer topN) {
        MapInfoResp mapInfoResp = new MapInfoResp();
        if (Objects.isNull(mapResp)) {
            return mapInfoResp;
        }
        BeanUtils.copyProperties(mapResp, mapInfoResp);
-        Map<Long, String> dataSetMap = mapResp.getMapInfo().generateDataSetMap();
+        Set<Long> dataSetIds = mapResp.getMapInfo().getDataSetElementMatches().keySet();
+        Map<Long, String> dataSetMap = dataSetService.getDataSetIdToNameMap(new ArrayList<>(dataSetIds));
+
        mapInfoResp.setMapFields(getMapFields(mapResp.getMapInfo(), dataSetMap));
        mapInfoResp.setTopFields(getTopFields(topN, mapResp.getMapInfo(), dataSetMap));
        return mapInfoResp;
    }

-    private static Map<String, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
-                                                                      Map<Long, String> dataSetMap) {
+    private Map<String, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
+                                                               Map<Long, String> dataSetMap) {
        Map<String, List<SchemaElementMatch>> result = new HashMap<>();
        for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
            List<SchemaElementMatch> values = entry.getValue();
@@ -51,13 +82,12 @@ public class MapInfoConverter {
        return result;
    }

-    private static Map<String, List<SchemaElementMatch>> getTopFields(Integer topN,
-                                                                      SchemaMapInfo mapInfo,
-                                                                      Map<Long, String> dataSetMap) {
+    private Map<String, List<SchemaElementMatch>> getTopFields(Integer topN,
+                                                               SchemaMapInfo mapInfo,
+                                                               Map<Long, String> dataSetMap) {
        Set<Long> dataSetIds = mapInfo.getDataSetElementMatches().keySet();
        Map<String, List<SchemaElementMatch>> result = new HashMap<>();

-        SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
        SemanticSchema semanticSchema = semanticService.getSemanticSchema();
        for (Long dataSetId : dataSetIds) {
            String dataSetName = dataSetMap.get(dataSetId);
@@ -89,7 +119,7 @@ public class MapInfoConverter {
     * @param dataSetName
     * @return
     */
-    private static SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
+    private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
        SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName)
                .type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();

@@ -100,7 +130,7 @@ public class MapInfoConverter {
        return timeDimensionMatch;
    }

-    private static Function<SchemaElement, SchemaElementMatch> mergeFunction() {
+    private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
        return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
                .frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
                .detectWord(schemaElement.getName()).build();
--- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java
+++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java
@@ -0,0 +1,31 @@
+package com.tencent.supersonic.headless;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
+import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
+import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Collections;
+
+public class MetaDiscoveryTest extends BaseTest {
+
+    @Autowired
+    protected MetaDiscoveryService metaDiscoveryService;
+
+    @Test
+    public void testGetMapMeta() throws Exception {
+        QueryMapReq queryMapReq = new QueryMapReq();
+        queryMapReq.setQueryText("对比alice和lucy的访问次数");
+        queryMapReq.setTopN(10);
+        queryMapReq.setUser(User.getFakeUser());
+        queryMapReq.setDataSetNames(Collections.singletonList("超音数"));
+        MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
+
+        Assert.assertNotNull(mapMeta);
+        Assert.assertNotEquals(0, mapMeta.getMapFields());
+        Assert.assertNotEquals(0, mapMeta.getTopFields());
+    }
+}