(improvement)(Headless) Migrate the map interface to headless, and change agentId to dataSetIds. (#899)

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java

@@ -41,13 +41,6 @@ public class ChatQueryController {
         return chatService.search(chatParseReq);
     }
 
-    @PostMapping("map")
-    public Object map(@RequestBody ChatParseReq chatParseReq,
-                      HttpServletRequest request, HttpServletResponse response) throws Exception {
-        chatParseReq.setUser(UserHolder.findUser(request, response));
-        return chatService.performMapping(chatParseReq);
-    }
-
     @PostMapping("parse")
     public Object parse(@RequestBody ChatParseReq chatParseReq,
                         HttpServletRequest request, HttpServletResponse response) throws Exception {

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/ChatService.java

@@ -6,7 +6,6 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
-import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
 import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
 import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
 import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
@@ -17,8 +16,6 @@ public interface ChatService {
 
     List<SearchResult> search(ChatParseReq chatParseReq);
 
-    MapInfoResp performMapping(ChatParseReq chatParseReq);
-
     ParseResp performParsing(ChatParseReq chatParseReq);
 
     QueryResult performExecution(ChatExecuteReq chatExecuteReq) throws Exception;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java

@@ -17,7 +17,6 @@ import com.tencent.supersonic.chat.server.service.AgentService;
 import com.tencent.supersonic.chat.server.service.ChatManageService;
 import com.tencent.supersonic.chat.server.service.ChatService;
 import com.tencent.supersonic.chat.server.util.ComponentFactory;
-import com.tencent.supersonic.chat.server.util.MapInfoConverter;
 import com.tencent.supersonic.chat.server.util.QueryReqConverter;
 import com.tencent.supersonic.chat.server.util.SimilarQueryManager;
 import com.tencent.supersonic.common.util.BeanMapper;
@@ -26,7 +25,6 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
-import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
 import com.tencent.supersonic.headless.api.pojo.response.MapResp;
 import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
 import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
@@ -64,11 +62,6 @@ public class ChatServiceImpl implements ChatService {
         return searchService.search(queryReq);
     }
 
-    @Override
-    public MapInfoResp performMapping(ChatParseReq chatParseReq) {
-        return getMapResp(chatParseReq);
-    }
-
     @Override
     public ParseResp performParsing(ChatParseReq chatParseReq) {
         ParseResp parseResp = new ParseResp(chatParseReq.getChatId(), chatParseReq.getQueryText());
@@ -113,17 +106,6 @@ public class ChatServiceImpl implements ChatService {
         return chatParseContext;
     }
 
-    private MapInfoResp getMapResp(ChatParseReq chatParseReq) {
-        ChatParseContext chatParseContext = new ChatParseContext();
-        BeanMapper.mapper(chatParseReq, chatParseContext);
-        AgentService agentService = ContextUtils.getBean(AgentService.class);
-        Agent agent = agentService.getAgent(chatParseReq.getAgentId());
-        chatParseContext.setAgent(agent);
-        QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
-        MapResp mapResp = chatQueryService.performMapping(queryReq);
-        return MapInfoConverter.convert(mapResp, chatParseReq.getTopN());
-    }
-
     private ChatExecuteContext buildExecuteContext(ChatExecuteReq chatExecuteReq) {
         ChatExecuteContext chatExecuteContext = new ChatExecuteContext();
         BeanMapper.mapper(chatExecuteReq, chatExecuteContext);

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MapModeEnum.java

@@ -0,0 +1,7 @@
+package com.tencent.supersonic.headless.api.pojo.enums;
+
+public enum MapModeEnum {
+    STRICT,
+    MODERATE,
+    LOOSE;
+}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMapReq.java

@@ -0,0 +1,18 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class QueryMapReq {
+    private String queryText;
+    private List<String> dataSetNames;
+    private User user;
+    private Integer topN = 10;
+    private MapModeEnum mapModeEnum;
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java

@@ -33,6 +33,8 @@ public class MetaFilter {
 
     private List<Long> ids;
 
+    private List<String> names;
+
     private List<String> fieldsDepend;
 
     private Integer isTag;

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java

@@ -2,59 +2,20 @@ package com.tencent.supersonic.headless.server.pojo;
 
 import lombok.Data;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.LinkedHashSet;
-import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 @Data
 public class ModelCluster {
 
     private static final String split = "_";
-
     private Set<Long> modelIds = new LinkedHashSet<>();
-
-    private Set<String> modelNames = new LinkedHashSet<>();
-
     private String key;
-
-    private String name;
-
     public static ModelCluster build(Set<Long> modelIds) {
         ModelCluster modelCluster = new ModelCluster();
         modelCluster.setModelIds(modelIds);
         modelCluster.setKey(StringUtils.join(modelIds, split));
         return modelCluster;
     }
-
-    public static ModelCluster build(String key) {
-        ModelCluster modelCluster = new ModelCluster();
-        modelCluster.setModelIds(getModelIdFromKey(key));
-        modelCluster.setKey(key);
-        return modelCluster;
-    }
-
-    public void buildName(Map<Long, String> modelNameMap) {
-        modelNames = modelNameMap.entrySet().stream().filter(entry ->
-                        modelIds.contains(entry.getKey())).map(Map.Entry::getValue)
-                .collect(Collectors.toSet());
-        name = String.join(split, modelNames);
-    }
-
-    public static Set<Long> getModelIdFromKey(String key) {
-        return Arrays.stream(key.split(split))
-                .map(Long::parseLong).collect(Collectors.toSet());
-    }
-
-    public Long getFirstModel() {
-        if (CollectionUtils.isEmpty(modelIds)) {
-            return -1L;
-        }
-        return new ArrayList<>(modelIds).get(0);
-    }
-
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetaDiscoveryApiController.java

@@ -0,0 +1,33 @@
+package com.tencent.supersonic.headless.server.rest.api;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
+import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@RestController
+@RequestMapping("/api/semantic/query")
+@Slf4j
+public class MetaDiscoveryApiController {
+
+    @Autowired
+    private MetaDiscoveryService metaDiscovery;
+
+    @PostMapping("map")
+    public Object map(@RequestBody QueryMapReq queryMapReq,
+                      HttpServletRequest request, HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        queryMapReq.setUser(user);
+        return metaDiscovery.getMapMeta(queryMapReq);
+    }
+
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java

@@ -30,6 +30,10 @@ public interface DataSetService {
 
     List<DataSetResp> getDataSets(String dataSetName, User user);
 
+    List<DataSetResp> getDataSets(List<String> dataSetNames, User user);
+
+    Map<Long, String> getDataSetIdToNameMap(List<Long> dataSetIds);
+
     List<DataSetResp> getDataSetsInheritAuth(User user, Long domainId);
 
     SemanticQueryReq convert(QueryDataSetReq queryDataSetReq);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaDiscoveryService.java

@@ -0,0 +1,10 @@
+package com.tencent.supersonic.headless.server.service;
+
+import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
+import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
+
+public interface MetaDiscoveryService {
+
+    MapInfoResp getMapMeta(QueryMapReq queryMapReq);
+
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java

@@ -38,17 +38,17 @@ import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
 import com.tencent.supersonic.headless.api.pojo.response.QueryState;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import com.tencent.supersonic.headless.core.chat.corrector.SemanticCorrector;
+import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
+import com.tencent.supersonic.headless.core.chat.knowledge.SearchService;
+import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
+import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.headless.core.chat.mapper.SchemaMapper;
 import com.tencent.supersonic.headless.core.chat.parser.SemanticParser;
 import com.tencent.supersonic.headless.core.chat.query.QueryManager;
 import com.tencent.supersonic.headless.core.chat.query.SemanticQuery;
 import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlQuery;
 import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery;
-import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
-import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
-import com.tencent.supersonic.headless.core.chat.knowledge.SearchService;
-import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
-import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.headless.core.pojo.ChatContext;
 import com.tencent.supersonic.headless.core.pojo.QueryContext;
 import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java

@@ -131,18 +131,38 @@ public class DataSetServiceImpl
 
     @Override
     public List<DataSetResp> getDataSets(User user) {
-        List<DataSetResp> dataSetResps = getDataSetList(new MetaFilter(), user);
+        MetaFilter metaFilter = new MetaFilter();
-        return getDataSetFilterByAuth(dataSetResps, user);
+        return getDataSetsByAuth(user, metaFilter);
     }
 
     @Override
     public List<DataSetResp> getDataSets(String dataSetName, User user) {
         MetaFilter metaFilter = new MetaFilter();
         metaFilter.setName(dataSetName);
+        return getDataSetsByAuth(user, metaFilter);
+    }
+
+    @Override
+    public List<DataSetResp> getDataSets(List<String> dataSetNames, User user) {
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setNames(dataSetNames);
+        return getDataSetsByAuth(user, metaFilter);
+    }
+
+    private List<DataSetResp> getDataSetsByAuth(User user, MetaFilter metaFilter) {
         List<DataSetResp> dataSetResps = getDataSetList(metaFilter, user);
         return getDataSetFilterByAuth(dataSetResps, user);
     }
 
+    @Override
+    public Map<Long, String> getDataSetIdToNameMap(List<Long> dataSetIds) {
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setIds(dataSetIds);
+        List<DataSetResp> dataSetResps = getDataSetList(metaFilter, User.getFakeUser());
+        return dataSetResps.stream().collect(
+                Collectors.toMap(DataSetResp::getId, DataSetResp::getName, (k1, k2) -> k1));
+    }
+
     @Override
     public List<DataSetResp> getDataSetsInheritAuth(User user, Long domainId) {
         List<DataSetResp> dataSetResps = getDataSetList(new MetaFilter(), user);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java

@@ -1,19 +1,25 @@
-package com.tencent.supersonic.chat.server.util;
+package com.tencent.supersonic.headless.server.service.impl;
 
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
+import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
 import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
 import com.tencent.supersonic.headless.api.pojo.response.MapResp;
 import com.tencent.supersonic.headless.core.chat.knowledge.builder.BaseWordBuilder;
-import com.tencent.supersonic.headless.server.service.impl.SemanticService;
+import com.tencent.supersonic.headless.server.service.ChatQueryService;
+import com.tencent.supersonic.headless.server.service.DataSetService;
+import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -25,22 +31,47 @@ import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-public class MapInfoConverter {
+@Service
+public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
 
-    public static MapInfoResp convert(MapResp mapResp, Integer topN) {
+    @Autowired
+    private DataSetService dataSetService;
+
+    @Autowired
+    private ChatQueryService chatQueryService;
+
+    @Autowired
+    private SemanticService semanticService;
+
+    @Override
+    public MapInfoResp getMapMeta(QueryMapReq queryMapReq) {
+
+        QueryReq queryReq = new QueryReq();
+        BeanUtils.copyProperties(queryMapReq, queryReq);
+        List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
+        Set<Long> dataSetIds = dataSets.stream().map(dataSetResp -> dataSetResp.getId()).collect(Collectors.toSet());
+        queryReq.setDataSetIds(dataSetIds);
+
+        MapResp mapResp = chatQueryService.performMapping(queryReq);
+        return convert(mapResp, queryMapReq.getTopN());
+    }
+
+    public MapInfoResp convert(MapResp mapResp, Integer topN) {
         MapInfoResp mapInfoResp = new MapInfoResp();
         if (Objects.isNull(mapResp)) {
             return mapInfoResp;
         }
         BeanUtils.copyProperties(mapResp, mapInfoResp);
-        Map<Long, String> dataSetMap = mapResp.getMapInfo().generateDataSetMap();
+        Set<Long> dataSetIds = mapResp.getMapInfo().getDataSetElementMatches().keySet();
+        Map<Long, String> dataSetMap = dataSetService.getDataSetIdToNameMap(new ArrayList<>(dataSetIds));
+
         mapInfoResp.setMapFields(getMapFields(mapResp.getMapInfo(), dataSetMap));
         mapInfoResp.setTopFields(getTopFields(topN, mapResp.getMapInfo(), dataSetMap));
         return mapInfoResp;
     }
 
-    private static Map<String, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
+    private Map<String, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
-                                                                      Map<Long, String> dataSetMap) {
+                                                               Map<Long, String> dataSetMap) {
         Map<String, List<SchemaElementMatch>> result = new HashMap<>();
         for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
             List<SchemaElementMatch> values = entry.getValue();
@@ -51,13 +82,12 @@ public class MapInfoConverter {
         return result;
     }
 
-    private static Map<String, List<SchemaElementMatch>> getTopFields(Integer topN,
+    private Map<String, List<SchemaElementMatch>> getTopFields(Integer topN,
-                                                                      SchemaMapInfo mapInfo,
+                                                               SchemaMapInfo mapInfo,
-                                                                      Map<Long, String> dataSetMap) {
+                                                               Map<Long, String> dataSetMap) {
         Set<Long> dataSetIds = mapInfo.getDataSetElementMatches().keySet();
         Map<String, List<SchemaElementMatch>> result = new HashMap<>();
 
-        SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
         SemanticSchema semanticSchema = semanticService.getSemanticSchema();
         for (Long dataSetId : dataSetIds) {
             String dataSetName = dataSetMap.get(dataSetId);
@@ -89,7 +119,7 @@ public class MapInfoConverter {
      * @param dataSetName
      * @return
      */
-    private static SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
+    private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
         SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName)
                 .type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();
 
@@ -100,7 +130,7 @@ public class MapInfoConverter {
         return timeDimensionMatch;
     }
 
-    private static Function<SchemaElement, SchemaElementMatch> mergeFunction() {
+    private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
         return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
                 .frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
                 .detectWord(schemaElement.getName()).build();

launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java

@@ -0,0 +1,31 @@
+package com.tencent.supersonic.headless;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
+import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
+import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Collections;
+
+public class MetaDiscoveryTest extends BaseTest {
+
+    @Autowired
+    protected MetaDiscoveryService metaDiscoveryService;
+
+    @Test
+    public void testGetMapMeta() throws Exception {
+        QueryMapReq queryMapReq = new QueryMapReq();
+        queryMapReq.setQueryText("对比alice和lucy的访问次数");
+        queryMapReq.setTopN(10);
+        queryMapReq.setUser(User.getFakeUser());
+        queryMapReq.setDataSetNames(Collections.singletonList("超音数"));
+        MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
+
+        Assert.assertNotNull(mapMeta);
+        Assert.assertNotEquals(0, mapMeta.getMapFields());
+        Assert.assertNotEquals(0, mapMeta.getTopFields());
+    }
+}