[improvement](chat) Merge HanlpDictMapper and FuzzyNameMapper into KeywordMapper. (#493)

Co-authored-by: lexluo <lexluo@tencent.com>

chat/core/src/main/java/com/tencent/supersonic/chat/mapper/DatabaseMatchStrategy.java

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
 import com.tencent.supersonic.chat.config.OptimizationConfig;
 import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.knowledge.dictionary.FuzzyResult;
+import com.tencent.supersonic.knowledge.dictionary.DatabaseMapResult;
 import com.tencent.supersonic.knowledge.service.SchemaService;
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -26,7 +26,7 @@ import org.springframework.util.CollectionUtils;
  */
 @Service
 @Slf4j
-public class FuzzyNameMatchStrategy extends BaseMatchStrategy<FuzzyResult> {
+public class DatabaseMatchStrategy extends BaseMatchStrategy<DatabaseMapResult> {
 
     @Autowired
     private OptimizationConfig optimizationConfig;
@@ -37,25 +37,25 @@ public class FuzzyNameMatchStrategy extends BaseMatchStrategy<FuzzyResult> {
     private List<SchemaElement> allElements;
 
     @Override
-    public Map<MatchText, List<FuzzyResult>> match(QueryContext queryContext, List<Term> terms,
+    public Map<MatchText, List<DatabaseMapResult>> match(QueryContext queryContext, List<Term> terms,
             Set<Long> detectModelIds) {
         this.allElements = getSchemaElements();
         return super.match(queryContext, terms, detectModelIds);
     }
 
     @Override
-    public boolean needDelete(FuzzyResult oneRoundResult, FuzzyResult existResult) {
+    public boolean needDelete(DatabaseMapResult oneRoundResult, DatabaseMapResult existResult) {
         return getMapKey(oneRoundResult).equals(getMapKey(existResult))
                 && existResult.getDetectWord().length() < oneRoundResult.getDetectWord().length();
     }
 
     @Override
-    public String getMapKey(FuzzyResult a) {
+    public String getMapKey(DatabaseMapResult a) {
         return a.getName() + Constants.UNDERLINE + a.getSchemaElement().getId()
                 + Constants.UNDERLINE + a.getSchemaElement().getName();
     }
 
-    public void detectByStep(QueryContext queryContext, Set<FuzzyResult> existResults, Set<Long> detectModelIds,
+    public void detectByStep(QueryContext queryContext, Set<DatabaseMapResult> existResults, Set<Long> detectModelIds,
             Integer startIndex, Integer index, int offset) {
         String detectSegment = queryContext.getRequest().getQueryText().substring(startIndex, index);
         if (StringUtils.isBlank(detectSegment)) {
@@ -80,11 +80,11 @@ public class FuzzyNameMatchStrategy extends BaseMatchStrategy<FuzzyResult> {
                         .collect(Collectors.toSet());
             }
             for (SchemaElement schemaElement : schemaElements) {
-                FuzzyResult fuzzyResult = new FuzzyResult();
+                DatabaseMapResult databaseMapResult = new DatabaseMapResult();
-                fuzzyResult.setDetectWord(detectSegment);
+                databaseMapResult.setDetectWord(detectSegment);
-                fuzzyResult.setName(schemaElement.getName());
+                databaseMapResult.setName(schemaElement.getName());
-                fuzzyResult.setSchemaElement(schemaElement);
+                databaseMapResult.setSchemaElement(schemaElement);
-                existResults.add(fuzzyResult);
+                existResults.add(databaseMapResult);
             }
         }
     }

chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java

@@ -15,7 +15,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 
 /***
- * A mapper that is capable of semantic understanding of text.
+ * A mapper that recognize elements through embedding.
  */
 @Slf4j
 public class EmbeddingMapper extends BaseMapper {
@@ -23,7 +23,6 @@ public class EmbeddingMapper extends BaseMapper {
     @Override
     public void doMap(QueryContext queryContext) {
         //1. query from embedding by queryText
-
         String queryText = queryContext.getRequest().getQueryText();
         List<Term> terms = HanlpHelper.getTerms(queryText);
 
@@ -39,11 +38,11 @@ public class EmbeddingMapper extends BaseMapper {
             SchemaElement schemaElement = JSONObject.parseObject(JSONObject.toJSONString(matchResult.getMetadata()),
                     SchemaElement.class);
 
-            if (StringUtils.isBlank(matchResult.getMetadata().get("modelId"))) {
+            String modelIdStr = matchResult.getMetadata().get("modelId");
+            if (StringUtils.isBlank(modelIdStr)) {
                 continue;
             }
-            long modelId = Long.parseLong(matchResult.getMetadata().get("modelId"));
+            long modelId = Long.parseLong(modelIdStr);
-
             schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId);
             if (schemaElement == null) {
                 continue;

chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java

@@ -1,67 +0,0 @@
-package com.tencent.supersonic.chat.mapper;
-
-import com.hankcs.hanlp.seg.common.Term;
-import com.tencent.supersonic.chat.api.pojo.QueryContext;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
-import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
-import com.tencent.supersonic.common.util.ContextUtils;
-import com.tencent.supersonic.knowledge.dictionary.FuzzyResult;
-import com.tencent.supersonic.knowledge.utils.HanlpHelper;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.util.CollectionUtils;
-
-/***
- * A mapper capable of fuzzy parsing of metric names and dimension names.
- */
-@Slf4j
-public class FuzzyNameMapper extends BaseMapper {
-
-    @Override
-    public void doMap(QueryContext queryContext) {
-
-        List<Term> terms = HanlpHelper.getTerms(queryContext.getRequest().getQueryText());
-
-        FuzzyNameMatchStrategy fuzzyNameMatchStrategy = ContextUtils.getBean(FuzzyNameMatchStrategy.class);
-
-        MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
-
-        List<FuzzyResult> matches = fuzzyNameMatchStrategy.getMatches(queryContext, terms);
-
-        for (FuzzyResult match : matches) {
-            SchemaElement schemaElement = match.getSchemaElement();
-            Set<Long> regElementSet = getRegElementSet(queryContext.getMapInfo(), schemaElement);
-            if (regElementSet.contains(schemaElement.getId())) {
-                continue;
-            }
-            SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
-                    .element(schemaElement)
-                    .word(schemaElement.getName())
-                    .detectWord(match.getDetectWord())
-                    .frequency(10000L)
-                    .similarity(mapperHelper.getSimilarity(match.getDetectWord(), schemaElement.getName()))
-                    .build();
-            log.info("add to schema, elementMatch {}", schemaElementMatch);
-            addToSchemaMap(queryContext.getMapInfo(), schemaElement.getModel(), schemaElementMatch);
-        }
-    }
-
-    private Set<Long> getRegElementSet(SchemaMapInfo schemaMap, SchemaElement schemaElement) {
-        List<SchemaElementMatch> elements = schemaMap.getMatchedElements(schemaElement.getModel());
-        if (CollectionUtils.isEmpty(elements)) {
-            return new HashSet<>();
-        }
-        return elements.stream()
-                .filter(elementMatch ->
-                        SchemaElementType.METRIC.equals(elementMatch.getElement().getType())
-                                || SchemaElementType.DIMENSION.equals(elementMatch.getElement().getType()))
-                .map(elementMatch -> elementMatch.getElement().getId())
-                .collect(Collectors.toSet());
-    }
-
-}

chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java

@@ -1,84 +0,0 @@
-package com.tencent.supersonic.chat.mapper;
-
-import com.hankcs.hanlp.seg.common.Term;
-import com.tencent.supersonic.chat.api.pojo.QueryContext;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
-import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
-import com.tencent.supersonic.common.util.ContextUtils;
-import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
-import com.tencent.supersonic.knowledge.utils.HanlpHelper;
-import com.tencent.supersonic.knowledge.utils.NatureHelper;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections.CollectionUtils;
-
-/***
- * A mapper capable of prefix and suffix similarity parsing for
- * domain names, dimension values, metric names, and dimension names.
- */
-@Slf4j
-public class HanlpDictMapper extends BaseMapper {
-
-    @Override
-    public void doMap(QueryContext queryContext) {
-
-        String queryText = queryContext.getRequest().getQueryText();
-        List<Term> terms = HanlpHelper.getTerms(queryText);
-
-        HanlpDictMatchStrategy matchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class);
-
-        List<HanlpMapResult> matches = matchStrategy.getMatches(queryContext, terms);
-
-        HanlpHelper.transLetterOriginal(matches);
-
-        convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo(), terms);
-    }
-
-    private void convertTermsToSchemaMapInfo(List<HanlpMapResult> hanlpMapResults, SchemaMapInfo schemaMap,
-            List<Term> terms) {
-        if (CollectionUtils.isEmpty(hanlpMapResults)) {
-            return;
-        }
-
-        Map<String, Long> wordNatureToFrequency = terms.stream().collect(
-                Collectors.toMap(entry -> entry.getWord() + entry.getNature(),
-                        term -> Long.valueOf(term.getFrequency()), (value1, value2) -> value2));
-
-        for (HanlpMapResult hanlpMapResult : hanlpMapResults) {
-            for (String nature : hanlpMapResult.getNatures()) {
-                Long modelId = NatureHelper.getModelId(nature);
-                if (Objects.isNull(modelId)) {
-                    continue;
-                }
-                SchemaElementType elementType = NatureHelper.convertToElementType(nature);
-                if (Objects.isNull(elementType)) {
-                    continue;
-                }
-                Long elementID = NatureHelper.getElementID(nature);
-                SchemaElement element = getSchemaElement(modelId, elementType, elementID);
-                if (element == null) {
-                    continue;
-                }
-                if (element.getType().equals(SchemaElementType.VALUE)) {
-                    element.setName(hanlpMapResult.getName());
-                }
-                Long frequency = wordNatureToFrequency.get(hanlpMapResult.getName() + nature);
-                SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
-                        .element(element)
-                        .frequency(frequency)
-                        .word(hanlpMapResult.getName())
-                        .similarity(hanlpMapResult.getSimilarity())
-                        .detectWord(hanlpMapResult.getDetectWord())
-                        .build();
-
-                addToSchemaMap(schemaMap, modelId, schemaElementMatch);
-            }
-        }
-    }
-
-}

chat/core/src/main/java/com/tencent/supersonic/chat/mapper/KeywordMapper.java

@@ -0,0 +1,122 @@
+package com.tencent.supersonic.chat.mapper;
+
+import com.hankcs.hanlp.seg.common.Term;
+import com.tencent.supersonic.chat.api.pojo.QueryContext;
+import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
+import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
+import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.knowledge.dictionary.DatabaseMapResult;
+import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
+import com.tencent.supersonic.knowledge.utils.HanlpHelper;
+import com.tencent.supersonic.knowledge.utils.NatureHelper;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.CollectionUtils;
+
+/***
+ * A mapper that recognize elements through keyword.
+ * It includes two recognition strategies: HanlpDictMatchStrategy and DatabaseMatchStrategy.
+ *
+ */
+@Slf4j
+public class KeywordMapper extends BaseMapper {
+
+    @Override
+    public void doMap(QueryContext queryContext) {
+        String queryText = queryContext.getRequest().getQueryText();
+        //1.hanlpDict Match
+        List<Term> terms = HanlpHelper.getTerms(queryText);
+        HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class);
+
+        List<HanlpMapResult> hanlpMapResults = hanlpMatchStrategy.getMatches(queryContext, terms);
+        convertHanlpMapResultToMapInfo(hanlpMapResults, queryContext.getMapInfo(), terms);
+
+        //2.database Match
+        DatabaseMatchStrategy databaseMatchStrategy = ContextUtils.getBean(DatabaseMatchStrategy.class);
+
+        List<DatabaseMapResult> databaseResults = databaseMatchStrategy.getMatches(queryContext, terms);
+        convertDatabaseMapResultToMapInfo(queryContext, databaseResults);
+    }
+
+    private void convertHanlpMapResultToMapInfo(List<HanlpMapResult> mapResults, SchemaMapInfo schemaMap,
+            List<Term> terms) {
+        if (CollectionUtils.isEmpty(mapResults)) {
+            return;
+        }
+        HanlpHelper.transLetterOriginal(mapResults);
+        Map<String, Long> wordNatureToFrequency = terms.stream().collect(
+                Collectors.toMap(entry -> entry.getWord() + entry.getNature(),
+                        term -> Long.valueOf(term.getFrequency()), (value1, value2) -> value2));
+
+        for (HanlpMapResult hanlpMapResult : mapResults) {
+            for (String nature : hanlpMapResult.getNatures()) {
+                Long modelId = NatureHelper.getModelId(nature);
+                if (Objects.isNull(modelId)) {
+                    continue;
+                }
+                SchemaElementType elementType = NatureHelper.convertToElementType(nature);
+                if (Objects.isNull(elementType)) {
+                    continue;
+                }
+                Long elementID = NatureHelper.getElementID(nature);
+                SchemaElement element = getSchemaElement(modelId, elementType, elementID);
+                if (element == null) {
+                    continue;
+                }
+                if (element.getType().equals(SchemaElementType.VALUE)) {
+                    element.setName(hanlpMapResult.getName());
+                }
+                Long frequency = wordNatureToFrequency.get(hanlpMapResult.getName() + nature);
+                SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
+                        .element(element)
+                        .frequency(frequency)
+                        .word(hanlpMapResult.getName())
+                        .similarity(hanlpMapResult.getSimilarity())
+                        .detectWord(hanlpMapResult.getDetectWord())
+                        .build();
+
+                addToSchemaMap(schemaMap, modelId, schemaElementMatch);
+            }
+        }
+    }
+
+    private void convertDatabaseMapResultToMapInfo(QueryContext queryContext, List<DatabaseMapResult> mapResults) {
+        MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
+        for (DatabaseMapResult match : mapResults) {
+            SchemaElement schemaElement = match.getSchemaElement();
+            Set<Long> regElementSet = getRegElementSet(queryContext.getMapInfo(), schemaElement);
+            if (regElementSet.contains(schemaElement.getId())) {
+                continue;
+            }
+            SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
+                    .element(schemaElement)
+                    .word(schemaElement.getName())
+                    .detectWord(match.getDetectWord())
+                    .frequency(10000L)
+                    .similarity(mapperHelper.getSimilarity(match.getDetectWord(), schemaElement.getName()))
+                    .build();
+            log.info("add to schema, elementMatch {}", schemaElementMatch);
+            addToSchemaMap(queryContext.getMapInfo(), schemaElement.getModel(), schemaElementMatch);
+        }
+    }
+
+    private Set<Long> getRegElementSet(SchemaMapInfo schemaMap, SchemaElement schemaElement) {
+        List<SchemaElementMatch> elements = schemaMap.getMatchedElements(schemaElement.getModel());
+        if (CollectionUtils.isEmpty(elements)) {
+            return new HashSet<>();
+        }
+        return elements.stream()
+                .filter(elementMatch ->
+                        SchemaElementType.METRIC.equals(elementMatch.getElement().getType())
+                                || SchemaElementType.DIMENSION.equals(elementMatch.getElement().getType()))
+                .map(elementMatch -> elementMatch.getElement().getId())
+                .collect(Collectors.toSet());
+    }
+}

chat/core/src/test/java/com/tencent/supersonic/chat/mapper/HanlpDictMapperTest.java

@@ -1,22 +0,0 @@
-package com.tencent.supersonic.chat.mapper;
-
-import com.tencent.supersonic.chat.api.pojo.QueryContext;
-import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
-import com.tencent.supersonic.chat.test.context.ContextTest;
-import org.junit.jupiter.api.Test;
-
-/**
- * HanlpDictMapperTest
- */
-class HanlpDictMapperTest extends ContextTest {
-
-    @Test
-    void map() {
-        QueryReq queryRequest = new QueryReq();
-        queryRequest.setChatId(1);
-        queryRequest.setModelId(2L);
-        queryRequest.setQueryText("supersonic按部门访问次数");
-        HanlpDictMapper hanlpDictMapper = new HanlpDictMapper();
-        hanlpDictMapper.map(new QueryContext(queryRequest));
-    }
-}

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DatabaseMapResult.java

@@ -7,7 +7,7 @@ import lombok.ToString;
 
 @Data
 @ToString
-public class FuzzyResult extends MapResult {
+public class DatabaseMapResult extends MapResult {
 
     private SchemaElement schemaElement;
 
@@ -19,7 +19,7 @@ public class FuzzyResult extends MapResult {
         if (o == null || getClass() != o.getClass()) {
             return false;
         }
-        FuzzyResult that = (FuzzyResult) o;
+        DatabaseMapResult that = (DatabaseMapResult) o;
         return Objects.equal(name, that.name) && Objects.equal(schemaElement, that.schemaElement);
     }
 

launchers/chat/src/main/resources/META-INF/spring.factories

@@ -1,7 +1,6 @@
 com.tencent.supersonic.chat.api.component.SchemaMapper=\
     com.tencent.supersonic.chat.mapper.EmbeddingMapper, \
-    com.tencent.supersonic.chat.mapper.HanlpDictMapper, \
+    com.tencent.supersonic.chat.mapper.KeywordMapper, \
-    com.tencent.supersonic.chat.mapper.FuzzyNameMapper, \
     com.tencent.supersonic.chat.mapper.QueryFilterMapper, \
     com.tencent.supersonic.chat.mapper.EntityMapper
 

launchers/standalone/src/main/resources/META-INF/spring.factories

@@ -1,7 +1,6 @@
 com.tencent.supersonic.chat.api.component.SchemaMapper=\
     com.tencent.supersonic.chat.mapper.EmbeddingMapper, \
-    com.tencent.supersonic.chat.mapper.HanlpDictMapper, \
+    com.tencent.supersonic.chat.mapper.KeywordMapper, \
-    com.tencent.supersonic.chat.mapper.FuzzyNameMapper, \
     com.tencent.supersonic.chat.mapper.QueryFilterMapper, \
     com.tencent.supersonic.chat.mapper.EntityMapper, \
     com.tencent.supersonic.chat.mapper.ModelClusterMapper

launchers/standalone/src/test/resources/META-INF/spring.factories

@@ -1,7 +1,6 @@
 com.tencent.supersonic.chat.api.component.SchemaMapper=\
     com.tencent.supersonic.chat.mapper.EmbeddingMapper, \
-    com.tencent.supersonic.chat.mapper.HanlpDictMapper, \
+    com.tencent.supersonic.chat.mapper.KeywordMapper, \
-    com.tencent.supersonic.chat.mapper.FuzzyNameMapper, \
     com.tencent.supersonic.chat.mapper.QueryFilterMapper, \
     com.tencent.supersonic.chat.mapper.EntityMapper, \
     com.tencent.supersonic.chat.mapper.ModelClusterMapper