topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 20:51:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) when added to SchemaMap, duplicate elements are removed (#365)

Browse Source
This commit is contained in:
lexluo09
2023-11-10 21:47:20 +08:00
committed by GitHub
parent 3241ef87a3
commit 24b0be7566
4 changed files with 27 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java
View File

@@ -41,13 +41,13 @@ public class SchemaElement implements Serializable {
SchemaElement schemaElement = (SchemaElement) o; SchemaElement schemaElement = (SchemaElement) o;
return Objects.equal(model, schemaElement.model) && Objects.equal(id, return Objects.equal(model, schemaElement.model) && Objects.equal(id,
schemaElement.id) && Objects.equal(name, schemaElement.name) schemaElement.id) && Objects.equal(name, schemaElement.name)
&& Objects.equal(bizName, schemaElement.bizName) && Objects.equal( && Objects.equal(bizName, schemaElement.bizName)
useCnt, schemaElement.useCnt) && Objects.equal(type, schemaElement.type); && Objects.equal(type, schemaElement.type);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hashCode(model, id, name, bizName, useCnt, type); return Objects.hashCode(model, id, name, bizName, type);
} }
} }

23
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/BaseMapper.java
View File

@@ -13,6 +13,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@@ -44,13 +45,31 @@ public abstract class BaseMapper implements SchemaMapper {
public abstract void doMap(QueryContext queryContext); public abstract void doMap(QueryContext queryContext);
public void addToSchemaMap(SchemaMapInfo schemaMap, Long modelId, SchemaElementMatch schemaElementMatch) { public void addToSchemaMap(SchemaMapInfo schemaMap, Long modelId, SchemaElementMatch newElementMatch) {
Map<Long, List<SchemaElementMatch>> modelElementMatches = schemaMap.getModelElementMatches(); Map<Long, List<SchemaElementMatch>> modelElementMatches = schemaMap.getModelElementMatches();
List<SchemaElementMatch> schemaElementMatches = modelElementMatches.putIfAbsent(modelId, new ArrayList<>()); List<SchemaElementMatch> schemaElementMatches = modelElementMatches.putIfAbsent(modelId, new ArrayList<>());
if (schemaElementMatches == null) { if (schemaElementMatches == null) {
schemaElementMatches = modelElementMatches.get(modelId); schemaElementMatches = modelElementMatches.get(modelId);
} }
schemaElementMatches.add(schemaElementMatch); //remove duplication
AtomicBoolean needAddNew = new AtomicBoolean(true);
schemaElementMatches.removeIf(
existElementMatch -> {
SchemaElement existElement = existElementMatch.getElement();
SchemaElement newElement = newElementMatch.getElement();
if (existElement.equals(newElement)) {
if (newElementMatch.getSimilarity() > existElementMatch.getSimilarity()) {
return true;
} else {
needAddNew.set(false);
}
}
return false;
}
);
if (needAddNew.get()) {
schemaElementMatches.add(newElementMatch);
}
} }
public SchemaElement getSchemaElement(Long modelId, SchemaElementType elementType, Long elementID) { public SchemaElement getSchemaElement(Long modelId, SchemaElementType elementType, Long elementID) {

8
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java
View File

@@ -33,18 +33,16 @@ public class EmbeddingMapper extends BaseMapper {
HanlpHelper.transLetterOriginal(matchResults); HanlpHelper.transLetterOriginal(matchResults);
//2. build SchemaElementMatch by info //2. build SchemaElementMatch by info
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
for (EmbeddingResult matchResult : matchResults) { for (EmbeddingResult matchResult : matchResults) {
Long elementId = Retrieval.getLongId(matchResult.getId()); Long elementId = Retrieval.getLongId(matchResult.getId());
SchemaElement schemaElement = JSONObject.parseObject(JSONObject.toJSONString(matchResult.getMetadata()), SchemaElement schemaElement = JSONObject.parseObject(JSONObject.toJSONString(matchResult.getMetadata()),
SchemaElement.class); SchemaElement.class);
String modelIdStr = matchResult.getMetadata().get("modelId"); if (StringUtils.isBlank(matchResult.getMetadata().get("modelId"))) {
if (StringUtils.isBlank(modelIdStr)) {
continue; continue;
} }
long modelId = Long.parseLong(modelIdStr); long modelId = Long.parseLong(matchResult.getMetadata().get("modelId"));
schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId); schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId);
if (schemaElement == null) { if (schemaElement == null) {
@@ -54,7 +52,7 @@ public class EmbeddingMapper extends BaseMapper {
.element(schemaElement) .element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY) .frequency(BaseWordBuilder.DEFAULT_FREQUENCY)
.word(matchResult.getName()) .word(matchResult.getName())
.similarity(mapperHelper.getSimilarity(matchResult.getName(), matchResult.getDetectWord())) .similarity(1 - matchResult.getDistance())
.detectWord(matchResult.getDetectWord()) .detectWord(matchResult.getDetectWord())
.build(); .build();
//3. add to mapInfo //3. add to mapInfo

1
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -131,7 +131,6 @@ public class QueryServiceImpl implements QueryService {
mapper.map(queryCtx); mapper.map(queryCtx);
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime)) timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(mapper.getClass().getSimpleName()).type(CostType.MAPPER.getType()).build()); .interfaceName(mapper.getClass().getSimpleName()).type(CostType.MAPPER.getType()).build());
log.info("{} result:{}", mapper.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
}); });
//3. parser //3. parser