topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Chat)Integration of tags into chat. (#798)

Browse Source
This commit is contained in:
lexluo09
2024-03-08 16:59:51 +08:00
committed by GitHub
parent 9d27031543
commit 3ad18b0ee0
20 changed files with 108 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataSetSchema.java
View File

@@ -19,6 +19,7 @@ public class DataSetSchema {
private Set<SchemaElement> dimensions = new HashSet<>();
private Set<SchemaElement> dimensionValues = new HashSet<>();
private Set<SchemaElement> tags = new HashSet<>();
private Set<SchemaElement> tagValues = new HashSet<>();
private SchemaElement entity = new SchemaElement();
private QueryConfig queryConfig;
@@ -44,34 +45,8 @@ public class DataSetSchema {
case TAG:
element = tags.stream().filter(e -> e.getId() == elementID).findFirst();
break;
default:
}
if (element.isPresent()) {
return element.get();
} else {
return null;
}
}
public SchemaElement getElement(SchemaElementType elementType, String name) {
Optional<SchemaElement> element = Optional.empty();
switch (elementType) {
case ENTITY:
element = Optional.ofNullable(entity);
break;
case DATASET:
element = Optional.of(dataSet);
break;
case METRIC:
element = metrics.stream().filter(e -> name.equals(e.getName())).findFirst();
break;
case DIMENSION:
element = dimensions.stream().filter(e -> name.equals(e.getName())).findFirst();
break;
case VALUE:
element = dimensionValues.stream().filter(e -> name.equals(e.getName())).findFirst();
case TAG_VALUE:
element = tagValues.stream().filter(e -> e.getId() == elementID).findFirst();
break;
default:
}

17
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java
View File

@@ -46,6 +46,9 @@ public class SemanticSchema implements Serializable {
case TAG:
element = getElementsById(elementID, getTags());
break;
case TAG_VALUE:
element = getElementsById(elementID, getTagValues());
break;
default:
}
@@ -91,13 +94,21 @@ public class SemanticSchema implements Serializable {
}
public List<SchemaElement> getTags(Long dataSetId) {
List<SchemaElement> tags = getTags();
return getElementsByDataSetId(dataSetId, tags);
}
public List<SchemaElement> getTagValues() {
List<SchemaElement> tags = new ArrayList<>();
dataSetSchemaList.stream().filter(schemaElement ->
dataSetId.equals(schemaElement.getDataSet().getDataSet()))
.forEach(d -> tags.addAll(d.getTags()));
dataSetSchemaList.stream().forEach(d -> tags.addAll(d.getTagValues()));
return tags;
}
public List<SchemaElement> getTagValues(Long dataSetId) {
List<SchemaElement> tags = getTagValues();
return getElementsByDataSetId(dataSetId, tags);
}
public List<SchemaElement> getMetrics() {
List<SchemaElement> metrics = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics()));

7
chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java
View File

@@ -90,7 +90,12 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy<EmbeddingResult> {
.map(retrieveQueryResult -> {
List<Retrieval> retrievals = retrieveQueryResult.getRetrieval();
if (CollectionUtils.isNotEmpty(retrievals)) {
retrievals.removeIf(retrieval -> retrieval.getDistance() > distance.doubleValue());
retrievals.removeIf(retrieval -> {
if (!retrieveQueryResult.getQuery().contains(retrieval.getQuery())) {
return retrieval.getDistance() > distance.doubleValue();
}
return false;
});
}
return retrieveQueryResult;
})

3
chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java
View File

@@ -73,7 +73,8 @@ public class KeywordMapper extends BaseMapper {
if (element == null) {
continue;
}
if (element.getType().equals(SchemaElementType.VALUE)) {
if (element.getType().equals(SchemaElementType.VALUE) || element.getType()
.equals(SchemaElementType.TAG_VALUE)) {
element.setName(hanlpMapResult.getName());
}
Long frequency = wordNatureToFrequency.get(hanlpMapResult.getName() + nature);

2
chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java
View File

@@ -64,7 +64,7 @@ public class SearchMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
// remove entity name where search
hanlpMapResults = hanlpMapResults.stream().filter(entry -> {
List<String> natures = entry.getNatures().stream()
.filter(nature -> !nature.endsWith(DictWordType.ENTITY.getType()))
.filter(nature -> !nature.endsWith(DictWordType.ENTITY.getTypeWithSpilt()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(natures)) {
return false;

19
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java
View File

@@ -51,6 +51,9 @@ public class DataSetSchemaBuilder {
Set<SchemaElement> tags = getTags(resp);
dataSetSchema.getTags().addAll(tags);
Set<SchemaElement> tagValues = getTagValues(resp);
dataSetSchema.getTagValues().addAll(tagValues);
SchemaElement entity = getEntity(resp);
if (Objects.nonNull(entity)) {
dataSetSchema.setEntity(entity);
@@ -91,6 +94,22 @@ public class DataSetSchemaBuilder {
return tags;
}
private static Set<SchemaElement> getTagValues(DataSetSchemaResp resp) {
Set<SchemaElement> dimensionValues = new HashSet<>();
for (TagResp tagResp : resp.getTags()) {
SchemaElement element = SchemaElement.builder()
.dataSet(resp.getId())
.model(tagResp.getModelId())
.id(tagResp.getId())
.name(tagResp.getName())
.bizName(tagResp.getBizName())
.type(SchemaElementType.TAG_VALUE)
.build();
dimensionValues.add(element);
}
return dimensionValues;
}
private static Set<SchemaElement> getDimensions(DataSetSchemaResp resp) {
Set<SchemaElement> dimensions = new HashSet<>();
for (DimSchemaResp dim : resp.getDimensions()) {

3
chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java
View File

@@ -32,7 +32,8 @@ public class SchemaDictUpdateListener implements ApplicationListener<DataEvent>
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());
String sign = DictWordType.NATURE_SPILT;
String nature = sign + 1 + sign + dataItem.getId() + dataItem.getType().name().toLowerCase();
String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);

1
common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java
View File

@@ -15,6 +15,7 @@ public class Constants {
public static final String COLON = ":";
public static final String MINUS = "-";
public static final String UNDERLINE = "_";
public static final String DICT_VALUE = "v";
public static final String UNDERLINE_DOUBLE = "__";
public static final String PARENTHESES_START = "(";
public static final String PARENTHESES_END = ")";

36
common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.pojo.enums;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
/***
@@ -12,15 +13,17 @@ public enum DictWordType {
DIMENSION("dimension"),
VALUE("v"),
VALUE("dv"),
VIEW("view"),
DATASET("dataset"),
ENTITY("entity"),
NUMBER("m"),
TAG("t"),
TAG("tag"),
TAG_VALUE("tv"),
SUFFIX("suffix");
@@ -32,7 +35,7 @@ public enum DictWordType {
this.type = type;
}
public String getType() {
public String getTypeWithSpilt() {
return NATURE_SPILT + type;
}
@@ -41,19 +44,36 @@ public enum DictWordType {
return null;
}
for (DictWordType dictWordType : values()) {
if (nature.endsWith(dictWordType.getType())) {
if (nature.endsWith(dictWordType.getTypeWithSpilt())) {
return dictWordType;
}
}
//view
//dataSet
String[] natures = nature.split(DictWordType.NATURE_SPILT);
if (natures.length == 2 && StringUtils.isNumeric(natures[1])) {
return VIEW;
return DATASET;
}
//dimension value
if (natures.length == 3 && StringUtils.isNumeric(natures[1]) && StringUtils.isNumeric(natures[2])) {
if (natures.length >= 3 && StringUtils.isNumeric(natures[1]) && StringUtils.isNumeric(natures[2])) {
return VALUE;
}
return null;
}
public static DictWordType of(TypeEnums type) {
for (DictWordType wordType : DictWordType.values()) {
if (wordType.name().equalsIgnoreCase(type.name())) {
return wordType;
}
}
return null;
}
public static String getSuffixNature(TypeEnums type) {
DictWordType wordType = of(type);
if (Objects.nonNull(wordType)) {
return wordType.type;
}
return "";
}
}

9
common/src/main/java/com/tencent/supersonic/common/util/JsonUtil.java
View File

@@ -13,11 +13,6 @@ import com.fasterxml.jackson.databind.type.CollectionType;
import com.fasterxml.jackson.databind.type.MapType;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.InvalidParameterException;
@@ -25,6 +20,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@Slf4j
@@ -32,7 +30,6 @@ public class JsonUtil {
public static final JsonUtil INSTANCE = new JsonUtil();
@Setter
@Getter
private final ObjectMapper objectMapper = new ObjectMapper();

1
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java
View File

@@ -7,6 +7,7 @@ public enum SchemaElementType {
VALUE,
ENTITY,
TAG,
TAG_VALUE,
ID,
DATE
}

6
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.api.pojo.response;
import static com.tencent.supersonic.common.pojo.Constants.DICT_VALUE;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
@@ -10,6 +11,7 @@ import lombok.Data;
@Data
public class DictItemResp {
private Long id;
private Long modelId;
@@ -31,7 +33,9 @@ public class DictItemResp {
private StatusEnum status;
public String getNature() {
return UNDERLINE + modelId + UNDERLINE + itemId + UNDERLINE + type.name().toLowerCase().substring(0, 1);
return UNDERLINE + modelId + UNDERLINE + itemId + UNDERLINE + type.name().toLowerCase().substring(0, 1)
+ DICT_VALUE;
}
public String fetchDictFileName() {

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java
View File

@@ -82,7 +82,7 @@ public class SearchService {
entry -> {
String name = entry.getKey().replace("#", " ");
List<String> natures = entry.getValue().stream()
.map(nature -> nature.replaceAll(DictWordType.SUFFIX.getType(), ""))
.map(nature -> nature.replaceAll(DictWordType.SUFFIX.getTypeWithSpilt(), ""))
.collect(Collectors.toList());
name = StringUtils.reverse(name);
return new HanlpMapResult(name, natures, key);
@@ -160,8 +160,8 @@ public class SearchService {
if (Objects.nonNull(natures) && natures.length > 0) {
trie.put(dictWord.getWord(), getValue(natures));
}
if (dictWord.getNature().contains(DictWordType.METRIC.getType()) || dictWord.getNature()
.contains(DictWordType.DIMENSION.getType())) {
if (dictWord.getNature().contains(DictWordType.METRIC.getTypeWithSpilt()) || dictWord.getNature()
.contains(DictWordType.DIMENSION.getTypeWithSpilt())) {
suffixTrie.remove(dictWord.getWord());
}
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java
View File

@@ -31,10 +31,10 @@ public class DimensionWordBuilder extends BaseWordWithAliasBuilder {
dictWord.setWord(word);
Long modelId = schemaElement.getModel();
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.DIMENSION.getType();
+ DictWordType.DIMENSION.getTypeWithSpilt();
if (isSuffix) {
nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.SUFFIX.getType() + DictWordType.DIMENSION.getType();
+ DictWordType.SUFFIX.getTypeWithSpilt() + DictWordType.DIMENSION.getTypeWithSpilt();
}
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));
return dictWord;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java
View File

@@ -29,7 +29,7 @@ public class EntityWordBuilder extends BaseWordWithAliasBuilder {
@Override
public DictWord getOneWordNature(String word, SchemaElement schemaElement, boolean isSuffix) {
String nature = DictWordType.NATURE_SPILT + schemaElement.getModel()
+ DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.ENTITY.getType();
+ DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.ENTITY.getTypeWithSpilt();
DictWord dictWord = new DictWord();
dictWord.setWord(word);
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY * 2, nature));

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java
View File

@@ -31,10 +31,10 @@ public class MetricWordBuilder extends BaseWordWithAliasBuilder {
dictWord.setWord(word);
Long modelId = schemaElement.getModel();
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.METRIC.getType();
+ DictWordType.METRIC.getTypeWithSpilt();
if (isSuffix) {
nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.SUFFIX.getType() + DictWordType.METRIC.getType();
+ DictWordType.SUFFIX.getTypeWithSpilt() + DictWordType.METRIC.getTypeWithSpilt();
}
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));
return dictWord;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/TagWordBuilder.java
View File

@@ -28,10 +28,10 @@ public class TagWordBuilder extends BaseWordWithAliasBuilder {
dictWord.setWord(word);
Long modelId = schemaElement.getModel();
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.TAG.getType();
+ DictWordType.TAG.getTypeWithSpilt();
if (isSuffix) {
nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.SUFFIX.getType() + DictWordType.TAG.getType();
+ DictWordType.SUFFIX.getTypeWithSpilt() + DictWordType.TAG.getTypeWithSpilt();
}
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));
return dictWord;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/WordBuilderFactory.java
View File

@@ -15,7 +15,7 @@ public class WordBuilderFactory {
static {
wordNatures.put(DictWordType.DIMENSION, new DimensionWordBuilder());
wordNatures.put(DictWordType.METRIC, new MetricWordBuilder());
wordNatures.put(DictWordType.VIEW, new ModelWordBuilder());
wordNatures.put(DictWordType.DATASET, new ModelWordBuilder());
wordNatures.put(DictWordType.ENTITY, new EntityWordBuilder());
wordNatures.put(DictWordType.VALUE, new ValueWordBuilder());
wordNatures.put(DictWordType.TAG, new TagWordBuilder());

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java
View File

@@ -40,7 +40,7 @@ public class NatureHelper {
case ENTITY:
result = SchemaElementType.ENTITY;
break;
case VIEW:
case DATASET:
result = SchemaElementType.DATASET;
break;
case VALUE:
@@ -49,6 +49,9 @@ public class NatureHelper {
case TAG:
result = SchemaElementType.TAG;
break;
case TAG_VALUE:
result = SchemaElementType.TAG_VALUE;
break;
default:
break;
}
@@ -57,7 +60,7 @@ public class NatureHelper {
private static boolean isDataSetOrEntity(S2Term term, Integer model) {
return (DictWordType.NATURE_SPILT + model).equals(term.nature.toString()) || term.nature.toString()
.endsWith(DictWordType.ENTITY.getType());
.endsWith(DictWordType.ENTITY.getTypeWithSpilt());
}
public static Integer getDataSetByNature(Nature nature) {
@@ -131,7 +134,8 @@ public class NatureHelper {
if (split.length <= 1) {
return false;
}
return !nature.endsWith(DictWordType.METRIC.getType()) && !nature.endsWith(DictWordType.DIMENSION.getType())
return !nature.endsWith(DictWordType.METRIC.getTypeWithSpilt()) && !nature.endsWith(
DictWordType.DIMENSION.getTypeWithSpilt())
&& StringUtils.isNumeric(split[1]);
}
@@ -154,12 +158,12 @@ public class NatureHelper {
private static long getDimensionCount(List<S2Term> terms) {
return terms.stream().filter(term -> term.nature.startsWith(DictWordType.NATURE_SPILT) && term.nature.toString()
.endsWith(DictWordType.DIMENSION.getType())).count();
.endsWith(DictWordType.DIMENSION.getTypeWithSpilt())).count();
}
private static long getMetricCount(List<S2Term> terms) {
return terms.stream().filter(term -> term.nature.startsWith(DictWordType.NATURE_SPILT) && term.nature.toString()
.endsWith(DictWordType.METRIC.getType())).count();
.endsWith(DictWordType.METRIC.getTypeWithSpilt())).count();
}
/**

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java
View File

@@ -31,7 +31,7 @@ public class KnowledgeServiceImpl implements KnowledgeService {
public void updateSemanticKnowledge(List<DictWord> natures) {
List<DictWord> prefixes = natures.stream()
.filter(entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType()))
.filter(entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getTypeWithSpilt()))
.collect(Collectors.toList());
for (DictWord nature : prefixes) {
@@ -39,7 +39,7 @@ public class KnowledgeServiceImpl implements KnowledgeService {
}
List<DictWord> suffixes = natures.stream()
.filter(entry -> entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType()))
.filter(entry -> entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getTypeWithSpilt()))
.collect(Collectors.toList());
SearchService.loadSuffix(suffixes);
@@ -80,7 +80,7 @@ public class KnowledgeServiceImpl implements KnowledgeService {
}
public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
Map<Long, List<Long>> modelIdToDataSetIds) {
Map<Long, List<Long>> modelIdToDataSetIds) {
return SearchService.prefixSearch(key, limit, modelIdToDataSetIds);
}