topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-15 22:46:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement) optimize schema data change monitoring (#333)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-06 22:11:56 +08:00
committed by GitHub
parent 6e3f871015
commit aa6c658a9a
14 changed files with 101 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java
View File

@@ -28,7 +28,7 @@ public class SchemaElement implements Serializable {
private String defaultAgg;
private int order;
private double order;
@Override
public boolean equals(Object o) {

6
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java
View File

@@ -54,7 +54,11 @@ public class SemanticParseInfo {
@Override
public int compare(SchemaElement o1, SchemaElement o2) {
if (o1.getOrder() != o2.getOrder()) {
return o1.getOrder() - o2.getOrder();
if (o1.getOrder() < o2.getOrder()) {
return -1;
} else {
return 1;
}
}
int len1 = o1.getName().length();
int len2 = o2.getName().length();

4
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java
View File

@@ -47,7 +47,9 @@ public class EmbeddingMapper extends BaseMapper {
long modelId = Long.parseLong(modelIdStr);
schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId);
if (schemaElement == null) {
continue;
}
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY)

1
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java
View File

@@ -109,6 +109,7 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
for (SchemaElementMatch schemaMatch : parseInfo.getElementMatches()) {
SchemaElement element = schemaMatch.getElement();
element.setOrder(1 - schemaMatch.getSimilarity());
switch (element.getType()) {
case ID:
SchemaElement entityElement = modelSchema.getElement(SchemaElementType.ENTITY, element.getId());

3
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java
View File

@@ -230,6 +230,9 @@ public class ConfigServiceImpl implements ConfigService {
BeanUtils.copyProperties(chatConfigResp, chatConfigRich);
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
if (modelSchema == null) {
return chatConfigRich;
}
chatConfigRich.setBizName(modelSchema.getModel().getBizName());
chatConfigRich.setModelName(modelSchema.getModel().getName());

3
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java
View File

@@ -51,6 +51,9 @@ public class RecommendServiceImpl implements RecommendService {
return new RecommendResp();
}
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
if (Objects.isNull(modelSchema)) {
return new RecommendResp();
}
List<Long> drillDownDimensions = Lists.newArrayList();
Set<SchemaElement> metricElements = modelSchema.getMetrics();
if (recommendReq.getMetricId() != null && !CollectionUtils.isEmpty(metricElements)) {

10
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java → chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/SchemaDictUpdateListener.java
View File

@@ -5,21 +5,29 @@ import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.knowledge.dictionary.DictWord;
import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationListener;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
public class DictUpdateListener implements ApplicationListener<DataEvent> {
public class SchemaDictUpdateListener implements ApplicationListener<DataEvent> {
@Autowired
private SchemaService schemaService;
@Async
@Override
public void onApplicationEvent(DataEvent dataEvent) {
if (CollectionUtils.isEmpty(dataEvent.getDataItems())) {
return;
}
schemaService.getCache().invalidateAll();
dataEvent.getDataItems().forEach(dataItem -> {
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());

4
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SchemaService.java
View File

@@ -18,11 +18,11 @@ public class SchemaService {
public static final String ALL_CACHE = "all";
private static final Integer META_CACHE_TIME = 2;
private static final Integer META_CACHE_TIME = 30;
private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
private LoadingCache<String, SemanticSchema> cache = CacheBuilder.newBuilder()
.expireAfterWrite(META_CACHE_TIME, TimeUnit.MINUTES)
.expireAfterWrite(META_CACHE_TIME, TimeUnit.SECONDS)
.build(
new CacheLoader<String, SemanticSchema>() {
@Override