topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement) optimize schema data change monitoring (#333)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-06 22:11:56 +08:00
committed by GitHub
parent 6e3f871015
commit aa6c658a9a
14 changed files with 101 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java
View File

@@ -28,7 +28,7 @@ public class SchemaElement implements Serializable {
private String defaultAgg;
private int order;
private double order;
@Override
public boolean equals(Object o) {

6
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java
View File

@@ -54,7 +54,11 @@ public class SemanticParseInfo {
@Override
public int compare(SchemaElement o1, SchemaElement o2) {
if (o1.getOrder() != o2.getOrder()) {
return o1.getOrder() - o2.getOrder();
if (o1.getOrder() < o2.getOrder()) {
return -1;
} else {
return 1;
}
}
int len1 = o1.getName().length();
int len2 = o2.getName().length();

4
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java
View File

@@ -47,7 +47,9 @@ public class EmbeddingMapper extends BaseMapper {
long modelId = Long.parseLong(modelIdStr);
schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId);
if (schemaElement == null) {
continue;
}
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY)

1
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java
View File

@@ -109,6 +109,7 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
for (SchemaElementMatch schemaMatch : parseInfo.getElementMatches()) {
SchemaElement element = schemaMatch.getElement();
element.setOrder(1 - schemaMatch.getSimilarity());
switch (element.getType()) {
case ID:
SchemaElement entityElement = modelSchema.getElement(SchemaElementType.ENTITY, element.getId());

3
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java
View File

@@ -230,6 +230,9 @@ public class ConfigServiceImpl implements ConfigService {
BeanUtils.copyProperties(chatConfigResp, chatConfigRich);
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
if (modelSchema == null) {
return chatConfigRich;
}
chatConfigRich.setBizName(modelSchema.getModel().getBizName());
chatConfigRich.setModelName(modelSchema.getModel().getName());

3
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java
View File

@@ -51,6 +51,9 @@ public class RecommendServiceImpl implements RecommendService {
return new RecommendResp();
}
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
if (Objects.isNull(modelSchema)) {
return new RecommendResp();
}
List<Long> drillDownDimensions = Lists.newArrayList();
Set<SchemaElement> metricElements = modelSchema.getMetrics();
if (recommendReq.getMetricId() != null && !CollectionUtils.isEmpty(metricElements)) {

10
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java → chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/SchemaDictUpdateListener.java
View File

@@ -5,21 +5,29 @@ import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.knowledge.dictionary.DictWord;
import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationListener;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
public class DictUpdateListener implements ApplicationListener<DataEvent> {
public class SchemaDictUpdateListener implements ApplicationListener<DataEvent> {
@Autowired
private SchemaService schemaService;
@Async
@Override
public void onApplicationEvent(DataEvent dataEvent) {
if (CollectionUtils.isEmpty(dataEvent.getDataItems())) {
return;
}
schemaService.getCache().invalidateAll();
dataEvent.getDataItems().forEach(dataItem -> {
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());

4
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/service/SchemaService.java
View File

@@ -18,11 +18,11 @@ public class SchemaService {
public static final String ALL_CACHE = "all";
private static final Integer META_CACHE_TIME = 2;
private static final Integer META_CACHE_TIME = 30;
private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
private LoadingCache<String, SemanticSchema> cache = CacheBuilder.newBuilder()
.expireAfterWrite(META_CACHE_TIME, TimeUnit.MINUTES)
.expireAfterWrite(META_CACHE_TIME, TimeUnit.SECONDS)
.build(
new CacheLoader<String, SemanticSchema>() {
@Override

17
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java
View File

@@ -230,6 +230,17 @@ public class DimensionServiceImpl implements DimensionService {
return dimensionResps;
}
@Override
public List<DataItem> getDataItems(Long modelId) {
DimensionFilter metaFilter = new DimensionFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
List<DimensionDO> dimensionDOS = queryDimension(metaFilter);
if (CollectionUtils.isEmpty(dimensionDOS)) {
return Lists.newArrayList();
}
return dimensionDOS.stream().map(this::getDataItem).collect(Collectors.toList());
}
@Override
public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
String mockAlias = chatGptHelper.mockAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(),
@@ -333,5 +344,11 @@ public class DimensionServiceImpl implements DimensionService {
Lists.newArrayList(dataItem), eventType));
}
private DataItem getDataItem(DimensionDO dimensionDO) {
return DataItem.builder().id(dimensionDO.getId()).name(dimensionDO.getName())
.bizName(dimensionDO.getBizName())
.modelId(dimensionDO.getModelId()).type(TypeEnums.DIMENSION).build();
}
}

17
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java
View File

@@ -43,7 +43,6 @@ import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -60,17 +59,18 @@ public class MetricServiceImpl implements MetricService {
private ChatGptHelper chatGptHelper;
@Autowired
private ApplicationEventPublisher eventPublisher;
public MetricServiceImpl(MetricRepository metricRepository,
ModelService modelService,
DomainService domainService,
ChatGptHelper chatGptHelper) {
ChatGptHelper chatGptHelper,
ApplicationEventPublisher eventPublisher) {
this.domainService = domainService;
this.metricRepository = metricRepository;
this.modelService = modelService;
this.chatGptHelper = chatGptHelper;
this.eventPublisher = eventPublisher;
}
@Override
@@ -276,6 +276,17 @@ public class MetricServiceImpl implements MetricService {
return modelResp.getDrillDownDimensions();
}
@Override
public List<DataItem> getDataItems(Long modelId) {
MetricFilter metaFilter = new MetricFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricDO> metricDOS = queryMetric(metaFilter);
if (CollectionUtils.isEmpty(metricDOS)) {
return Lists.newArrayList();
}
return metricDOS.stream().map(this::getDataItem).collect(Collectors.toList());
}
private void checkParam(MetricReq metricReq) {
MetricTypeParams typeParams = metricReq.getTypeParams();
List<Measure> measures = typeParams.getMeasures();

31
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java
View File

@@ -3,7 +3,9 @@ package com.tencent.supersonic.semantic.model.application;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
@@ -43,6 +45,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -59,12 +62,13 @@ public class ModelServiceImpl implements ModelService {
private final UserService userService;
private final DatabaseService databaseService;
private final Catalog catalog;
private ApplicationEventPublisher eventPublisher;
public ModelServiceImpl(ModelRepository modelRepository, @Lazy MetricService metricService,
@Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService,
@Lazy DomainService domainService, UserService userService,
@Lazy DatabaseService databaseService,
@Lazy Catalog catalog) {
@Lazy Catalog catalog, ApplicationEventPublisher eventPublisher) {
this.modelRepository = modelRepository;
this.metricService = metricService;
this.dimensionService = dimensionService;
@@ -73,6 +77,7 @@ public class ModelServiceImpl implements ModelService {
this.userService = userService;
this.databaseService = databaseService;
this.catalog = catalog;
this.eventPublisher = eventPublisher;
}
@Override
@@ -86,6 +91,7 @@ public class ModelServiceImpl implements ModelService {
public void updateModel(ModelReq modelReq, User user) {
ModelDO modelDO = getModelDO(modelReq.getId());
modelReq.updatedBy(user.getName());
int oldStatus = modelDO.getStatus();
BeanMapper.mapper(modelReq, modelDO);
if (modelReq.getEntity() != null) {
modelDO.setEntity(JsonUtil.toString(modelReq.getEntity()));
@@ -94,6 +100,29 @@ public class ModelServiceImpl implements ModelService {
modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions()));
}
modelRepository.updateModel(modelDO);
statusPublish(oldStatus, modelDO);
}
private void statusPublish(Integer oldStatus, ModelDO modelDO) {
if (oldStatus.equals(modelDO.getStatus())) {
return;
}
if (oldStatus.equals(StatusEnum.ONLINE.getCode())
&& modelDO.getStatus().equals(StatusEnum.OFFLINE.getCode())) {
publishEvent(EventType.DELETE, modelDO);
} else if (oldStatus.equals(StatusEnum.OFFLINE.getCode())
&& modelDO.getStatus().equals(StatusEnum.ONLINE.getCode())) {
publishEvent(EventType.ADD, modelDO);
}
}
private void publishEvent(EventType eventType, ModelDO modelDO) {
eventPublisher.publishEvent(
new DataEvent(this, metricService.getDataItems(modelDO.getId()),
eventType));
eventPublisher.publishEvent(
new DataEvent(this, dimensionService.getDataItems(modelDO.getId()),
eventType));
}
@Override

3
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.model.domain;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
@@ -31,6 +32,8 @@ public interface DimensionService {
void deleteDimension(Long id, User user);
List<DataItem> getDataItems(Long modelId);
List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user);
List<DimValueMap> mockDimensionValueAlias(DimensionReq dimensionReq, User user);

3
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.model.domain;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.MetricReq;
@@ -34,4 +35,6 @@ public interface MetricService {
Set<String> getMetricTags();
List<DrillDownDimension> getDrillDownDimension(Long metricId);
List<DataItem> getDataItems(Long modelId);
}

7
semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml
View File

@@ -151,6 +151,13 @@
#{model}
</foreach>
</if>
<if test="ids != null and ids.size >0">
and id in
<foreach collection="ids" index="index" item="id" open="(" close=")"
separator=",">
#{id}
</foreach>
</if>
<if test="createdBy != null">
and created_by = #{createdBy}
</if>