mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-14 22:25:19 +00:00
[improvement][headless]Refactor headless infra to support advanced semantic modelling.
This commit is contained in:
jerryjzhang
@@ -27,7 +27,7 @@ public class SchemaDictUpdateListener implements ApplicationListener<DataEvent>
|
||||
dictWord.setWord(dataItem.getName());
|
||||
String sign = DictWordType.NATURE_SPILT;
|
||||
String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
|
||||
String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
|
||||
String nature = sign + dataItem.getModelId() + sign + dataItem.getId() + suffixNature;
|
||||
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
|
||||
dictWord.setNature(nature);
|
||||
dictWord.setNatureWithFrequency(natureWithFrequency);
|
||||
|
||||
@@ -47,7 +47,7 @@ public interface DimensionService {
|
||||
|
||||
void sendDimensionEventBatch(List<Long> modelIds, EventType eventType);
|
||||
|
||||
DataEvent getDataEvent();
|
||||
DataEvent getAllDataEvents();
|
||||
|
||||
Boolean updateDimValueAlias(DimValueAliasReq req, User user);
|
||||
}
|
||||
|
||||
@@ -6,12 +6,9 @@ import com.alibaba.fastjson.TypeReference;
|
||||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
||||
import com.github.pagehelper.PageHelper;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DataEvent;
|
||||
import com.tencent.supersonic.common.pojo.DataItem;
|
||||
import com.tencent.supersonic.common.pojo.ModelRela;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.*;
|
||||
import com.tencent.supersonic.common.pojo.enums.EventType;
|
||||
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
@@ -25,23 +22,14 @@ import com.tencent.supersonic.headless.api.pojo.request.DimValueAliasReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.*;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DataSetService;
|
||||
import com.tencent.supersonic.headless.server.service.DatabaseService;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelRelaService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.service.TagMetaService;
|
||||
import com.tencent.supersonic.headless.server.service.*;
|
||||
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
|
||||
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
|
||||
@@ -152,11 +140,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
DimensionConverter.convert(dimensionDO, dimensionReq);
|
||||
dimensionRepository.updateDimension(dimensionDO);
|
||||
if (!oldName.equals(dimensionDO.getName())) {
|
||||
sendEvent(
|
||||
DataItem.builder().modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
|
||||
.newName(dimensionReq.getName()).name(oldName).type(TypeEnums.DIMENSION)
|
||||
.id(dimensionDO.getId() + Constants.UNDERLINE).build(),
|
||||
EventType.UPDATE);
|
||||
sendEvent(getDataItem(dimensionDO), EventType.UPDATE);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,7 +408,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
eventPublisher.publishEvent(dataEvent);
|
||||
}
|
||||
|
||||
public DataEvent getDataEvent() {
|
||||
public DataEvent getAllDataEvents() {
|
||||
DimensionFilter dimensionFilter = new DimensionFilter();
|
||||
List<DimensionDO> dimensionDOS = queryDimension(dimensionFilter);
|
||||
return getDataEvent(dimensionDOS, EventType.ADD);
|
||||
@@ -464,13 +448,18 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
return true;
|
||||
}
|
||||
|
||||
private DataItem getDataItem(DimensionDO dimensionDO) {
|
||||
ModelResp modelResp = modelService.getModel(dimensionDO.getModelId());
|
||||
DimensionResp dimensionResp = DimensionConverter.convert2DimensionResp(dimensionDO,
|
||||
ImmutableMap.of(modelResp.getId(), modelResp));
|
||||
return DataItem.builder().id(dimensionResp.getId().toString()).name(dimensionResp.getName())
|
||||
.bizName(dimensionResp.getBizName()).modelId(dimensionResp.getModelId().toString())
|
||||
.domainId(dimensionResp.getDomainId().toString()).type(TypeEnums.DIMENSION).build();
|
||||
}
|
||||
|
||||
private DataEvent getDataEvent(List<DimensionDO> dimensionDOS, EventType eventType) {
|
||||
List<DataItem> dataItems = dimensionDOS.stream()
|
||||
.map(dimensionDO -> DataItem.builder().id(dimensionDO.getId() + Constants.UNDERLINE)
|
||||
.name(dimensionDO.getName())
|
||||
.modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
|
||||
.type(TypeEnums.DIMENSION).build())
|
||||
.collect(Collectors.toList());
|
||||
List<DataItem> dataItems =
|
||||
dimensionDOS.stream().map(this::getDataItem).collect(Collectors.toList());
|
||||
return new DataEvent(this, dataItems, eventType);
|
||||
}
|
||||
|
||||
|
||||
@@ -5,59 +5,26 @@ import com.alibaba.fastjson.TypeReference;
|
||||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
||||
import com.github.pagehelper.PageHelper;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DataEvent;
|
||||
import com.tencent.supersonic.common.pojo.DataItem;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.*;
|
||||
import com.tencent.supersonic.common.pojo.enums.*;
|
||||
import com.tencent.supersonic.common.util.BeanMapper;
|
||||
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.Measure;
|
||||
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.headless.api.pojo.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.*;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.*;
|
||||
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
|
||||
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
|
||||
import com.tencent.supersonic.headless.server.service.CollectService;
|
||||
import com.tencent.supersonic.headless.server.service.DataSetService;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.service.TagMetaService;
|
||||
import com.tencent.supersonic.headless.server.pojo.*;
|
||||
import com.tencent.supersonic.headless.server.service.*;
|
||||
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||
@@ -70,18 +37,7 @@ import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@@ -667,12 +623,13 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
}
|
||||
|
||||
private DataItem getDataItem(MetricDO metricDO) {
|
||||
MetricResp metricResp =
|
||||
MetricConverter.convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
|
||||
ModelResp modelResp = modelService.getModel(metricDO.getModelId());
|
||||
MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO,
|
||||
ImmutableMap.of(modelResp.getId(), modelResp), Lists.newArrayList());
|
||||
fillDefaultAgg(metricResp);
|
||||
return DataItem.builder().id(metricDO.getId() + Constants.UNDERLINE)
|
||||
.name(metricDO.getName()).bizName(metricDO.getBizName())
|
||||
.modelId(metricDO.getModelId() + Constants.UNDERLINE).type(TypeEnums.METRIC)
|
||||
return DataItem.builder().id(metricResp.getId().toString()).name(metricResp.getName())
|
||||
.bizName(metricResp.getBizName()).modelId(metricResp.getModelId().toString())
|
||||
.domainId(metricResp.getDomainId().toString()).type(TypeEnums.METRIC)
|
||||
.defaultAgg(metricResp.getDefaultAgg()).build();
|
||||
}
|
||||
|
||||
|
||||
@@ -459,7 +459,7 @@ public class ModelServiceImpl implements ModelService {
|
||||
}
|
||||
ModelFilter modelFilter = new ModelFilter();
|
||||
modelFilter.setDomainIds(domainIds);
|
||||
modelFilter.setIncludesDetail(false);
|
||||
modelFilter.setIncludesDetail(true);
|
||||
List<ModelResp> modelResps = getModelList(modelFilter);
|
||||
if (CollectionUtils.isEmpty(modelResps)) {
|
||||
return modelResps;
|
||||
|
||||
@@ -69,7 +69,7 @@ public class MetaEmbeddingTask implements CommandLineRunner {
|
||||
embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
|
||||
TextSegmentConvert.convertToEmbedding(metricDataItems));
|
||||
|
||||
List<DataItem> dimensionDataItems = dimensionService.getDataEvent().getDataItems();
|
||||
List<DataItem> dimensionDataItems = dimensionService.getAllDataEvents().getDataItems();
|
||||
embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
|
||||
TextSegmentConvert.convertToEmbedding(dimensionDataItems));
|
||||
} catch (Exception e) {
|
||||
|
||||
@@ -105,6 +105,8 @@ public class DimensionConverter {
|
||||
dimensionResp.setType(getType(dimensionDO.getType()));
|
||||
dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
|
||||
dimensionResp.setIsTag(dimensionDO.getIsTag());
|
||||
dimensionResp.setDomainId(modelRespMap
|
||||
.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getDomainId());
|
||||
return dimensionResp;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user