topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvment)(chat) optimize parse performance (#197)

Browse Source 
* (improvment)(chat) optimize parse performance
---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-10-12 11:51:57 +08:00
committed by GitHub
parent b753eda9b9
commit e7b8c68dba
26 changed files with 214 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java
View File

@@ -26,7 +26,8 @@ public class EntityInfoParseResponder implements ParseResponder {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryReq.getUser());
if (QueryManager.isEntityQuery(parseInfo.getQueryMode())) {
if (QueryManager.isEntityQuery(parseInfo.getQueryMode())
|| QueryManager.isMetricQuery(parseInfo.getQueryMode())) {
parseInfo.setEntityInfo(entityInfo);
}
//2. set native value

23
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SolvedQueryParseResponder.java
View File

@@ -1,23 +0,0 @@
package com.tencent.supersonic.chat.responder.parse;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import com.tencent.supersonic.chat.utils.SolvedQueryManager;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@Slf4j
public class SolvedQueryParseResponder implements ParseResponder {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext) {
SolvedQueryManager solvedQueryManager = ContextUtils.getBean(SolvedQueryManager.class);
List<SolvedQueryRecallResp> solvedQueryRecallResps =
solvedQueryManager.recallSolvedQuery(queryContext.getRequest().getQueryText(),
queryContext.getRequest().getAgentId());
parseResp.setSimilarSolvedQuery(solvedQueryRecallResps);
}
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java
View File

@@ -4,6 +4,7 @@ package com.tencent.supersonic.chat.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
@@ -85,4 +86,10 @@ public class ChatController {
return chatService.queryShowCase(pageQueryInfoCommand, agentId);
}
@RequestMapping("/getSolvedQuery")
public List<SolvedQueryRecallResp> getSolvedQuery(@RequestParam(value = "queryText") String queryText,
@RequestParam(value = "agentId") Integer agentId) {
return chatService.getSolvedQuery(queryText, agentId);
}
}

35
chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java
View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
@@ -22,23 +23,23 @@ public interface ChatService {
* @param chatId
* @return
*/
public Long getContextModel(Integer chatId);
Long getContextModel(Integer chatId);
public ChatContext getOrCreateContext(int chatId);
ChatContext getOrCreateContext(int chatId);
public void updateContext(ChatContext chatCtx);
void updateContext(ChatContext chatCtx);
public void switchContext(ChatContext chatCtx);
void switchContext(ChatContext chatCtx);
public Boolean addChat(User user, String chatName, Integer agentId);
Boolean addChat(User user, String chatName, Integer agentId);
public List<ChatDO> getAll(String userName, Integer agentId);
List<ChatDO> getAll(String userName, Integer agentId);
public boolean updateChatName(Long chatId, String chatName, String userName);
boolean updateChatName(Long chatId, String chatName, String userName);
public boolean updateFeedback(Integer id, Integer score, String feedback);
boolean updateFeedback(Integer id, Integer score, String feedback);
public boolean updateChatIsTop(Long chatId, int isTop);
boolean updateChatIsTop(Long chatId, int isTop);
Boolean deleteChat(Long chatId, String userName);
@@ -46,20 +47,22 @@ public interface ChatService {
ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoCommend, int agentId);
public void addQuery(QueryResult queryResult, ChatContext chatCtx);
void addQuery(QueryResult queryResult, ChatContext chatCtx);
public void batchAddParse(ChatContext chatCtx, QueryReq queryReq,
void batchAddParse(ChatContext chatCtx, QueryReq queryReq,
ParseResp parseResult,
List<SemanticParseInfo> candidateParses,
List<SemanticParseInfo> selectedParses);
public ChatQueryDO getLastQuery(long chatId);
ChatQueryDO getLastQuery(long chatId);
public int updateQuery(ChatQueryDO chatQueryDO);
int updateQuery(ChatQueryDO chatQueryDO);
public Boolean updateQuery(Long questionId, QueryResult queryResult, ChatContext chatCtx);
Boolean updateQuery(Long questionId, QueryResult queryResult, ChatContext chatCtx);
public ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
public Boolean deleteChatQuery(Long questionId);
Boolean deleteChatQuery(Long questionId);
List<SolvedQueryRecallResp> getSolvedQuery(String queryText, Integer agentId);
}

11
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java
View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
@@ -25,6 +26,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import com.tencent.supersonic.chat.service.ChatService;
import com.tencent.supersonic.chat.utils.SolvedQueryManager;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Primary;
@@ -38,12 +40,14 @@ public class ChatServiceImpl implements ChatService {
private ChatContextRepository chatContextRepository;
private ChatRepository chatRepository;
private ChatQueryRepository chatQueryRepository;
private SolvedQueryManager solvedQueryManager;
public ChatServiceImpl(ChatContextRepository chatContextRepository, ChatRepository chatRepository,
ChatQueryRepository chatQueryRepository) {
ChatQueryRepository chatQueryRepository, SolvedQueryManager solvedQueryManager) {
this.chatContextRepository = chatContextRepository;
this.chatRepository = chatRepository;
this.chatQueryRepository = chatQueryRepository;
this.solvedQueryManager = solvedQueryManager;
}
@Override
@@ -192,4 +196,9 @@ public class ChatServiceImpl implements ChatService {
return chatQueryRepository.deleteChatQuery(questionId);
}
@Override
public List<SolvedQueryRecallResp> getSolvedQuery(String queryText, Integer agentId) {
return solvedQueryManager.recallSolvedQuery(queryText, agentId);
}
}

23
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -96,7 +96,7 @@ public class QueryServiceImpl implements QueryService {
// in order to support multi-turn conversation, chat context is needed
ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId());
List<StatisticsDO> timeCostDOList = new ArrayList<>();
schemaMappers.stream().forEach(mapper -> {
for (SchemaMapper mapper : schemaMappers) {
Long startTime = System.currentTimeMillis();
mapper.map(queryCtx);
Long endTime = System.currentTimeMillis();
@@ -104,8 +104,8 @@ public class QueryServiceImpl implements QueryService {
timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime))
.interfaceName(className).type(CostType.MAPPER.getType()).build());
log.info("{} result:{}", className, JsonUtil.toString(queryCtx));
});
semanticParsers.stream().forEach(parser -> {
}
for (SemanticParser parser : semanticParsers) {
Long startTime = System.currentTimeMillis();
parser.parse(queryCtx, chatCtx);
Long endTime = System.currentTimeMillis();
@@ -113,7 +113,7 @@ public class QueryServiceImpl implements QueryService {
timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime))
.interfaceName(className).type(CostType.PARSER.getType()).build());
log.info("{} result:{}", className, JsonUtil.toString(queryCtx));
});
}
ParseResp parseResult;
if (queryCtx.getCandidateQueries().size() > 0) {
log.debug("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
@@ -122,12 +122,8 @@ public class QueryServiceImpl implements QueryService {
log.debug("pick after [{}]", selectedQueries.stream().collect(
Collectors.toList()));
List<SemanticParseInfo> selectedParses = selectedQueries.stream()
.map(SemanticQuery::getParseInfo)
.sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
.collect(Collectors.toList());
List<SemanticParseInfo> candidateParses = queryCtx.getCandidateQueries().stream()
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
List<SemanticParseInfo> selectedParses = convertParseInfo(selectedQueries);
List<SemanticParseInfo> candidateParses = convertParseInfo(queryCtx.getCandidateQueries());
parseResult = ParseResp.builder()
.chatId(queryReq.getChatId())
.queryText(queryReq.getQueryText())
@@ -151,6 +147,13 @@ public class QueryServiceImpl implements QueryService {
return parseResult;
}
private List<SemanticParseInfo> convertParseInfo(List<SemanticQuery> semanticQueries) {
return semanticQueries.stream()
.map(SemanticQuery::getParseInfo)
.sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
.collect(Collectors.toList());
}
@Override
public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),

3
launchers/chat/src/main/resources/META-INF/spring.factories
View File

@@ -40,8 +40,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
com.tencent.supersonic.chat.responder.parse.ParseResponder=\
com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder, \
com.tencent.supersonic.chat.responder.parse.SolvedQueryParseResponder
com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder
com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder

3
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -40,8 +40,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
com.tencent.supersonic.chat.responder.parse.ParseResponder=\
com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder, \
com.tencent.supersonic.chat.responder.parse.SolvedQueryParseResponder
com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder
com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder

2
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/MeasureResp.java
View File

@@ -25,4 +25,6 @@ public class MeasureResp {
private String bizName;
private Long modelId;
}

4
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java
View File

@@ -103,15 +103,13 @@ public class CatalogImpl implements Catalog {
}
@Override
public String getAgg(Long modelId, String metricBizName) {
public String getAgg(List<MetricResp> metricResps, List<MeasureResp> measureRespList, String metricBizName) {
try {
List<MetricResp> metricResps = getMetrics(modelId);
if (!CollectionUtils.isEmpty(metricResps)) {
Optional<MetricResp> metric = metricResps.stream()
.filter(m -> m.getBizName().equalsIgnoreCase(metricBizName)).findFirst();
if (metric.isPresent() && Objects.nonNull(metric.get().getTypeParams()) && !CollectionUtils.isEmpty(
metric.get().getTypeParams().getMeasures())) {
List<MeasureResp> measureRespList = datasourceService.getMeasureListOfModel(modelId);
if (!CollectionUtils.isEmpty(measureRespList)) {
String measureName = metric.get().getTypeParams().getMeasures().get(0).getBizName();
Optional<MeasureResp> measure = measureRespList.stream()

29
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java
View File

@@ -146,6 +146,28 @@ public class DatasourceServiceImpl implements DatasourceService {
return measureResps;
}
@Override
public List<MeasureResp> getMeasureListOfModel(List<Long> modelIds) {
if (CollectionUtils.isEmpty(modelIds)) {
return Lists.newArrayList();
}
List<DatasourceResp> datasourceResps = getDatasourceList().stream().filter(datasourceResp ->
modelIds.contains(datasourceResp.getModelId())).collect(Collectors.toList());
List<MeasureResp> measureResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(datasourceResps)) {
for (DatasourceResp datasourceDesc : datasourceResps) {
DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail();
List<Measure> measures = datasourceDetail.getMeasures();
if (!CollectionUtils.isEmpty(measures)) {
measureResps.addAll(
measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc))
.collect(Collectors.toList()));
}
}
}
return measureResps;
}
private void batchCreateDimension(Datasource datasource, User user) throws Exception {
List<DimensionReq> dimensionReqs = DatasourceConverter.convertDimensionList(datasource);
@@ -239,13 +261,6 @@ public class DatasourceServiceImpl implements DatasourceService {
return DatasourceConverter.convertList(datasourceRepository.getDatasourceList());
}
@Override
public List<DatasourceResp> getDatasourceListByDatabaseId(Long databaseId) {
return getDatasourceList().stream()
.filter(datasourceResp -> datasourceResp.getDatabaseId().equals(databaseId))
.collect(Collectors.toList());
}
@Override
public List<DatasourceResp> getDatasourceListNoMeasurePrefix(Long modelId) {
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);

25
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java
View File

@@ -120,19 +120,16 @@ public class DimensionServiceImpl implements DimensionService {
Dimension dimension = DimensionConverter.convert(dimensionReq);
dimension.updatedBy(user.getName());
log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
List<DimensionResp> dimensionRespList = getDimensions(dimensionReq.getModelId()).stream().filter(
o -> o.getId().equals(dimensionReq.getId())).collect(Collectors.toList());
updateDimension(dimension);
DimensionResp dimensionResp = getDimension(dimensionReq.getId());
//动态更新字典
String type = DictWordType.DIMENSION.getType();
if (!CollectionUtils.isEmpty(dimensionRespList)) {
log.info("dimensionRespList size:{}", dimensionRespList.size());
log.info("name:{}", dimensionRespList.get(0).getName());
if (dimensionResp != null) {
applicationEventPublisher.publishEvent(
new DataUpdateEvent(this, dimensionRespList.get(0).getName(),
new DataUpdateEvent(this, dimensionResp.getName(),
dimensionReq.getName(),
dimension.getModelId(),
dimensionRespList.get(0).getId(), type));
dimensionResp.getId(), type));
}
}
@@ -156,6 +153,12 @@ public class DimensionServiceImpl implements DimensionService {
return null;
}
@Override
public DimensionResp getDimension(Long id) {
DimensionDO dimensionDO = dimensionRepository.getDimensionById(id);
return DimensionConverter.convert2DimensionResp(dimensionDO, new HashMap<>(), new HashMap<>());
}
@Override
public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq) {
DimensionFilter dimensionFilter = new DimensionFilter();
@@ -199,6 +202,12 @@ public class DimensionServiceImpl implements DimensionService {
return convertList(getDimensionDOS(), datasourceService.getDatasourceMap());
}
@Override
public List<DimensionResp> getDimensionsByModelIds(List<Long> modelIds) {
List<DimensionDO> dimensionDOS = dimensionRepository.getDimensionListOfmodelIds(modelIds);
return convertList(dimensionDOS, datasourceService.getDatasourceMap());
}
@Override
public List<DimensionResp> getDimensionsByDatasource(Long datasourceId) {
List<DimensionResp> dimensionResps = Lists.newArrayList();
@@ -239,7 +248,7 @@ public class DimensionServiceImpl implements DimensionService {
protected List<DimensionDO> getDimensionDOS(Long modelId) {
return dimensionRepository.getDimensionListOfDomain(modelId);
return dimensionRepository.getDimensionListOfmodel(modelId);
}
protected List<DimensionDO> getDimensionDOS() {

9
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java
View File

@@ -121,7 +121,14 @@ public class MetricServiceImpl implements MetricService {
}
public List<MetricResp> getMetrics(List<Long> ids) {
return convertList(metricRepository.getMetricListByIds(ids));
List<MetricDO> metricDOS = metricRepository.getMetricListByIds(ids);
return convertList(metricDOS);
}
@Override
public List<MetricResp> getMetricsByModelIds(List<Long> modelIds) {
List<MetricDO> metricDOS = metricRepository.getMetricList(modelIds);
return convertList(metricDOS);
}
@Override

71
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java
View File

@@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
@@ -226,7 +227,7 @@ public class ModelServiceImpl implements ModelService {
@Override
public Map<Long, String> getModelFullPathMap() {
return getModelList().stream().filter(m -> m != null).collect(Collectors.toMap(ModelResp::getId,
return getModelList().stream().filter(Objects::nonNull).collect(Collectors.toMap(ModelResp::getId,
ModelResp::getFullPath, (k1, k2) -> k1));
}
@@ -277,12 +278,29 @@ public class ModelServiceImpl implements ModelService {
if (CollectionUtils.isEmpty(modelIds)) {
modelIds = generateModelIdsReq(modelSchemaFilterReq);
}
modelIds.stream().forEach(modelId -> {
ModelSchemaResp modelSchemaResp = fetchSingleModelSchema(modelId);
if (Objects.nonNull(modelSchemaResp)) {
modelSchemaRespList.add(modelSchemaResp);
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetricsByModelIds(modelIds)
.stream().collect(Collectors.groupingBy(MetricResp::getModelId));
Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensionsByModelIds(modelIds)
.stream().collect(Collectors.groupingBy(DimensionResp::getModelId));
Map<Long, List<MeasureResp>> measureRespsMap = datasourceService.getMeasureListOfModel(modelIds)
.stream().collect(Collectors.groupingBy(MeasureResp::getModelId));
for (Long modelId : modelIds) {
ModelResp modelResp = getModelMap().get(modelId);
if (modelResp == null) {
continue;
}
});
List<MeasureResp> measureResps = measureRespsMap.getOrDefault(modelId, Lists.newArrayList());
List<MetricResp> metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList());
List<MetricSchemaResp> metricSchemaResps = metricResps.stream().map(metricResp ->
convert(metricResp, metricResps, measureResps)).collect(Collectors.toList());
List<DimSchemaResp> dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList())
.stream().map(this::convert).collect(Collectors.toList());
ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
BeanUtils.copyProperties(modelResp, modelSchemaResp);
modelSchemaResp.setDimensions(dimensionResps);
modelSchemaResp.setMetrics(metricSchemaResps);
modelSchemaRespList.add(modelSchemaResp);
}
return modelSchemaRespList;
}
@@ -298,30 +316,33 @@ public class ModelServiceImpl implements ModelService {
private List<MetricSchemaResp> generateMetricSchema(Long modelId) {
List<MetricSchemaResp> metricSchemaDescList = new ArrayList<>();
List<MetricResp> metricDescList = metricService.getMetrics(modelId);
metricDescList.stream().forEach(metricDesc -> {
MetricSchemaResp metricSchemaDesc = new MetricSchemaResp();
BeanUtils.copyProperties(metricDesc, metricSchemaDesc);
metricSchemaDesc.setUseCnt(0L);
String agg = catalog.getAgg(modelId, metricSchemaDesc.getBizName());
metricSchemaDesc.setDefaultAgg(agg);
metricSchemaDescList.add(metricSchemaDesc);
}
);
List<MetricResp> metricResps = metricService.getMetrics(modelId);
List<MeasureResp> measureResps = datasourceService.getMeasureListOfModel(modelId);
metricResps.stream().forEach(metricResp ->
metricSchemaDescList.add(convert(metricResp, metricResps, measureResps)));
return metricSchemaDescList;
}
private List<DimSchemaResp> generateDimSchema(Long modelId) {
List<DimSchemaResp> dimSchemaDescList = new ArrayList<>();
List<DimensionResp> dimDescList = dimensionService.getDimensions(modelId);
dimDescList.stream().forEach(dimDesc -> {
DimSchemaResp dimSchemaDesc = new DimSchemaResp();
BeanUtils.copyProperties(dimDesc, dimSchemaDesc);
dimSchemaDesc.setUseCnt(0L);
dimSchemaDescList.add(dimSchemaDesc);
}
);
return dimSchemaDescList;
return dimDescList.stream().map(this::convert).collect(Collectors.toList());
}
private DimSchemaResp convert(DimensionResp dimensionResp) {
DimSchemaResp dimSchemaResp = new DimSchemaResp();
BeanUtils.copyProperties(dimensionResp, dimSchemaResp);
dimSchemaResp.setUseCnt(0L);
return dimSchemaResp;
}
private MetricSchemaResp convert(MetricResp metricResp, List<MetricResp> metricResps,
List<MeasureResp> measureResps) {
MetricSchemaResp metricSchemaResp = new MetricSchemaResp();
BeanUtils.copyProperties(metricResp, metricSchemaResp);
metricSchemaResp.setUseCnt(0L);
String agg = catalog.getAgg(metricResps, measureResps, metricSchemaResp.getBizName());
metricSchemaResp.setDefaultAgg(agg);
return metricSchemaResp;
}
private List<Long> generateModelIdsReq(ModelSchemaFilterReq filter) {

3
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java
View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
@@ -37,6 +38,6 @@ public interface Catalog {
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
String getAgg(Long modelId, String metricBizName);
String getAgg(List<MetricResp> metricResps, List<MeasureResp> measureRespList, String metricBizName);
}

4
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java
View File

@@ -24,8 +24,6 @@ public interface DatasourceService {
List<DatasourceResp> getDatasourceListNoMeasurePrefix(Long modelId);
List<DatasourceResp> getDatasourceListByDatabaseId(Long databaseId);
List<DatasourceResp> getDatasourceList();
List<DatasourceResp> getDatasourceList(Long modelId);
@@ -44,6 +42,8 @@ public interface DatasourceService {
List<MeasureResp> getMeasureListOfModel(Long modelId);
List<MeasureResp> getMeasureListOfModel(List<Long> modelIds);
void getModelYamlTplByModelIds(Set<Long> modelIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DatasourceYamlTpl> datasourceYamlTplList, List<MetricYamlTpl> metricYamlTplList);

6
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java
View File

@@ -16,7 +16,11 @@ public interface DimensionService {
List<DimensionResp> getDimensions();
DimensionResp getDimension(String bizName, Long domainId);
DimensionResp getDimension(Long id);
DimensionResp getDimension(String bizName, Long modelId);
List<DimensionResp> getDimensionsByModelIds(List<Long> modelIds);
void createDimension(DimensionReq dimensionReq, User user) throws Exception;

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java
View File

@@ -23,6 +23,8 @@ public interface MetricService {
void createMetricBatch(List<MetricReq> metricReqs, User user) throws Exception;
List<MetricResp> getMetricsByModelIds(List<Long> modelIds);
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
MetricResp getMetric(Long modelId, String bizName);

4
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java
View File

@@ -15,7 +15,9 @@ public interface DimensionRepository {
List<DimensionDO> getDimensionListOfDatasource(Long datasourceId);
List<DimensionDO> getDimensionListOfDomain(Long domainId);
List<DimensionDO> getDimensionListOfmodel(Long domainId);
List<DimensionDO> getDimensionListOfmodelIds(List<Long> modelIds);
List<DimensionDO> getDimensionList();

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java
View File

@@ -17,6 +17,8 @@ public interface MetricRepository {
List<MetricDO> getMetricList(Long domainId);
List<MetricDO> getMetricList(List<Long> modelIds);
List<MetricDO> getMetricList();
List<MetricDO> getMetricListByIds(List<Long> ids);

15
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java
View File

@@ -88,13 +88,14 @@ public class DatasourceConverter {
return datasourceDesc;
}
public static MeasureResp convert(Measure measure, DatasourceResp datasourceDesc) {
MeasureResp measureDesc = new MeasureResp();
BeanUtils.copyProperties(measure, measureDesc);
measureDesc.setDatasourceId(datasourceDesc.getId());
measureDesc.setDatasourceName(datasourceDesc.getName());
measureDesc.setDatasourceBizName(datasourceDesc.getBizName());
return measureDesc;
public static MeasureResp convert(Measure measure, DatasourceResp datasourceResp) {
MeasureResp measureResp = new MeasureResp();
BeanUtils.copyProperties(measure, measureResp);
measureResp.setDatasourceId(datasourceResp.getId());
measureResp.setDatasourceName(datasourceResp.getName());
measureResp.setDatasourceBizName(datasourceResp.getBizName());
measureResp.setModelId(datasourceResp.getModelId());
return measureResp;
}
public static DimensionReq convert(Dim dim, Datasource datasource) {

3
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/NameCheckUtils.java
View File

@@ -5,6 +5,9 @@ import java.util.regex.Pattern;
public class NameCheckUtils {
public static boolean containsSpecialCharacters(String str) {
if (str == null) {
return false;
}
String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
return Pattern.compile(regex).matcher(str).find();
}

11
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java
View File

@@ -48,9 +48,16 @@ public class DimensionRepositoryImpl implements DimensionRepository {
}
@Override
public List<DimensionDO> getDimensionListOfDomain(Long domainId) {
public List<DimensionDO> getDimensionListOfmodel(Long modelId) {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
dimensionDOExample.createCriteria().andModelIdEqualTo(domainId);
dimensionDOExample.createCriteria().andModelIdEqualTo(modelId);
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}
@Override
public List<DimensionDO> getDimensionListOfmodelIds(List<Long> modelIds) {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
dimensionDOExample.createCriteria().andModelIdIn(modelIds);
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}

11
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java
View File

@@ -42,9 +42,16 @@ public class MetricRepositoryImpl implements MetricRepository {
}
@Override
public List<MetricDO> getMetricList(Long domainId) {
public List<MetricDO> getMetricList(Long modelId) {
MetricDOExample metricDOExample = new MetricDOExample();
metricDOExample.createCriteria().andModelIdEqualTo(domainId);
metricDOExample.createCriteria().andModelIdEqualTo(modelId);
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
}
@Override
public List<MetricDO> getMetricList(List<Long> modelIds) {
MetricDOExample metricDOExample = new MetricDOExample();
metricDOExample.createCriteria().andModelIdIn(modelIds);
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
}

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepositoryImpl.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.semantic.query.persistence.repository;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
@@ -16,6 +15,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Repository;
@@ -37,9 +37,10 @@ public class StatRepositoryImpl implements StatRepository {
}
@Override
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend) {
@SneakyThrows
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
List<ItemUseResp> result = new ArrayList<>();
List<QueryStat> statInfos = statMapper.getStatInfo(itemUseCommend);
List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
Map<String, Long> map = new ConcurrentHashMap<>();
statInfos.stream().forEach(stat -> {
String dimensions = stat.getDimensions();
@@ -55,9 +56,8 @@ public class StatRepositoryImpl implements StatRepository {
result.add(new ItemUseResp(classId, type, nameEn, v));
});
List<ItemUseResp> itemUseResps = result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
return result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
.collect(Collectors.toList());
return itemUseResps;
}
@Override

18
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -1,9 +1,11 @@
package com.tencent.supersonic.semantic.query.service;
import com.google.common.cache.CacheBuilder;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
@@ -31,6 +33,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -42,6 +45,8 @@ import org.springframework.stereotype.Service;
@Slf4j
public class QueryServiceImpl implements QueryService {
protected final com.google.common.cache.Cache<String, List<ItemUseResp>> itemUseCache =
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
private final StatUtils statUtils;
private final CacheUtils cacheUtils;
@@ -206,9 +211,16 @@ public class QueryServiceImpl implements QueryService {
}
@Override
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend) {
List<ItemUseResp> statInfos = statUtils.getStatInfo(itemUseCommend);
return statInfos;
@SneakyThrows
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
if (itemUseReq.getCacheEnable()) {
return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> {
List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
itemUseCache.put(JsonUtil.toString(itemUseReq), data);
return data;
});
}
return statUtils.getStatInfo(itemUseReq);
}
@Override