topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-15 06:27:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Chat) Simplify processor in Headless and Chat (#822)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-03-15 12:39:14 +08:00
committed by GitHub
parent aad8bb4367
commit 988a025cdf
19 changed files with 85 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java
View File

@@ -1,43 +0,0 @@
package com.tencent.supersonic.headless.server.processor;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.core.chat.query.QueryManager;
import com.tencent.supersonic.headless.core.pojo.ChatContext;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
import org.springframework.util.CollectionUtils;
import java.util.List;
/**
* EntityInfoProcessor fills core attributes of an entity so that
* users get to know which entity is parsed out.
*/
public class EntityInfoProcessor implements ResultProcessor {
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
if (CollectionUtils.isEmpty(selectedParses)) {
return;
}
selectedParses.forEach(parseInfo -> {
String queryMode = parseInfo.getQueryMode();
if (QueryManager.containsRuleQuery(queryMode)) {
return;
}
//1. set entity info
DataSetSchema dataSetSchema =
queryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, queryContext.getUser());
if (QueryManager.isTagQuery(queryMode)
|| QueryManager.isMetricQuery(queryMode)) {
parseInfo.setEntityInfo(entityInfo);
}
});
}
}

84
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/QueryRankProcessor.java
View File

@@ -1,84 +0,0 @@
package com.tencent.supersonic.headless.server.processor;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.core.pojo.ChatContext;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.chat.query.SemanticQuery;
import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* QueryRankProcessor ranks candidate parsing results based on
* a heuristic scoring algorithm and then takes topN.
**/
@Slf4j
public class QueryRankProcessor implements ResultProcessor {
private static final int candidateTopSize = 5;
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries();
candidateQueries = rank(candidateQueries);
queryContext.setCandidateQueries(candidateQueries);
}
public List<SemanticQuery> rank(List<SemanticQuery> candidateQueries) {
log.debug("pick before [{}]", candidateQueries);
if (CollectionUtils.isEmpty(candidateQueries)) {
return candidateQueries;
}
List<SemanticQuery> selectedQueries = new ArrayList<>();
if (candidateQueries.size() == 1) {
selectedQueries.addAll(candidateQueries);
} else {
selectedQueries = getTopCandidateQuery(candidateQueries);
}
generateParseInfoId(selectedQueries);
log.debug("pick after [{}]", selectedQueries);
return selectedQueries;
}
public List<SemanticQuery> getTopCandidateQuery(List<SemanticQuery> semanticQueries) {
return semanticQueries.stream()
.filter(query -> !checkFullyInherited(query))
.sorted((o1, o2) -> {
if (o1.getParseInfo().getScore() < o2.getParseInfo().getScore()) {
return 1;
} else if (o1.getParseInfo().getScore() > o2.getParseInfo().getScore()) {
return -1;
}
return 0;
}).limit(candidateTopSize)
.collect(Collectors.toList());
}
private void generateParseInfoId(List<SemanticQuery> semanticQueries) {
for (int i = 0; i < semanticQueries.size(); i++) {
SemanticQuery query = semanticQueries.get(i);
query.getParseInfo().setId(i + 1);
}
}
private boolean checkFullyInherited(SemanticQuery query) {
SemanticParseInfo parseInfo = query.getParseInfo();
if (!(query instanceof RuleSemanticQuery)) {
return false;
}
for (SchemaElementMatch match : parseInfo.getElementMatches()) {
if (!match.isInherited()) {
return false;
}
}
return parseInfo.getDateInfo() == null || parseInfo.getDateInfo().isInherited();
}
}

33
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/RespBuildProcessor.java
View File

@@ -1,33 +0,0 @@
package com.tencent.supersonic.headless.server.processor;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.core.chat.query.SemanticQuery;
import com.tencent.supersonic.headless.core.pojo.ChatContext;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.stream.Collectors;
/**
* RespBuildProcessor fill response object with parsing results.
**/
@Slf4j
public class RespBuildProcessor implements ResultProcessor {
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
parseResp.setChatId(queryContext.getChatId());
parseResp.setQueryText(queryContext.getQueryText());
List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries();
if (candidateQueries.size() > 0) {
List<SemanticParseInfo> candidateParses = candidateQueries.stream()
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
parseResp.setSelectedParses(candidateParses);
parseResp.setState(ParseResp.ParseState.COMPLETED);
} else {
parseResp.setState(ParseResp.ParseState.FAILED);
}
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java
View File

@@ -40,9 +40,7 @@ public class SqlInfoProcessor implements ResultProcessor {
}
List<SemanticParseInfo> selectedParses = semanticQueries.stream().map(SemanticQuery::getParseInfo)
.collect(Collectors.toList());
long startTime = System.currentTimeMillis();
addSqlInfo(queryContext, selectedParses);
parseResp.getParseTimeCost().setSqlTime(System.currentTimeMillis() - startTime);
}
private void addSqlInfo(QueryContext queryContext, List<SemanticParseInfo> semanticParseInfos) {

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/TimeCostProcessor.java
View File

@@ -1,22 +0,0 @@
package com.tencent.supersonic.headless.server.processor;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.core.pojo.ChatContext;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import lombok.extern.slf4j.Slf4j;
/**
* TimeCostProcessor adds time cost of parsing.
**/
@Slf4j
public class TimeCostProcessor implements ResultProcessor {
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
long parseStartTime = parseResp.getParseTimeCost().getParseStartTime();
parseResp.getParseTimeCost().setParseTime(
System.currentTimeMillis() - parseStartTime - parseResp.getParseTimeCost().getSqlTime());
}
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java
View File

@@ -121,7 +121,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Override
public ParseResp performParsing(QueryReq queryReq) {
ParseResp parseResult = new ParseResp();
ParseResp parseResult = new ParseResp(queryReq.getChatId(), queryReq.getQueryText());
// build queryContext and chatContext
QueryContext queryCtx = buildQueryContext(queryReq);
@@ -157,6 +157,9 @@ public class ChatQueryServiceImpl implements ChatQueryService {
resultProcessors.forEach(processor -> {
processor.process(parseResult, queryCtx, chatCtx);
});
List<SemanticParseInfo> parseInfos = queryCtx.getCandidateQueries().stream()
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
parseResult.setSelectedParses(parseInfos);
return parseResult;
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java
View File

@@ -55,7 +55,7 @@ public class SemanticService {
}
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
if (parseInfo != null && parseInfo.getDataSetId() > 0) {
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) {
entityInfo.setMetrics(null);