topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][chat]Rename PostProcessor to ResponseProcessor

Browse Source
This commit is contained in:
jerryjzhang
2023-11-29 10:28:49 +08:00
parent 95e3138ab2
commit 8171d754e0
14 changed files with 63 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/CostType.java
View File

@@ -5,7 +5,7 @@ public enum CostType {
PARSER(2, "parser"),
QUERY(3, "query"),
PARSERRESPONDER(4, "responder"),
POSTPROCESSOR(5, "postprocessor");
PROCESSOR(5, "processor");
private Integer type;
private String name;

4
chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/EntityInfoPostProcessor.java → chat/core/src/main/java/com/tencent/supersonic/chat/processor/EntityInfoProcessor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -16,7 +16,7 @@ import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
public class EntityInfoPostProcessor implements PostProcessor {
public class EntityInfoProcessor implements ResponseProcessor {
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {

6
chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessor.java → chat/core/src/main/java/com/tencent/supersonic/chat/processor/MetricCheckProcessor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -27,10 +27,10 @@ import java.util.Set;
import java.util.stream.Collectors;
/**
* MetricCheckPostProcessor used to verify whether the dimensions
* MetricCheckProcessor used to verify whether the dimensions
* involved in the query in metric mode can drill down on the metric.
*/
public class MetricCheckPostProcessor implements PostProcessor {
public class MetricCheckProcessor implements ResponseProcessor {
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {

4
chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/ParseInfoUpdateProcessor.java → chat/core/src/main/java/com/tencent/supersonic/chat/processor/ParseInfoProcessor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
@@ -36,7 +36,7 @@ import java.util.stream.Collectors;
* update parse info from correct sql
*/
@Slf4j
public class ParseInfoUpdateProcessor implements PostProcessor {
public class ParseInfoProcessor implements ResponseProcessor {
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {

4
chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/QueryRankPostProcessor.java → chat/core/src/main/java/com/tencent/supersonic/chat/processor/QueryRankProcessor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -18,7 +18,7 @@ import java.util.stream.Collectors;
* Rank queries by score.
*/
@Slf4j
public class QueryRankPostProcessor implements PostProcessor {
public class QueryRankProcessor implements ResponseProcessor {
private static final int candidateTopSize = 5;

4
chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/RespBuildPostProcessor.java → chat/core/src/main/java/com/tencent/supersonic/chat/processor/RespBuildProcessor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -13,7 +13,7 @@ import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class RespBuildPostProcessor implements PostProcessor {
public class RespBuildProcessor implements ResponseProcessor {
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {

7
chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/PostProcessor.java → chat/core/src/main/java/com/tencent/supersonic/chat/processor/ResponseProcessor.java
View File

@@ -1,13 +1,12 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
/**
* A post processor do some logic after parser and corrector
* A response processor wraps things up before responding to users.
*/
public interface PostProcessor {
public interface ResponseProcessor {
void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext);

4
chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/SqlOutputPostProcessor.java → chat/core/src/main/java/com/tencent/supersonic/chat/processor/SqlInfoProcessor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -13,7 +13,7 @@ import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
public class SqlOutputPostProcessor implements PostProcessor {
public class SqlInfoProcessor implements ResponseProcessor {
@Override
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {

4
chat/core/src/main/java/com/tencent/supersonic/chat/postprocessor/TimeCostPostProcessor.java → chat/core/src/main/java/com/tencent/supersonic/chat/processor/TimeCostProcessor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TimeCostPostProcessor implements PostProcessor {
public class TimeCostProcessor implements ResponseProcessor {
@Override

27
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -28,7 +28,7 @@ import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.persistence.dataobject.CostType;
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.chat.postprocessor.PostProcessor;
import com.tencent.supersonic.chat.processor.ResponseProcessor;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
@@ -111,20 +111,20 @@ public class QueryServiceImpl implements QueryService {
private List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
private List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
private List<PostProcessor> postProcessors = ComponentFactory.getPostProcessors();
private List<ResponseProcessor> responseProcessors = ComponentFactory.getPostProcessors();
private List<QueryResponder> executeResponders = ComponentFactory.getExecuteResponders();
private List<SemanticCorrector> semanticCorrectors = ComponentFactory.getSqlCorrections();
private List<SemanticCorrector> semanticCorrectors = ComponentFactory.getSemanticCorrectors();
@Override
public ParseResp performParsing(QueryReq queryReq) {
ParseResp parseResult = new ParseResp();
//1. build queryContext and chatContext
// build queryContext and chatContext
QueryContext queryCtx = new QueryContext(queryReq);
// in order to support multi-turn conversation, chat context is needed
ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId());
List<StatisticsDO> timeCostDOList = new ArrayList<>();
//2. mapper
// 1. mapper
schemaMappers.forEach(mapper -> {
long startTime = System.currentTimeMillis();
mapper.map(queryCtx);
@@ -132,7 +132,7 @@ public class QueryServiceImpl implements QueryService {
.interfaceName(mapper.getClass().getSimpleName()).type(CostType.MAPPER.getType()).build());
});
//3. parser
// 2. parser
semanticParsers.forEach(parser -> {
long startTime = System.currentTimeMillis();
parser.parse(queryCtx, chatCtx);
@@ -141,7 +141,7 @@ public class QueryServiceImpl implements QueryService {
log.info("{} result:{}", parser.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
});
//4. corrector
// 3. corrector
List<SemanticQuery> candidateQueries = queryCtx.getCandidateQueries();
if (CollectionUtils.isNotEmpty(candidateQueries)) {
for (SemanticQuery semanticQuery : candidateQueries) {
@@ -154,14 +154,15 @@ public class QueryServiceImpl implements QueryService {
});
}
}
//5. postProcessor
postProcessors.forEach(postProcessor -> {
// 4. processor
responseProcessors.forEach(processor -> {
long startTime = System.currentTimeMillis();
postProcessor.process(parseResult, queryCtx, chatCtx);
processor.process(parseResult, queryCtx, chatCtx);
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(postProcessor.getClass().getSimpleName())
.type(CostType.POSTPROCESSOR.getType()).build());
log.info("{} result:{}", postProcessor.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
.interfaceName(processor.getClass().getSimpleName())
.type(CostType.PROCESSOR.getType()).build());
log.info("{} result:{}", processor.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
});
if (Objects.nonNull(parseResult.getQueryId()) && timeCostDOList.size() > 0) {

22
chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java
View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.parser.LLMInterpreter;
import com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver;
import com.tencent.supersonic.chat.postprocessor.PostProcessor;
import com.tencent.supersonic.chat.processor.ResponseProcessor;
import com.tencent.supersonic.chat.responder.QueryResponder;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.core.io.support.SpringFactoriesLoader;
@@ -18,11 +18,11 @@ public class ComponentFactory {
private static List<SchemaMapper> schemaMappers = new ArrayList<>();
private static List<SemanticParser> semanticParsers = new ArrayList<>();
private static List<SemanticCorrector> s2SQLCorrections = new ArrayList<>();
private static List<SemanticCorrector> semanticCorrectors = new ArrayList<>();
private static SemanticInterpreter semanticInterpreter;
private static LLMInterpreter llmInterpreter;
private static List<PostProcessor> postProcessors = new ArrayList<>();
private static List<ResponseProcessor> responseProcessors = new ArrayList<>();
private static List<QueryResponder> executeResponders = new ArrayList<>();
private static ModelResolver modelResolver;
@@ -34,13 +34,14 @@ public class ComponentFactory {
return CollectionUtils.isEmpty(semanticParsers) ? init(SemanticParser.class, semanticParsers) : semanticParsers;
}
public static List<SemanticCorrector> getSqlCorrections() {
return CollectionUtils.isEmpty(s2SQLCorrections) ? init(SemanticCorrector.class,
s2SQLCorrections) : s2SQLCorrections;
public static List<SemanticCorrector> getSemanticCorrectors() {
return CollectionUtils.isEmpty(semanticCorrectors) ? init(SemanticCorrector.class,
semanticCorrectors) : semanticCorrectors;
}
public static List<PostProcessor> getPostProcessors() {
return CollectionUtils.isEmpty(postProcessors) ? init(PostProcessor.class, postProcessors) : postProcessors;
public static List<ResponseProcessor> getPostProcessors() {
return CollectionUtils.isEmpty(responseProcessors) ? init(ResponseProcessor.class,
responseProcessors) : responseProcessors;
}
public static List<QueryResponder> getExecuteResponders() {
@@ -55,11 +56,6 @@ public class ComponentFactory {
return semanticInterpreter;
}
public static void setSemanticLayer(SemanticInterpreter layer) {
semanticInterpreter = layer;
}
public static LLMInterpreter getLLMInterpreter() {
if (Objects.isNull(llmInterpreter)) {
llmInterpreter = init(LLMInterpreter.class);

22
chat/core/src/test/java/com/tencent/supersonic/chat/postprocessor/MetricCheckPostProcessorTest.java → chat/core/src/test/java/com/tencent/supersonic/chat/processor/MetricCheckProcessorTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.postprocessor;
package com.tencent.supersonic.chat.processor;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -14,11 +14,11 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Set;
class MetricCheckPostProcessorTest {
class MetricCheckProcessorTest {
@Test
void testProcessCorrectSql_necessaryDimension_groupBy() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 用户名, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 用户名";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema());
@@ -28,7 +28,7 @@ class MetricCheckPostProcessorTest {
@Test
void testProcessCorrectSql_necessaryDimension_where() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 用户名, sum(访问次数), count(distinct 访问用户数) from 超音数 where 部门 = 'HR' group by 用户名";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema());
@@ -39,7 +39,7 @@ class MetricCheckPostProcessorTest {
@Test
void testProcessCorrectSql_dimensionNotDrillDown_groupBy() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 页面, 部门, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 页面, 部门";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema());
@@ -49,7 +49,7 @@ class MetricCheckPostProcessorTest {
@Test
void testProcessCorrectSql_dimensionNotDrillDown_where() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 部门, sum(访问次数), count(distinct 访问用户数) from 超音数 where 页面 = 'P1' group by 部门";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema());
@@ -59,7 +59,7 @@ class MetricCheckPostProcessorTest {
@Test
void testProcessCorrectSql_dimensionNotDrillDown_necessaryDimension() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 页面, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 页面";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema());
@@ -69,7 +69,7 @@ class MetricCheckPostProcessorTest {
@Test
void testProcessCorrectSql_dimensionDrillDown() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 用户名, 部门, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 用户名, 部门";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo, mockModelSchema());
@@ -79,7 +79,7 @@ class MetricCheckPostProcessorTest {
@Test
void testProcessCorrectSql_noDrillDownDimensionSetting() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 页面, 用户名, sum(访问次数), count(distinct 访问用户数) from 超音数 group by 页面, 用户名";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo,
@@ -90,7 +90,7 @@ class MetricCheckPostProcessorTest {
@Test
void testProcessCorrectSql_noDrillDownDimensionSetting_noAgg() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 访问次数 from 超音数";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo,
@@ -101,7 +101,7 @@ class MetricCheckPostProcessorTest {
@Test
void testProcessCorrectSql_noDrillDownDimensionSetting_count() {
MetricCheckPostProcessor metricCheckPostProcessor = new MetricCheckPostProcessor();
MetricCheckProcessor metricCheckPostProcessor = new MetricCheckProcessor();
String correctSql = "select 部门, count(*) from 超音数 group by 部门";
SemanticParseInfo parseInfo = mockParseInfo(correctSql);
String actualProcessedSql = metricCheckPostProcessor.processCorrectSql(parseInfo,

2
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java
View File

@@ -223,7 +223,7 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
if (customNatureCollector.isEmpty()) {
for (int i = Nature.begin.ordinal() + 1; i < Nature.values().length; ++i) {
Nature nature = Nature.values()[i];
if(Objects.nonNull(nature)) {
if (Objects.nonNull(nature)) {
customNatureCollector.add(nature);
}
}

18
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -21,6 +21,15 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
com.tencent.supersonic.chat.corrector.HavingCorrector, \
com.tencent.supersonic.chat.corrector.FromCorrector
com.tencent.supersonic.chat.processor.ResponseProcessor=\
com.tencent.supersonic.chat.processor.MetricCheckProcessor, \
com.tencent.supersonic.chat.processor.ParseInfoProcessor, \
com.tencent.supersonic.chat.processor.QueryRankProcessor, \
com.tencent.supersonic.chat.processor.EntityInfoProcessor, \
com.tencent.supersonic.chat.processor.SqlInfoProcessor, \
com.tencent.supersonic.chat.processor.TimeCostProcessor, \
com.tencent.supersonic.chat.processor.RespBuildProcessor
com.tencent.supersonic.chat.parser.LLMInterpreter=\
com.tencent.supersonic.chat.parser.HttpLLMInterpreter
@@ -36,15 +45,6 @@ com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor
com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor
com.tencent.supersonic.chat.postprocessor.PostProcessor=\
com.tencent.supersonic.chat.postprocessor.MetricCheckPostProcessor, \
com.tencent.supersonic.chat.postprocessor.ParseInfoUpdateProcessor, \
com.tencent.supersonic.chat.postprocessor.QueryRankPostProcessor, \
com.tencent.supersonic.chat.postprocessor.EntityInfoPostProcessor, \
com.tencent.supersonic.chat.postprocessor.SqlOutputPostProcessor, \
com.tencent.supersonic.chat.postprocessor.TimeCostPostProcessor, \
com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
com.tencent.supersonic.chat.responder.QueryResponder=\
com.tencent.supersonic.chat.responder.EntityInfoQueryResponder, \
com.tencent.supersonic.chat.responder.SimilarMetricQueryResponder