mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-13 13:07:32 +00:00
(improvement)(Chat) Simplify processor in Headless and Chat (#822)
Co-authored-by: jolunoluo
This commit is contained in:
@@ -70,6 +70,11 @@ public class Agent extends RecordInfo {
|
|||||||
return !CollectionUtils.isEmpty(getParserTools(AgentToolType.NL2SQL_LLM));
|
return !CollectionUtils.isEmpty(getParserTools(AgentToolType.NL2SQL_LLM));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean containsNL2SQLTool() {
|
||||||
|
return !CollectionUtils.isEmpty(getParserTools(AgentToolType.NL2SQL_LLM))
|
||||||
|
|| !CollectionUtils.isEmpty(getParserTools(AgentToolType.NL2SQL_RULE));
|
||||||
|
}
|
||||||
|
|
||||||
public Set<Long> getDataSetIds() {
|
public Set<Long> getDataSetIds() {
|
||||||
Set<Long> dataSetIds = getDataSetIds(null);
|
Set<Long> dataSetIds = getDataSetIds(null);
|
||||||
if (containsAllModel(dataSetIds)) {
|
if (containsAllModel(dataSetIds)) {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ public class PluginExecutor implements ChatExecutor {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
PluginSemanticQuery query = PluginQueryManager.getPluginQuery(parseInfo.getQueryMode());
|
PluginSemanticQuery query = PluginQueryManager.getPluginQuery(parseInfo.getQueryMode());
|
||||||
|
query.setParseInfo(parseInfo);
|
||||||
return query.build();
|
return query.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.server.util.ComponentFactory;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public class Text2PluginParser implements ChatParser {
|
public class NL2PluginParser implements ChatParser {
|
||||||
|
|
||||||
private final List<PluginRecognizer> pluginRecognizers = ComponentFactory.getPluginRecognizers();
|
private final List<PluginRecognizer> pluginRecognizers = ComponentFactory.getPluginRecognizers();
|
||||||
|
|
||||||
@@ -7,10 +7,13 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
||||||
import com.tencent.supersonic.headless.server.service.ChatQueryService;
|
import com.tencent.supersonic.headless.server.service.ChatQueryService;
|
||||||
|
|
||||||
public class Text2SqlParser implements ChatParser {
|
public class NL2SQLParser implements ChatParser {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void parse(ChatParseContext chatParseContext, ParseResp parseResp) {
|
public void parse(ChatParseContext chatParseContext, ParseResp parseResp) {
|
||||||
|
if (!chatParseContext.enableNL2SQL()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
|
QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
|
||||||
ChatQueryService chatQueryService = ContextUtils.getBean(ChatQueryService.class);
|
ChatQueryService chatQueryService = ContextUtils.getBean(ChatQueryService.class);
|
||||||
ParseResp text2SqlParseResp = chatQueryService.performParsing(queryReq);
|
ParseResp text2SqlParseResp = chatQueryService.performParsing(queryReq);
|
||||||
@@ -12,7 +12,6 @@ import com.tencent.supersonic.chat.server.plugin.event.PluginAddEvent;
|
|||||||
import com.tencent.supersonic.chat.server.plugin.event.PluginDelEvent;
|
import com.tencent.supersonic.chat.server.plugin.event.PluginDelEvent;
|
||||||
import com.tencent.supersonic.chat.server.plugin.event.PluginUpdateEvent;
|
import com.tencent.supersonic.chat.server.plugin.event.PluginUpdateEvent;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
||||||
import com.tencent.supersonic.chat.server.service.AgentService;
|
|
||||||
import com.tencent.supersonic.chat.server.service.PluginService;
|
import com.tencent.supersonic.chat.server.service.PluginService;
|
||||||
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||||
import com.tencent.supersonic.common.util.ComponentFactory;
|
import com.tencent.supersonic.common.util.ComponentFactory;
|
||||||
@@ -54,9 +53,7 @@ public class PluginManager {
|
|||||||
|
|
||||||
public static List<Plugin> getPluginAgentCanSupport(ChatParseContext chatParseContext) {
|
public static List<Plugin> getPluginAgentCanSupport(ChatParseContext chatParseContext) {
|
||||||
PluginService pluginService = ContextUtils.getBean(PluginService.class);
|
PluginService pluginService = ContextUtils.getBean(PluginService.class);
|
||||||
AgentService agentService = ContextUtils.getBean(AgentService.class);
|
Agent agent = chatParseContext.getAgent();
|
||||||
Agent agent = agentService.getAgent(chatParseContext.getAgentId());
|
|
||||||
|
|
||||||
List<Plugin> plugins = pluginService.getPluginList();
|
List<Plugin> plugins = pluginService.getPluginList();
|
||||||
if (Objects.isNull(agent)) {
|
if (Objects.isNull(agent)) {
|
||||||
return plugins;
|
return plugins;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.server.pojo;
|
package com.tencent.supersonic.chat.server.pojo;
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -9,9 +10,16 @@ import lombok.Data;
|
|||||||
public class ChatParseContext {
|
public class ChatParseContext {
|
||||||
private String queryText;
|
private String queryText;
|
||||||
private Integer chatId;
|
private Integer chatId;
|
||||||
private Integer agentId;
|
private Agent agent;
|
||||||
private User user;
|
private User user;
|
||||||
private QueryFilters queryFilters;
|
private QueryFilters queryFilters;
|
||||||
private boolean saveAnswer = true;
|
private boolean saveAnswer = true;
|
||||||
private SchemaMapInfo mapInfo = new SchemaMapInfo();
|
private SchemaMapInfo mapInfo = new SchemaMapInfo();
|
||||||
|
|
||||||
|
public boolean enableNL2SQL() {
|
||||||
|
if (agent == null) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return agent.containsNL2SQLTool();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,25 +1,25 @@
|
|||||||
package com.tencent.supersonic.headless.server.processor;
|
package com.tencent.supersonic.chat.server.processor.parse;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
||||||
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
|
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
||||||
import com.tencent.supersonic.headless.core.chat.query.QueryManager;
|
import com.tencent.supersonic.headless.core.chat.query.QueryManager;
|
||||||
import com.tencent.supersonic.headless.core.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.headless.core.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
|
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* EntityInfoProcessor fills core attributes of an entity so that
|
* EntityInfoProcessor fills core attributes of an entity so that
|
||||||
* users get to know which entity is parsed out.
|
* users get to know which entity is parsed out.
|
||||||
*/
|
*/
|
||||||
public class EntityInfoProcessor implements ResultProcessor {
|
public class EntityInfoProcessor implements ParseResultProcessor {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
public void process(ChatParseContext chatParseContext, ParseResp parseResp) {
|
||||||
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
|
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
|
||||||
if (CollectionUtils.isEmpty(selectedParses)) {
|
if (CollectionUtils.isEmpty(selectedParses)) {
|
||||||
return;
|
return;
|
||||||
@@ -30,10 +30,9 @@ public class EntityInfoProcessor implements ResultProcessor {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
//1. set entity info
|
//1. set entity info
|
||||||
DataSetSchema dataSetSchema =
|
|
||||||
queryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
|
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
||||||
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, queryContext.getUser());
|
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(parseInfo.getDataSetId());
|
||||||
|
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, chatParseContext.getUser());
|
||||||
if (QueryManager.isTagQuery(queryMode)
|
if (QueryManager.isTagQuery(queryMode)
|
||||||
|| QueryManager.isMetricQuery(queryMode)) {
|
|| QueryManager.isMetricQuery(queryMode)) {
|
||||||
parseInfo.setEntityInfo(entityInfo);
|
parseInfo.setEntityInfo(entityInfo);
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.server.processor.parse;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.collections4.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* RespBuildProcessor fill response object with parsing results.
|
|
||||||
**/
|
|
||||||
@Slf4j
|
|
||||||
public class RespBuildProcessor implements ParseResultProcessor {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void process(ChatParseContext chatParseContext, ParseResp parseResp) {
|
|
||||||
parseResp.setChatId(chatParseContext.getChatId());
|
|
||||||
parseResp.setQueryText(chatParseContext.getQueryText());
|
|
||||||
List<SemanticParseInfo> parseInfos = parseResp.getSelectedParses();
|
|
||||||
if (CollectionUtils.isNotEmpty(parseInfos)) {
|
|
||||||
parseResp.setState(ParseResp.ParseState.COMPLETED);
|
|
||||||
} else {
|
|
||||||
parseResp.setState(ParseResp.ParseState.FAILED);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,19 +1,18 @@
|
|||||||
package com.tencent.supersonic.headless.server.processor;
|
package com.tencent.supersonic.chat.server.processor.parse;
|
||||||
|
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
||||||
import com.tencent.supersonic.headless.core.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.headless.core.pojo.QueryContext;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* TimeCostProcessor adds time cost of parsing.
|
* TimeCostProcessor adds time cost of parsing.
|
||||||
**/
|
**/
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class TimeCostProcessor implements ResultProcessor {
|
public class TimeCostProcessor implements ParseResultProcessor {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
public void process(ChatParseContext chatParseContext, ParseResp parseResp) {
|
||||||
long parseStartTime = parseResp.getParseTimeCost().getParseStartTime();
|
long parseStartTime = parseResp.getParseTimeCost().getParseStartTime();
|
||||||
parseResp.getParseTimeCost().setParseTime(
|
parseResp.getParseTimeCost().setParseTime(
|
||||||
System.currentTimeMillis() - parseStartTime - parseResp.getParseTimeCost().getSqlTime());
|
System.currentTimeMillis() - parseStartTime - parseResp.getParseTimeCost().getSqlTime());
|
||||||
@@ -7,6 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
|||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
|
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
|
import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
|
||||||
|
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||||
import com.tencent.supersonic.chat.server.executor.ChatExecutor;
|
import com.tencent.supersonic.chat.server.executor.ChatExecutor;
|
||||||
import com.tencent.supersonic.chat.server.parser.ChatParser;
|
import com.tencent.supersonic.chat.server.parser.ChatParser;
|
||||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO;
|
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO;
|
||||||
@@ -18,10 +19,12 @@ import com.tencent.supersonic.chat.server.persistence.repository.ChatRepository;
|
|||||||
import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext;
|
import com.tencent.supersonic.chat.server.pojo.ChatExecuteContext;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
||||||
import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor;
|
import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor;
|
||||||
|
import com.tencent.supersonic.chat.server.service.AgentService;
|
||||||
import com.tencent.supersonic.chat.server.service.ChatService;
|
import com.tencent.supersonic.chat.server.service.ChatService;
|
||||||
import com.tencent.supersonic.chat.server.util.ComponentFactory;
|
import com.tencent.supersonic.chat.server.util.ComponentFactory;
|
||||||
import com.tencent.supersonic.chat.server.util.QueryReqConverter;
|
import com.tencent.supersonic.chat.server.util.QueryReqConverter;
|
||||||
import com.tencent.supersonic.common.util.BeanMapper;
|
import com.tencent.supersonic.common.util.BeanMapper;
|
||||||
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
|
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
|
||||||
@@ -73,7 +76,7 @@ public class ChatServiceImpl implements ChatService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ParseResp performParsing(ChatParseReq chatParseReq) {
|
public ParseResp performParsing(ChatParseReq chatParseReq) {
|
||||||
ParseResp parseResp = new ParseResp();
|
ParseResp parseResp = new ParseResp(chatParseReq.getChatId(), chatParseReq.getQueryText());
|
||||||
ChatParseContext chatParseContext = buildParseContext(chatParseReq);
|
ChatParseContext chatParseContext = buildParseContext(chatParseReq);
|
||||||
for (ChatParser chatParser : chatParsers) {
|
for (ChatParser chatParser : chatParsers) {
|
||||||
chatParser.parse(chatParseContext, parseResp);
|
chatParser.parse(chatParseContext, parseResp);
|
||||||
@@ -102,6 +105,9 @@ public class ChatServiceImpl implements ChatService {
|
|||||||
private ChatParseContext buildParseContext(ChatParseReq chatParseReq) {
|
private ChatParseContext buildParseContext(ChatParseReq chatParseReq) {
|
||||||
ChatParseContext chatParseContext = new ChatParseContext();
|
ChatParseContext chatParseContext = new ChatParseContext();
|
||||||
BeanMapper.mapper(chatParseReq, chatParseContext);
|
BeanMapper.mapper(chatParseReq, chatParseContext);
|
||||||
|
AgentService agentService = ContextUtils.getBean(AgentService.class);
|
||||||
|
Agent agent = agentService.getAgent(chatParseReq.getAgentId());
|
||||||
|
chatParseContext.setAgent(agent);
|
||||||
QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
|
QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
|
||||||
MapResp mapResp = chatQueryService.performMapping(queryReq);
|
MapResp mapResp = chatQueryService.performMapping(queryReq);
|
||||||
chatParseContext.setMapInfo(mapResp.getMapInfo());
|
chatParseContext.setMapInfo(mapResp.getMapInfo());
|
||||||
|
|||||||
@@ -2,9 +2,7 @@ package com.tencent.supersonic.chat.server.util;
|
|||||||
|
|
||||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
|
||||||
import com.tencent.supersonic.chat.server.service.AgentService;
|
|
||||||
import com.tencent.supersonic.common.util.BeanMapper;
|
import com.tencent.supersonic.common.util.BeanMapper;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
||||||
|
|
||||||
public class QueryReqConverter {
|
public class QueryReqConverter {
|
||||||
@@ -12,11 +10,7 @@ public class QueryReqConverter {
|
|||||||
public static QueryReq buildText2SqlQueryReq(ChatParseContext chatParseContext) {
|
public static QueryReq buildText2SqlQueryReq(ChatParseContext chatParseContext) {
|
||||||
QueryReq queryReq = new QueryReq();
|
QueryReq queryReq = new QueryReq();
|
||||||
BeanMapper.mapper(chatParseContext, queryReq);
|
BeanMapper.mapper(chatParseContext, queryReq);
|
||||||
if (chatParseContext.getAgentId() == null) {
|
Agent agent = chatParseContext.getAgent();
|
||||||
return queryReq;
|
|
||||||
}
|
|
||||||
AgentService agentService = ContextUtils.getBean(AgentService.class);
|
|
||||||
Agent agent = agentService.getAgent(chatParseContext.getAgentId());
|
|
||||||
if (agent == null) {
|
if (agent == null) {
|
||||||
return queryReq;
|
return queryReq;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,11 @@ package com.tencent.supersonic.headless.api.pojo.response;
|
|||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
|
|
||||||
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public class ParseResp {
|
public class ParseResp {
|
||||||
@@ -20,4 +24,33 @@ public class ParseResp {
|
|||||||
FAILED
|
FAILED
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<SemanticParseInfo> getSelectedParses() {
|
||||||
|
selectedParses = selectedParses.stream()
|
||||||
|
.sorted(Comparator.comparingDouble(SemanticParseInfo::getScore))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
generateParseInfoId(selectedParses);
|
||||||
|
return selectedParses;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ParseState getState() {
|
||||||
|
if (CollectionUtils.isNotEmpty(selectedParses)) {
|
||||||
|
this.state = ParseResp.ParseState.COMPLETED;
|
||||||
|
} else {
|
||||||
|
this.state = ParseState.FAILED;
|
||||||
|
}
|
||||||
|
return this.state;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void generateParseInfoId(List<SemanticParseInfo> selectedParses) {
|
||||||
|
for (int i = 0; i < selectedParses.size(); i++) {
|
||||||
|
SemanticParseInfo parseInfo = selectedParses.get(i);
|
||||||
|
parseInfo.setId(i + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public ParseResp(Integer chatId, String queryText) {
|
||||||
|
this.chatId = chatId;
|
||||||
|
this.queryText = queryText;
|
||||||
|
parseTimeCost.setParseStartTime(System.currentTimeMillis());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,84 +0,0 @@
|
|||||||
package com.tencent.supersonic.headless.server.processor;
|
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
|
||||||
import com.tencent.supersonic.headless.core.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.headless.core.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.headless.core.chat.query.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* QueryRankProcessor ranks candidate parsing results based on
|
|
||||||
* a heuristic scoring algorithm and then takes topN.
|
|
||||||
**/
|
|
||||||
@Slf4j
|
|
||||||
public class QueryRankProcessor implements ResultProcessor {
|
|
||||||
|
|
||||||
private static final int candidateTopSize = 5;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
|
||||||
List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries();
|
|
||||||
candidateQueries = rank(candidateQueries);
|
|
||||||
queryContext.setCandidateQueries(candidateQueries);
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<SemanticQuery> rank(List<SemanticQuery> candidateQueries) {
|
|
||||||
log.debug("pick before [{}]", candidateQueries);
|
|
||||||
if (CollectionUtils.isEmpty(candidateQueries)) {
|
|
||||||
return candidateQueries;
|
|
||||||
}
|
|
||||||
List<SemanticQuery> selectedQueries = new ArrayList<>();
|
|
||||||
if (candidateQueries.size() == 1) {
|
|
||||||
selectedQueries.addAll(candidateQueries);
|
|
||||||
} else {
|
|
||||||
selectedQueries = getTopCandidateQuery(candidateQueries);
|
|
||||||
}
|
|
||||||
generateParseInfoId(selectedQueries);
|
|
||||||
log.debug("pick after [{}]", selectedQueries);
|
|
||||||
return selectedQueries;
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<SemanticQuery> getTopCandidateQuery(List<SemanticQuery> semanticQueries) {
|
|
||||||
return semanticQueries.stream()
|
|
||||||
.filter(query -> !checkFullyInherited(query))
|
|
||||||
.sorted((o1, o2) -> {
|
|
||||||
if (o1.getParseInfo().getScore() < o2.getParseInfo().getScore()) {
|
|
||||||
return 1;
|
|
||||||
} else if (o1.getParseInfo().getScore() > o2.getParseInfo().getScore()) {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}).limit(candidateTopSize)
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
private void generateParseInfoId(List<SemanticQuery> semanticQueries) {
|
|
||||||
for (int i = 0; i < semanticQueries.size(); i++) {
|
|
||||||
SemanticQuery query = semanticQueries.get(i);
|
|
||||||
query.getParseInfo().setId(i + 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean checkFullyInherited(SemanticQuery query) {
|
|
||||||
SemanticParseInfo parseInfo = query.getParseInfo();
|
|
||||||
if (!(query instanceof RuleSemanticQuery)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (SchemaElementMatch match : parseInfo.getElementMatches()) {
|
|
||||||
if (!match.isInherited()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return parseInfo.getDateInfo() == null || parseInfo.getDateInfo().isInherited();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
package com.tencent.supersonic.headless.server.processor;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
|
||||||
import com.tencent.supersonic.headless.core.chat.query.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.headless.core.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.headless.core.pojo.QueryContext;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* RespBuildProcessor fill response object with parsing results.
|
|
||||||
**/
|
|
||||||
@Slf4j
|
|
||||||
public class RespBuildProcessor implements ResultProcessor {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) {
|
|
||||||
parseResp.setChatId(queryContext.getChatId());
|
|
||||||
parseResp.setQueryText(queryContext.getQueryText());
|
|
||||||
List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries();
|
|
||||||
if (candidateQueries.size() > 0) {
|
|
||||||
List<SemanticParseInfo> candidateParses = candidateQueries.stream()
|
|
||||||
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
|
|
||||||
parseResp.setSelectedParses(candidateParses);
|
|
||||||
parseResp.setState(ParseResp.ParseState.COMPLETED);
|
|
||||||
} else {
|
|
||||||
parseResp.setState(ParseResp.ParseState.FAILED);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -40,9 +40,7 @@ public class SqlInfoProcessor implements ResultProcessor {
|
|||||||
}
|
}
|
||||||
List<SemanticParseInfo> selectedParses = semanticQueries.stream().map(SemanticQuery::getParseInfo)
|
List<SemanticParseInfo> selectedParses = semanticQueries.stream().map(SemanticQuery::getParseInfo)
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
addSqlInfo(queryContext, selectedParses);
|
addSqlInfo(queryContext, selectedParses);
|
||||||
parseResp.getParseTimeCost().setSqlTime(System.currentTimeMillis() - startTime);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addSqlInfo(QueryContext queryContext, List<SemanticParseInfo> semanticParseInfos) {
|
private void addSqlInfo(QueryContext queryContext, List<SemanticParseInfo> semanticParseInfos) {
|
||||||
|
|||||||
@@ -121,7 +121,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ParseResp performParsing(QueryReq queryReq) {
|
public ParseResp performParsing(QueryReq queryReq) {
|
||||||
ParseResp parseResult = new ParseResp();
|
ParseResp parseResult = new ParseResp(queryReq.getChatId(), queryReq.getQueryText());
|
||||||
// build queryContext and chatContext
|
// build queryContext and chatContext
|
||||||
QueryContext queryCtx = buildQueryContext(queryReq);
|
QueryContext queryCtx = buildQueryContext(queryReq);
|
||||||
|
|
||||||
@@ -157,6 +157,9 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
resultProcessors.forEach(processor -> {
|
resultProcessors.forEach(processor -> {
|
||||||
processor.process(parseResult, queryCtx, chatCtx);
|
processor.process(parseResult, queryCtx, chatCtx);
|
||||||
});
|
});
|
||||||
|
List<SemanticParseInfo> parseInfos = queryCtx.getCandidateQueries().stream()
|
||||||
|
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
|
||||||
|
parseResult.setSelectedParses(parseInfos);
|
||||||
return parseResult;
|
return parseResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ public class SemanticService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
|
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
|
||||||
if (parseInfo != null && parseInfo.getDataSetId() > 0) {
|
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
|
||||||
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
|
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
|
||||||
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) {
|
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) {
|
||||||
entityInfo.setMetrics(null);
|
entityInfo.setMetrics(null);
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ com.tencent.supersonic.headless.core.chat.parser.SemanticParser=\
|
|||||||
com.tencent.supersonic.headless.core.chat.parser.QueryTypeParser
|
com.tencent.supersonic.headless.core.chat.parser.QueryTypeParser
|
||||||
|
|
||||||
com.tencent.supersonic.chat.server.parser.ChatParser=\
|
com.tencent.supersonic.chat.server.parser.ChatParser=\
|
||||||
com.tencent.supersonic.chat.server.parser.Text2PluginParser, \
|
com.tencent.supersonic.chat.server.parser.NL2PluginParser, \
|
||||||
com.tencent.supersonic.chat.server.parser.Text2SqlParser
|
com.tencent.supersonic.chat.server.parser.NL2SQLParser
|
||||||
|
|
||||||
com.tencent.supersonic.chat.server.executor.ChatExecutor=\
|
com.tencent.supersonic.chat.server.executor.ChatExecutor=\
|
||||||
com.tencent.supersonic.chat.server.executor.PluginExecutor, \
|
com.tencent.supersonic.chat.server.executor.PluginExecutor, \
|
||||||
@@ -42,10 +42,7 @@ com.tencent.supersonic.headless.core.cache.QueryCache=\
|
|||||||
|
|
||||||
com.tencent.supersonic.headless.server.processor.ResultProcessor=\
|
com.tencent.supersonic.headless.server.processor.ResultProcessor=\
|
||||||
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor, \
|
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor, \
|
||||||
com.tencent.supersonic.headless.server.processor.QueryRankProcessor, \
|
com.tencent.supersonic.headless.server.processor.SqlInfoProcessor
|
||||||
com.tencent.supersonic.headless.server.processor.SqlInfoProcessor, \
|
|
||||||
com.tencent.supersonic.headless.server.processor.TimeCostProcessor, \
|
|
||||||
com.tencent.supersonic.headless.server.processor.RespBuildProcessor
|
|
||||||
|
|
||||||
com.tencent.supersonic.headless.core.chat.parser.llm.DataSetResolver=\
|
com.tencent.supersonic.headless.core.chat.parser.llm.DataSetResolver=\
|
||||||
com.tencent.supersonic.headless.core.chat.parser.llm.HeuristicDataSetResolver
|
com.tencent.supersonic.headless.core.chat.parser.llm.HeuristicDataSetResolver
|
||||||
@@ -60,8 +57,9 @@ com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer=\
|
|||||||
com.tencent.supersonic.chat.server.plugin.recognize.embedding.EmbeddingRecallRecognizer
|
com.tencent.supersonic.chat.server.plugin.recognize.embedding.EmbeddingRecallRecognizer
|
||||||
|
|
||||||
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
|
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
|
||||||
com.tencent.supersonic.chat.server.processor.parse.RespBuildProcessor,\
|
com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor,\
|
||||||
com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor
|
com.tencent.supersonic.chat.server.processor.parse.EntityInfoProcessor,\
|
||||||
|
com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor
|
||||||
|
|
||||||
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
|
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
|
||||||
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
|
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ com.tencent.supersonic.headless.server.processor.ResultProcessor=\
|
|||||||
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor, \
|
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor, \
|
||||||
com.tencent.supersonic.headless.server.processor.QueryRankProcessor, \
|
com.tencent.supersonic.headless.server.processor.QueryRankProcessor, \
|
||||||
com.tencent.supersonic.headless.server.processor.SqlInfoProcessor, \
|
com.tencent.supersonic.headless.server.processor.SqlInfoProcessor, \
|
||||||
com.tencent.supersonic.headless.server.processor.TimeCostProcessor, \
|
com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor, \
|
||||||
com.tencent.supersonic.headless.server.processor.RespBuildProcessor
|
com.tencent.supersonic.headless.server.processor.RespBuildProcessor
|
||||||
|
|
||||||
com.tencent.supersonic.headless.core.chat.parser.llm.DataSetResolver=\
|
com.tencent.supersonic.headless.core.chat.parser.llm.DataSetResolver=\
|
||||||
|
|||||||
Reference in New Issue
Block a user