(improvement)(headless)Rename QueryTextReq to QueryNLReq to explicitly reflect natural language concept.

This commit is contained in:
jerryjzhang
2024-07-08 10:20:20 +08:00
parent 9911e6772c
commit efd617b2e5
12 changed files with 83 additions and 84 deletions

View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
@@ -69,11 +69,11 @@ public class NL2SQLParser implements ChatParser {
}
processMultiTurn(chatParseContext);
QueryTextReq queryTextReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
addDynamicExemplars(chatParseContext.getAgent().getId(), queryTextReq);
QueryNLReq queryNLReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
addDynamicExemplars(chatParseContext.getAgent().getId(), queryNLReq);
ChatQueryService chatQueryService = ContextUtils.getBean(ChatQueryService.class);
ParseResp text2SqlParseResp = chatQueryService.performParsing(queryTextReq);
ParseResp text2SqlParseResp = chatQueryService.performParsing(queryNLReq);
if (!ParseResp.ParseState.FAILED.equals(text2SqlParseResp.getState())) {
parseResp.getSelectedParses().addAll(text2SqlParseResp.getSelectedParses());
}
@@ -149,8 +149,8 @@ public class NL2SQLParser implements ChatParser {
// derive mapping result of current question and parsing result of last question.
ChatQueryService chatQueryService = ContextUtils.getBean(ChatQueryService.class);
QueryTextReq queryTextReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
MapResp currentMapResult = chatQueryService.performMapping(queryTextReq);
QueryNLReq queryNLReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
MapResp currentMapResult = chatQueryService.performMapping(queryNLReq);
List<ParseResp> historyParseResults = getHistoryParseResult(chatParseContext.getChatId(), 1);
if (historyParseResults.size() == 0) {
@@ -168,7 +168,7 @@ public class NL2SQLParser implements ChatParser {
.curtSchema(curtMapStr)
.histSchema(histMapStr)
.histSQL(histSQL)
.modelConfig(queryTextReq.getModelConfig())
.modelConfig(queryNLReq.getModelConfig())
.build());
chatParseContext.setQueryText(rewrittenQuery);
log.info("Last Query: {} Current Query: {}, Rewritten Query: {}",
@@ -225,13 +225,13 @@ public class NL2SQLParser implements ChatParser {
return contextualList;
}
private void addDynamicExemplars(Integer agentId, QueryTextReq queryTextReq) {
private void addDynamicExemplars(Integer agentId, QueryNLReq queryNLReq) {
ExemplarServiceImpl exemplarManager = ContextUtils.getBean(ExemplarServiceImpl.class);
EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class);
String memoryCollectionName = embeddingConfig.getMemoryCollectionName(agentId);
List<SqlExemplar> exemplars = exemplarManager.recallExemplars(memoryCollectionName,
queryTextReq.getQueryText(), 5);
queryTextReq.getDynamicExemplars().addAll(exemplars);
queryNLReq.getQueryText(), 5);
queryNLReq.getDynamicExemplars().addAll(exemplars);
}
@Builder

View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.chat.server.service.ChatService;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
@@ -76,7 +76,7 @@ public class ChatQueryController {
}
@PostMapping("queryContext")
public Object queryContext(@RequestBody QueryTextReq queryCtx,
public Object queryContext(@RequestBody QueryNLReq queryCtx,
HttpServletRequest request, HttpServletResponse response) {
queryCtx.setUser(UserHolder.findUser(request, response));
return chatService.queryContext(queryCtx.getChatId());

View File

@@ -21,7 +21,7 @@ import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
@@ -60,8 +60,8 @@ public class ChatServiceImpl implements ChatService {
if (!agent.enableSearch()) {
return Lists.newArrayList();
}
QueryTextReq queryTextReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
return retrieveService.retrieve(queryTextReq);
QueryNLReq queryNLReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
return retrieveService.retrieve(queryNLReq);
}
@Override
@@ -137,8 +137,8 @@ public class ChatServiceImpl implements ChatService {
}
private void supplyMapInfo(ChatParseContext chatParseContext) {
QueryTextReq queryTextReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
MapResp mapResp = chatQueryService.performMapping(queryTextReq);
QueryNLReq queryNLReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext);
MapResp mapResp = chatQueryService.performMapping(queryNLReq);
chatParseContext.setMapInfo(mapResp.getMapInfo());
}

View File

@@ -4,35 +4,35 @@ import com.tencent.supersonic.chat.server.agent.Agent;
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
import com.tencent.supersonic.common.pojo.enums.Text2SQLType;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import org.apache.commons.collections.MapUtils;
import java.util.Objects;
public class QueryReqConverter {
public static QueryTextReq buildText2SqlQueryReq(ChatParseContext chatParseContext) {
QueryTextReq queryTextReq = new QueryTextReq();
BeanMapper.mapper(chatParseContext, queryTextReq);
public static QueryNLReq buildText2SqlQueryReq(ChatParseContext chatParseContext) {
QueryNLReq queryNLReq = new QueryNLReq();
BeanMapper.mapper(chatParseContext, queryNLReq);
Agent agent = chatParseContext.getAgent();
if (agent == null) {
return queryTextReq;
return queryNLReq;
}
if (agent.containsLLMParserTool() && agent.containsRuleTool()) {
queryTextReq.setText2SQLType(Text2SQLType.RULE_AND_LLM);
queryNLReq.setText2SQLType(Text2SQLType.RULE_AND_LLM);
} else if (agent.containsLLMParserTool()) {
queryTextReq.setText2SQLType(Text2SQLType.ONLY_LLM);
queryNLReq.setText2SQLType(Text2SQLType.ONLY_LLM);
} else if (agent.containsRuleTool()) {
queryTextReq.setText2SQLType(Text2SQLType.ONLY_RULE);
queryNLReq.setText2SQLType(Text2SQLType.ONLY_RULE);
}
queryTextReq.setDataSetIds(agent.getDataSetIds());
if (Objects.nonNull(queryTextReq.getMapInfo())
&& MapUtils.isNotEmpty(queryTextReq.getMapInfo().getDataSetElementMatches())) {
queryTextReq.setMapInfo(queryTextReq.getMapInfo());
queryNLReq.setDataSetIds(agent.getDataSetIds());
if (Objects.nonNull(queryNLReq.getMapInfo())
&& MapUtils.isNotEmpty(queryNLReq.getMapInfo().getDataSetElementMatches())) {
queryNLReq.setMapInfo(queryNLReq.getMapInfo());
}
queryTextReq.setModelConfig(agent.getModelConfig());
queryTextReq.setPromptConfig(agent.getPromptConfig());
return queryTextReq;
queryNLReq.setModelConfig(agent.getModelConfig());
queryNLReq.setPromptConfig(agent.getPromptConfig());
return queryNLReq;
}
}

View File

@@ -16,7 +16,7 @@ import java.util.List;
import java.util.Set;
@Data
public class QueryTextReq {
public class QueryNLReq {
private String queryText;
private Integer chatId;
private Set<Long> dataSetIds = Sets.newHashSet();

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.chat.parser.llm;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp;
import lombok.AllArgsConstructor;
@@ -22,7 +22,7 @@ public class ParseResult {
private LLMResp llmResp;
private QueryTextReq request;
private QueryNLReq request;
private List<LLMReq.ElementValue> linkingValues;
}

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.server.facade.rest;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import com.tencent.supersonic.headless.server.facade.service.RetrieveService;
@@ -26,27 +26,27 @@ public class ChatQueryApiController {
private RetrieveService retrieveService;
@PostMapping("/chat/search")
public Object search(@RequestBody QueryTextReq queryTextReq,
public Object search(@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
queryTextReq.setUser(UserHolder.findUser(request, response));
return retrieveService.retrieve(queryTextReq);
queryNLReq.setUser(UserHolder.findUser(request, response));
return retrieveService.retrieve(queryNLReq);
}
@PostMapping("/chat/map")
public MapResp map(@RequestBody QueryTextReq queryTextReq,
public MapResp map(@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response) {
queryTextReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.performMapping(queryTextReq);
queryNLReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.performMapping(queryNLReq);
}
@PostMapping("/chat/parse")
public Object parse(@RequestBody QueryTextReq queryTextReq,
public Object parse(@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
queryTextReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.performParsing(queryTextReq);
queryNLReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.performParsing(queryNLReq);
}
}

View File

@@ -7,23 +7,23 @@ import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
/***
/***dd
* SemanticLayerService for query and search
*/
public interface ChatQueryService {
MapResp performMapping(QueryTextReq queryTextReq);
MapResp performMapping(QueryNLReq queryNLReq);
MapInfoResp map(QueryMapReq queryMapReq);
ParseResp performParsing(QueryTextReq queryTextReq);
ParseResp performParsing(QueryNLReq queryNLReq);
@Deprecated
QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception;

View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.headless.server.facade.service;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import java.util.List;
public interface RetrieveService {
List<SearchResult> retrieve(QueryTextReq queryCtx);
List<SearchResult> retrieve(QueryNLReq queryCtx);
}

View File

@@ -29,7 +29,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.CostType;
import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.request.TranslateSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
@@ -118,40 +118,40 @@ public class ChatQueryServiceImpl implements ChatQueryService {
private ChatWorkflowEngine chatWorkflowEngine;
@Override
public MapResp performMapping(QueryTextReq queryTextReq) {
public MapResp performMapping(QueryNLReq queryNLReq) {
MapResp mapResp = new MapResp();
ChatQueryContext queryCtx = buildQueryContext(queryTextReq);
ChatQueryContext queryCtx = buildChatQueryContext(queryNLReq);
ComponentFactory.getSchemaMappers().forEach(mapper -> {
mapper.map(queryCtx);
});
SchemaMapInfo mapInfo = queryCtx.getMapInfo();
mapResp.setMapInfo(mapInfo);
mapResp.setQueryText(queryTextReq.getQueryText());
mapResp.setQueryText(queryNLReq.getQueryText());
return mapResp;
}
@Override
public MapInfoResp map(QueryMapReq queryMapReq) {
QueryTextReq queryTextReq = new QueryTextReq();
BeanUtils.copyProperties(queryMapReq, queryTextReq);
QueryNLReq queryNLReq = new QueryNLReq();
BeanUtils.copyProperties(queryMapReq, queryNLReq);
List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
queryTextReq.setDataSetIds(dataSetIds);
MapResp mapResp = performMapping(queryTextReq);
queryNLReq.setDataSetIds(dataSetIds);
MapResp mapResp = performMapping(queryNLReq);
dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet());
return convert(mapResp, queryMapReq.getTopN(), dataSetIds);
}
@Override
public ParseResp performParsing(QueryTextReq queryTextReq) {
ParseResp parseResult = new ParseResp(queryTextReq.getChatId(), queryTextReq.getQueryText());
public ParseResp performParsing(QueryNLReq queryNLReq) {
ParseResp parseResult = new ParseResp(queryNLReq.getChatId(), queryNLReq.getQueryText());
// build queryContext and chatContext
ChatQueryContext queryCtx = buildQueryContext(queryTextReq);
ChatQueryContext queryCtx = buildChatQueryContext(queryNLReq);
// in order to support multi-turn conversation, chat context is needed
ChatContext chatCtx = chatContextService.getOrCreateContext(queryTextReq.getChatId());
ChatContext chatCtx = chatContextService.getOrCreateContext(queryNLReq.getChatId());
chatWorkflowEngine.execute(queryCtx, chatCtx, parseResult);
@@ -161,21 +161,20 @@ public class ChatQueryServiceImpl implements ChatQueryService {
return parseResult;
}
public ChatQueryContext buildQueryContext(QueryTextReq queryTextReq) {
public ChatQueryContext buildChatQueryContext(QueryNLReq queryNLReq) {
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds();
ChatQueryContext queryCtx = ChatQueryContext.builder()
.queryFilters(queryTextReq.getQueryFilters())
.queryFilters(queryNLReq.getQueryFilters())
.semanticSchema(semanticSchema)
.candidateQueries(new ArrayList<>())
.mapInfo(new SchemaMapInfo())
.modelIdToDataSetIds(modelIdToDataSetIds)
.text2SQLType(queryTextReq.getText2SQLType())
.mapModeEnum(queryTextReq.getMapModeEnum())
.dataSetIds(queryTextReq.getDataSetIds())
.text2SQLType(queryNLReq.getText2SQLType())
.mapModeEnum(queryNLReq.getMapModeEnum())
.dataSetIds(queryNLReq.getDataSetIds())
.build();
BeanUtils.copyProperties(queryTextReq, queryCtx);
BeanUtils.copyProperties(queryNLReq, queryCtx);
return queryCtx;
}
@@ -278,9 +277,9 @@ public class ChatQueryServiceImpl implements ChatQueryService {
validFilter(semanticQuery.getParseInfo().getMetricFilters());
//init s2sql
semanticQuery.initS2Sql(semanticSchema, user);
QueryTextReq queryTextReq = new QueryTextReq();
queryTextReq.setQueryFilters(new QueryFilters());
queryTextReq.setUser(user);
QueryNLReq queryNLReq = new QueryNLReq();
queryNLReq.setQueryFilters(new QueryFilters());
queryNLReq.setUser(user);
}
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
QueryResult queryResult = doExecution(semanticQueryReq, semanticQuery.getParseInfo(), user);

View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
import com.tencent.supersonic.headless.api.pojo.request.QueryTextReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
@@ -64,9 +64,9 @@ public class RetrieveServiceImpl implements RetrieveService {
@Autowired
private SearchMatchStrategy searchMatchStrategy;
@Override
public List<SearchResult> retrieve(QueryTextReq queryTextReq) {
public List<SearchResult> retrieve(QueryNLReq queryNLReq) {
String queryText = queryTextReq.getQueryText();
String queryText = queryNLReq.getQueryText();
// 1.get meta info
SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema();
List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
@@ -76,10 +76,10 @@ public class RetrieveServiceImpl implements RetrieveService {
// 2.detect by segment
List<S2Term> originals = knowledgeBaseService.getTerms(queryText, modelIdToDataSetIds);
log.debug("hanlp parse result: {}", originals);
Set<Long> dataSetIds = queryTextReq.getDataSetIds();
Set<Long> dataSetIds = queryNLReq.getDataSetIds();
ChatQueryContext chatQueryContext = new ChatQueryContext();
BeanUtils.copyProperties(queryTextReq, chatQueryContext);
BeanUtils.copyProperties(queryNLReq, chatQueryContext);
chatQueryContext.setModelIdToDataSetIds(dataSetService.getModelIdToDataSetIds());
Map<MatchText, List<HanlpMapResult>> regTextMap =
@@ -100,12 +100,12 @@ public class RetrieveServiceImpl implements RetrieveService {
return Lists.newArrayList();
}
Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry = mostSimilarSearchResult.get();
log.debug("searchTextEntry:{},queryTextReq:{}", searchTextEntry, queryTextReq);
log.debug("searchTextEntry:{},queryNLReq:{}", searchTextEntry, queryNLReq);
Set<SearchResult> searchResults = new LinkedHashSet();
DataSetInfoStat dataSetInfoStat = NatureHelper.getDataSetStat(originals);
List<Long> possibleDataSets = getPossibleDataSets(queryTextReq, originals, dataSetInfoStat, dataSetIds);
List<Long> possibleDataSets = getPossibleDataSets(queryNLReq, originals, dataSetInfoStat, dataSetIds);
// 5.1 priority dimension metric
boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleDataSets), dataSetIdToName,
@@ -120,14 +120,14 @@ public class RetrieveServiceImpl implements RetrieveService {
Set<SearchResult> searchResultSet = searchDimensionValue(metricsDb, dataSetIdToName,
dataSetInfoStat.getMetricDataSetCount(), existMetricAndDimension,
matchText, natureToNameMap, natureToNameEntry, queryTextReq.getQueryFilters());
matchText, natureToNameMap, natureToNameEntry, queryNLReq.getQueryFilters());
searchResults.addAll(searchResultSet);
}
return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList());
}
private List<Long> getPossibleDataSets(QueryTextReq queryCtx, List<S2Term> originals,
private List<Long> getPossibleDataSets(QueryNLReq queryCtx, List<S2Term> originals,
DataSetInfoStat dataSetInfoStat, Set<Long> dataSetIds) {
if (CollectionUtils.isNotEmpty(dataSetIds)) {
return new ArrayList<>(dataSetIds);

View File

@@ -22,9 +22,9 @@ import java.util.Arrays;
import java.util.List;
/**
* QueryTextReqBuilderTest
* QueryNLReqBuilderTest
*/
class QueryTextReqBuilderTest {
class QueryNLReqBuilderTest {
@Test
void buildS2SQLReq() {