topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2026-04-18 20:34:19 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) dsl supports revision (#254)

Browse Source
This commit is contained in:
mainmain
2023-10-18 17:44:14 +08:00
committed by GitHub
parent 7b861f563c
commit 7d770d2a6d
16 changed files with 343 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/CostType.java
View File

@@ -3,7 +3,8 @@ package com.tencent.supersonic.chat.persistence.dataobject;
public enum CostType {
MAPPER(1, "mapper"),
PARSER(2, "parser"),
QUERY(3, "query");
QUERY(3, "query"),
PARSERRESPONDER(4, "responder");
private Integer type;
private String name;

2
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/ChatParseMapper.java
View File

@@ -14,7 +14,7 @@ public interface ChatParseMapper {
boolean updateParseInfo(ChatParseDO chatParseDO);
ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
ChatParseDO getParseInfo(Long questionId, int parseId);
List<ChatParseDO> getParseInfoList(List<Long> questionIds);

2
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java
View File

@@ -34,7 +34,7 @@ public interface ChatQueryRepository {
List<SemanticParseInfo> candidateParses,
List<SemanticParseInfo> selectedParses);
public ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
public ChatParseDO getParseInfo(Long questionId, int parseId);
List<ChatParseDO> getParseInfoList(List<Long> questionIds);

6
chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java
View File

@@ -188,9 +188,9 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
return chatQueryDOMapper.updateByPrimaryKeyWithBLOBs(chatQueryDO);
}
@Override
public ChatParseDO getParseInfo(Long questionId, String userName, int parseId) {
return chatParseMapper.getParseInfo(questionId, userName, parseId);
public ChatParseDO getParseInfo(Long questionId, int parseId) {
return chatParseMapper.getParseInfo(questionId, parseId);
}
@Override

2
chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java
View File

@@ -62,7 +62,7 @@ public interface ChatService {
Boolean updateQuery(Long questionId, QueryResult queryResult, ChatContext chatCtx);
ChatParseDO getParseInfo(Long questionId, String userName, int parseId);
ChatParseDO getParseInfo(Long questionId, int parseId);
Boolean deleteChatQuery(Long questionId);

4
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java
View File

@@ -219,8 +219,8 @@ public class ChatServiceImpl implements ChatService {
return tempDate.format(new java.util.Date());
}
public ChatParseDO getParseInfo(Long questionId, String userName, int parseId) {
return chatQueryRepository.getParseInfo(questionId, userName, parseId);
public ChatParseDO getParseInfo(Long questionId, int parseId) {
return chatQueryRepository.getParseInfo(questionId, parseId);
}
public Boolean deleteChatQuery(Long questionId) {

228
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.service.impl;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
@@ -42,25 +43,41 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.knowledge.dictionary.MapResult;
import com.tencent.supersonic.knowledge.service.SearchService;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Objects;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.InExpression;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.schema.Column;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.compress.utils.Lists;
@@ -143,7 +160,15 @@ public class QueryServiceImpl implements QueryService {
.build();
}
for (ParseResponder parseResponder : parseResponders) {
Long startTime = System.currentTimeMillis();
parseResponder.fillResponse(parseResult, queryCtx, chatParseDOS);
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(parseResponder.getClass().getSimpleName())
.type(CostType.PARSERRESPONDER.getType()).build());
}
if (timeCostDOList.size() > 0) {
saveInfo(timeCostDOList, queryReq.getQueryText(), parseResult.getQueryId(),
queryReq.getUser().getName(), queryReq.getChatId().longValue());
}
chatService.updateChatParse(chatParseDOS);
return parseResult;
@@ -157,7 +182,7 @@ public class QueryServiceImpl implements QueryService {
}
private List<SemanticParseInfo> getTop5CandidateParseInfo(List<SemanticParseInfo> selectedParses,
List<SemanticParseInfo> candidateParses) {
List<SemanticParseInfo> candidateParses) {
if (CollectionUtils.isEmpty(selectedParses) || CollectionUtils.isEmpty(candidateParses)) {
return candidateParses;
}
@@ -182,7 +207,7 @@ public class QueryServiceImpl implements QueryService {
@Override
public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),
queryReq.getUser().getName(), queryReq.getParseId());
queryReq.getParseId());
ChatQueryDO chatQueryDO = chatService.getLastQuery(queryReq.getChatId());
List<StatisticsDO> timeCostDOList = new ArrayList<>();
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
@@ -303,7 +328,7 @@ public class QueryServiceImpl implements QueryService {
@Override
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException {
ChatParseDO chatParseDO = chatService.getParseInfo(queryData.getQueryId(),
queryData.getUser().getName(), queryData.getParseId());
queryData.getParseId());
SemanticParseInfo parseInfo = getSemanticParseInfo(queryData, chatParseDO);
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
@@ -316,21 +341,33 @@ public class QueryServiceImpl implements QueryService {
LLMResp llmResp = parseResult.getLlmResp();
String correctorSql = llmResp.getCorrectorSql();
log.info("correctorSql before replacing:{}", correctorSql);
List<FilterExpression> filterExpressionList = SqlParserSelectHelper.getFilterExpression(correctorSql);
updateFilters(filedNameToValueMap, filterExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters());
updateFilters(havingFiledNameToValueMap, filterExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters());
updateDateInfo(queryData, parseInfo, filedNameToValueMap, filterExpressionList);
List<FilterExpression> whereExpressionList = SqlParserSelectHelper.getWhereExpressions(correctorSql);
List<FilterExpression> havingExpressionList = SqlParserSelectHelper.getHavingExpressions(correctorSql);
List<Expression> addWhereConditions = new ArrayList<>();
List<Expression> addHavingConditions = new ArrayList<>();
Set<String> removeWhereFieldNames = new HashSet<>();
Set<String> removeHavingFieldNames = new HashSet<>();
updateFilters(filedNameToValueMap, whereExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addWhereConditions, removeWhereFieldNames);
updateDateInfo(queryData, parseInfo, filedNameToValueMap,
whereExpressionList, addWhereConditions, removeWhereFieldNames);
log.info("filedNameToValueMap:{}", filedNameToValueMap);
log.info("removeWhereFieldNames:{}", removeWhereFieldNames);
correctorSql = SqlParserReplaceHelper.replaceValue(correctorSql, filedNameToValueMap);
correctorSql = SqlParserRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames);
updateFilters(havingFiledNameToValueMap, havingExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addHavingConditions, removeHavingFieldNames);
log.info("havingFiledNameToValueMap:{}", havingFiledNameToValueMap);
log.info("removeHavingFieldNames:{}", removeHavingFieldNames);
correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap);
correctorSql = SqlParserRemoveHelper.removeHavingCondition(correctorSql, removeHavingFieldNames);
log.info("addWhereConditions:{}", addWhereConditions);
log.info("addHavingConditions:{}", addHavingConditions);
correctorSql = SqlParserAddHelper.addWhere(correctorSql, addWhereConditions);
correctorSql = SqlParserAddHelper.addHaving(correctorSql, addHavingConditions);
log.info("correctorSql after replacing:{}", correctorSql);
llmResp.setCorrectorSql(correctorSql);
parseResult.setLlmResp(llmResp);
@@ -344,34 +381,54 @@ public class QueryServiceImpl implements QueryService {
parseInfo.getSqlInfo().setQuerySql(explain.getSql());
}
}
log.info("parseInfo:{}", JsonUtil.toString(semanticQuery.getParseInfo().getProperties()));
semanticQuery.setParseInfo(parseInfo);
QueryResult queryResult = semanticQuery.execute(user);
queryResult.setChatContext(semanticQuery.getParseInfo());
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, user);
queryResult.setEntityInfo(entityInfo);
return queryResult;
}
@Override
public EntityInfo getEntityInfo(Long queryId, Integer parseId, User user) {
ChatParseDO chatParseDO = chatService.getParseInfo(queryId, user.getName(), parseId);
ChatParseDO chatParseDO = chatService.getParseInfo(queryId, parseId);
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
return semanticService.getEntityInfo(parseInfo, user);
}
private void updateDateInfo(QueryDataReq queryData, SemanticParseInfo parseInfo,
Map<String, Map<String, String>> filedNameToValueMap, List<FilterExpression> filterExpressionList) {
Map<String, Map<String, String>> filedNameToValueMap,
List<FilterExpression> filterExpressionList,
List<Expression> addConditions,
Set<String> removeFieldNames) {
if (Objects.isNull(queryData.getDateInfo())) {
return;
}
Map<String, String> map = new HashMap<>();
String dateField = DateUtils.DATE_FIELD;
if (queryData.getDateInfo().getUnit() > 1) {
queryData.getDateInfo().setStartDate(DateUtils.getBeforeDate(queryData.getDateInfo().getUnit() + 1));
queryData.getDateInfo().setEndDate(DateUtils.getBeforeDate(1));
}
if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) {
for (FilterExpression filterExpression : filterExpressionList) {
if (DateUtils.DATE_FIELD.equals(filterExpression.getFieldName())) {
dateField = filterExpression.getFieldName();
map.put(filterExpression.getFieldValue().toString(),
queryData.getDateInfo().getStartDate());
if (filterExpression.getOperator().equals(FilterOperatorEnum.EQUALS)) {
dateField = filterExpression.getFieldName();
map.put(filterExpression.getFieldValue().toString(),
queryData.getDateInfo().getStartDate());
filedNameToValueMap.put(dateField, map);
} else {
removeFieldNames.add(DateUtils.DATE_FIELD);
EqualsTo equalsTo = new EqualsTo();
Column column = new Column(DateUtils.DATE_FIELD);
StringValue stringValue = new StringValue(queryData.getDateInfo().getStartDate());
equalsTo.setLeftExpression(column);
equalsTo.setRightExpression(stringValue);
addConditions.add(equalsTo);
}
break;
}
}
@@ -389,38 +446,133 @@ public class QueryServiceImpl implements QueryService {
map.put(filterExpression.getFieldValue().toString(),
queryData.getDateInfo().getEndDate());
}
filedNameToValueMap.put(dateField, map);
if (FilterOperatorEnum.EQUALS.getValue().equals(filterExpression.getOperator())) {
removeFieldNames.add(DateUtils.DATE_FIELD);
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addTimeCondition(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions);
MinorThanEquals minorThanEquals = new MinorThanEquals();
addTimeCondition(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions);
}
}
}
}
filedNameToValueMap.put(dateField, map);
parseInfo.setDateInfo(queryData.getDateInfo());
}
public <T extends ComparisonOperator> void addTimeCondition(String date,
T comparisonExpression,
List<Expression> addConditions) {
Column column = new Column(DateUtils.DATE_FIELD);
StringValue stringValue = new StringValue(date);
comparisonExpression.setLeftExpression(column);
comparisonExpression.setRightExpression(stringValue);
addConditions.add(comparisonExpression);
}
private void updateFilters(Map<String, Map<String, String>> filedNameToValueMap,
List<FilterExpression> filterExpressionList, Set<QueryFilter> metricFilters,
Set<QueryFilter> contextMetricFilters) {
List<FilterExpression> filterExpressionList,
Set<QueryFilter> metricFilters,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions,
Set<String> removeFieldNames) {
if (CollectionUtils.isEmpty(metricFilters)) {
return;
}
for (QueryFilter queryFilter : metricFilters) {
for (QueryFilter dslQueryFilter : metricFilters) {
Map<String, String> map = new HashMap<>();
for (FilterExpression filterExpression : filterExpressionList) {
if (filterExpression.getFieldName() != null
&& filterExpression.getFieldName().contains(queryFilter.getName())
&& queryFilter.getOperator().getValue().equals(filterExpression.getOperator())) {
map.put(filterExpression.getFieldValue().toString(), queryFilter.getValue().toString());
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(queryFilter.getName())) {
o.setValue(queryFilter.getValue());
&& filterExpression.getFieldName().contains(dslQueryFilter.getName())) {
if (dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())
&& Objects.nonNull(dslQueryFilter.getValue())) {
map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString());
filedNameToValueMap.put(dslQueryFilter.getName(), map);
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {
o.setValue(dslQueryFilter.getValue());
}
});
} else {
removeFieldNames.add(dslQueryFilter.getName());
if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
EqualsTo equalsTo = new EqualsTo();
addWhereCondition(dslQueryFilter, equalsTo, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN_EQUALS)) {
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addWhereCondition(dslQueryFilter, greaterThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.GREATER_THAN)) {
GreaterThan greaterThan = new GreaterThan();
addWhereCondition(dslQueryFilter, greaterThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN_EQUALS)) {
MinorThanEquals minorThanEquals = new MinorThanEquals();
addWhereCondition(dslQueryFilter, minorThanEquals, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.MINOR_THAN)) {
MinorThan minorThan = new MinorThan();
addWhereCondition(dslQueryFilter, minorThan, contextMetricFilters, addConditions);
} else if (dslQueryFilter.getOperator().equals(FilterOperatorEnum.IN)) {
InExpression inExpression = new InExpression();
addWhereInCondition(dslQueryFilter, inExpression, contextMetricFilters, addConditions);
}
});
}
break;
}
}
filedNameToValueMap.put(queryFilter.getName(), map);
}
}
public void addWhereInCondition(QueryFilter dslQueryFilter,
InExpression inExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
Column column = new Column(dslQueryFilter.getName());
ExpressionList expressionList = new ExpressionList();
List<Expression> expressions = new ArrayList<>();
List<String> valueList = JsonUtil.toList(
JsonUtil.toString(dslQueryFilter.getValue()), String.class);
if (CollectionUtils.isEmpty(valueList)) {
return;
}
valueList.stream().forEach(o -> {
StringValue stringValue = new StringValue(o);
expressions.add(stringValue);
});
expressionList.setExpressions(expressions);
inExpression.setLeftExpression(column);
inExpression.setRightItemsList(expressionList);
addConditions.add(inExpression);
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {
o.setValue(dslQueryFilter.getValue());
o.setOperator(dslQueryFilter.getOperator());
}
});
}
public <T extends ComparisonOperator> void addWhereCondition(QueryFilter dslQueryFilter,
T comparisonExpression,
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
String columnName = dslQueryFilter.getName();
if (StringUtils.isNotBlank(dslQueryFilter.getFunction())) {
columnName = dslQueryFilter.getFunction() + "(" + dslQueryFilter.getName() + ")";
}
if (Objects.isNull(dslQueryFilter.getValue())) {
return;
}
Column column = new Column(columnName);
LongValue longValue = new LongValue(Long.parseLong(dslQueryFilter.getValue().toString()));
comparisonExpression.setLeftExpression(column);
comparisonExpression.setRightExpression(longValue);
addConditions.add(comparisonExpression);
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {
o.setValue(dslQueryFilter.getValue());
o.setOperator(dslQueryFilter.getOperator());
}
});
}
private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) {
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);

3
chat/core/src/main/resources/mapper/ChatParseMapper.xml
View File

@@ -34,8 +34,7 @@
<select id="getParseInfo" resultMap="ChatParse">
select *
from s2_chat_parse
where question_id = #{questionId} and user_name = #{userName}
and parse_id = #{parseId} limit 1
where question_id = #{questionId} and parse_id = #{parseId} limit 1
</select>
<select id="getParseInfoList" resultMap="ChatParse">

4
chat/core/src/main/resources/mapper/custom/ShowCaseCustomMapper.xml
View File

@@ -59,8 +59,8 @@
join (
select distinct chat_id
from s2_chat_query
where query_state = 1 and agent_id = ${agentId}
<if test="userName != null and userName != ''">
where query_state = 1 and agent_id = ${agentId} and (score is null or score > 1)
<if test="userName != null and userName != ''">
and user_name = #{userName}
</if>
order by chat_id desc