topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][Chat] Add TimeCorrector and rename the associated SqlParserHelper. (#707)

Browse Source
This commit is contained in:
lexluo09
2024-02-01 15:29:07 +08:00
committed by GitHub
parent 2c1c443b3e
commit 491c76368c
43 changed files with 749 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/MetricCheckProcessor.java
View File

@@ -15,8 +15,8 @@ import com.tencent.supersonic.chat.server.service.SemanticService;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
@@ -61,9 +61,9 @@ public class MetricCheckProcessor implements ParseResultProcessor {
public String processCorrectSql(SemanticParseInfo parseInfo, SemanticSchema semanticSchema) {
String correctSql = parseInfo.getSqlInfo().getCorrectS2SQL();
List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(correctSql);
List<String> metricFields = SqlParserSelectHelper.getAggregateFields(correctSql);
List<String> whereFields = SqlParserSelectHelper.getWhereFields(correctSql);
List<String> groupByFields = SqlSelectHelper.getGroupByFields(correctSql);
List<String> metricFields = SqlSelectHelper.getAggregateFields(correctSql);
List<String> whereFields = SqlSelectHelper.getWhereFields(correctSql);
List<String> dimensionFields = getDimensionFields(groupByFields, whereFields);
if (CollectionUtils.isEmpty(metricFields) || StringUtils.isBlank(correctSql)) {
return correctSql;
@@ -195,8 +195,8 @@ public class MetricCheckProcessor implements ParseResultProcessor {
}
private boolean checkHasMetric(String correctSql, SemanticSchema semanticSchema) {
List<String> selectFields = SqlParserSelectHelper.getSelectFields(correctSql);
List<String> aggFields = SqlParserSelectHelper.getAggregateFields(correctSql);
List<String> selectFields = SqlSelectHelper.getSelectFields(correctSql);
List<String> aggFields = SqlSelectHelper.getAggregateFields(correctSql);
List<String> collect = semanticSchema.getMetrics().stream()
.map(SchemaElement::getName).collect(Collectors.toList());
for (String field : selectFields) {
@@ -209,11 +209,11 @@ public class MetricCheckProcessor implements ParseResultProcessor {
private static String removeFieldInSql(String sql, Set<String> metricToRemove,
Set<String> dimensionByToRemove, Set<String> whereFieldsToRemove) {
sql = SqlParserRemoveHelper.removeWhereCondition(sql, whereFieldsToRemove);
sql = SqlParserRemoveHelper.removeSelect(sql, metricToRemove);
sql = SqlParserRemoveHelper.removeSelect(sql, dimensionByToRemove);
sql = SqlParserRemoveHelper.removeGroupBy(sql, dimensionByToRemove);
sql = SqlParserRemoveHelper.removeNumberFilter(sql);
sql = SqlRemoveHelper.removeWhereCondition(sql, whereFieldsToRemove);
sql = SqlRemoveHelper.removeSelect(sql, metricToRemove);
sql = SqlRemoveHelper.removeSelect(sql, dimensionByToRemove);
sql = SqlRemoveHelper.removeGroupBy(sql, dimensionByToRemove);
sql = SqlRemoveHelper.removeNumberFilter(sql);
return sql;
}

10
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java
View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -60,7 +60,7 @@ public class ParseInfoProcessor implements ParseResultProcessor {
if (correctS2SQL.equals(sqlInfo.getS2SQL())) {
return;
}
List<FieldExpression> expressions = SqlParserSelectHelper.getFilterExpression(correctS2SQL);
List<FieldExpression> expressions = SqlSelectHelper.getFilterExpression(correctS2SQL);
//set dataInfo
try {
if (!org.apache.commons.collections.CollectionUtils.isEmpty(expressions)) {
@@ -87,15 +87,15 @@ public class ParseInfoProcessor implements ParseResultProcessor {
if (Objects.isNull(semanticSchema)) {
return;
}
List<String> allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL()));
List<String> allFields = getFieldsExceptDate(SqlSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL()));
Set<SchemaElement> metrics = getElements(viewId, allFields, semanticSchema.getMetrics());
parseInfo.setMetrics(metrics);
if (QueryType.METRIC.equals(parseInfo.getQueryType())) {
List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL());
List<String> groupByFields = SqlSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL());
List<String> groupByDimensions = getFieldsExceptDate(groupByFields);
parseInfo.setDimensions(getElements(viewId, groupByDimensions, semanticSchema.getDimensions()));
} else if (QueryType.TAG.equals(parseInfo.getQueryType())) {
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL());
List<String> selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL());
List<String> selectDimensions = getFieldsExceptDate(selectFields);
parseInfo.setDimensions(getElements(viewId, selectDimensions, semanticSchema.getDimensions()));
}

28
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java
View File

@@ -60,10 +60,10 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import lombok.extern.slf4j.Slf4j;
@@ -318,7 +318,7 @@ public class QueryServiceImpl implements QueryService {
if (Objects.nonNull(parseInfo.getSqlInfo())
&& StringUtils.isNotBlank(parseInfo.getSqlInfo().getCorrectS2SQL())) {
String correctorSql = parseInfo.getSqlInfo().getCorrectS2SQL();
fields = SqlParserSelectHelper.getAllFields(correctorSql);
fields = SqlSelectHelper.getAllFields(correctorSql);
}
if (LLMSqlQuery.QUERY_MODE.equalsIgnoreCase(parseInfo.getQueryMode())
&& checkMetricReplace(fields, queryData.getMetrics())) {
@@ -373,8 +373,8 @@ public class QueryServiceImpl implements QueryService {
String correctorSql = parseInfo.getSqlInfo().getCorrectS2SQL();
log.info("correctorSql before replacing:{}", correctorSql);
// get where filter and having filter
List<FieldExpression> whereExpressionList = SqlParserSelectHelper.getWhereExpressions(correctorSql);
List<FieldExpression> havingExpressionList = SqlParserSelectHelper.getHavingExpressions(correctorSql);
List<FieldExpression> whereExpressionList = SqlSelectHelper.getWhereExpressions(correctorSql);
List<FieldExpression> havingExpressionList = SqlSelectHelper.getHavingExpressions(correctorSql);
List<Expression> addWhereConditions = new ArrayList<>();
List<Expression> addHavingConditions = new ArrayList<>();
Set<String> removeWhereFieldNames = new HashSet<>();
@@ -384,16 +384,16 @@ public class QueryServiceImpl implements QueryService {
parseInfo.getDimensionFilters(), addWhereConditions, removeWhereFieldNames);
updateDateInfo(queryData, parseInfo, filedNameToValueMap,
whereExpressionList, addWhereConditions, removeWhereFieldNames);
correctorSql = SqlParserReplaceHelper.replaceValue(correctorSql, filedNameToValueMap);
correctorSql = SqlParserRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames);
correctorSql = SqlReplaceHelper.replaceValue(correctorSql, filedNameToValueMap);
correctorSql = SqlRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames);
// replace having filter
updateFilters(havingExpressionList, queryData.getDimensionFilters(),
parseInfo.getDimensionFilters(), addHavingConditions, removeHavingFieldNames);
correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap);
correctorSql = SqlParserRemoveHelper.removeHavingCondition(correctorSql, removeHavingFieldNames);
correctorSql = SqlReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap);
correctorSql = SqlRemoveHelper.removeHavingCondition(correctorSql, removeHavingFieldNames);
correctorSql = SqlParserAddHelper.addWhere(correctorSql, addWhereConditions);
correctorSql = SqlParserAddHelper.addHaving(correctorSql, addHavingConditions);
correctorSql = SqlAddHelper.addWhere(correctorSql, addWhereConditions);
correctorSql = SqlAddHelper.addHaving(correctorSql, addHavingConditions);
log.info("correctorSql after replacing:{}", correctorSql);
return correctorSql;
}
@@ -407,7 +407,7 @@ public class QueryServiceImpl implements QueryService {
Map<String, Pair<String, String>> fieldMap = new HashMap<>();
if (CollectionUtils.isNotEmpty(oriMetrics) && !oriMetrics.contains(metric.getName())) {
fieldMap.put(oriMetrics.get(0), Pair.of(metric.getName(), metric.getDefaultAgg()));
correctorSql = SqlParserReplaceHelper.replaceAggFields(correctorSql, fieldMap);
correctorSql = SqlReplaceHelper.replaceAggFields(correctorSql, fieldMap);
}
log.info("after replaceMetrics:{}", correctorSql);
parseInfo.getSqlInfo().setCorrectS2SQL(correctorSql);