From 741ed4191b4eacfdaf8b8644563a11fb2813b5eb Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Sat, 30 Sep 2023 23:35:58 +0800 Subject: [PATCH] (improvement)(chat) support show sum metric in chinese name and support multiple conditions in having and if not exist metirc than not addAggregateToMetric (#165) --- .../chat/corrector/GlobalBeforeCorrector.java | 2 -- .../chat/corrector/GroupByCorrector.java | 11 +++---- .../jsqlparser/GroupByReplaceVisitor.java | 9 ++++-- .../jsqlparser/SqlParserUpdateHelper.java | 7 ++++- .../parser/convert/QueryReqConverter.java | 2 +- .../semantic/query/utils/QueryUtils.java | 31 +++++++++++++++++-- 6 files changed, 46 insertions(+), 16 deletions(-) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java index d185dc16e..2d9742fd3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java @@ -30,8 +30,6 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector { correctFieldName(semanticCorrectInfo); } - - private void replaceAlias(SemanticCorrectInfo semanticCorrectInfo) { String replaceAlias = SqlParserUpdateHelper.replaceAlias(semanticCorrectInfo.getSql()); semanticCorrectInfo.setSql(replaceAlias); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java index 12e4be4d5..a09075ce4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java @@ -30,21 +30,18 @@ public class GroupByCorrector extends BaseSemanticCorrector { if (!CollectionUtils.isEmpty(selectFields) && !CollectionUtils.isEmpty(metrics) - && selectFields.stream().anyMatch(s -> metrics.contains(s))) { - return; + && !selectFields.stream().anyMatch(s -> metrics.contains(s))) { + //add aggregate to all metric + addAggregateToMetric(semanticCorrectInfo); } - //add aggregate to all metric - addAggregateToMetric(semanticCorrectInfo); - //add dimension group by String sql = semanticCorrectInfo.getSql(); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); Set dimensions = semanticSchema.getDimensions(modelId).stream() - .filter(schemaElement -> !DateUtils.DATE_FIELD.equals(schemaElement.getBizName())) .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet()); - + dimensions.add(DateUtils.DATE_FIELD); selectFields = SqlParserSelectHelper.getSelectFields(sql); if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) { diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByReplaceVisitor.java index 6fb3d8f34..eb46506fc 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByReplaceVisitor.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.common.util.jsqlparser; import java.util.List; import java.util.Map; +import java.util.Objects; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -32,8 +33,12 @@ public class GroupByReplaceVisitor implements GroupByVisitor { for (int i = 0; i < groupByExpressions.size(); i++) { Expression expression = groupByExpressions.get(i); - - String replaceColumn = parseVisitorHelper.getReplaceColumn(expression.toString(), fieldNameMap, + String columnName = expression.toString(); + if (expression instanceof Function && Objects.nonNull( + ((Function) expression).getParameters().getExpressions().get(0))) { + columnName = ((Function) expression).getParameters().getExpressions().get(0).toString(); + } + String replaceColumn = parseVisitorHelper.getReplaceColumn(columnName, fieldNameMap, exactReplace); if (StringUtils.isNotEmpty(replaceColumn)) { if (expression instanceof Column) { diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java index 8ed43dff5..7c8095a96 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java @@ -468,7 +468,12 @@ public class SqlParserUpdateHelper { List waitingForAdds = visitor.getWaitingForAdds(); if (!CollectionUtils.isEmpty(waitingForAdds)) { for (Expression waitingForAdd : waitingForAdds) { - plainSelect.setHaving(waitingForAdd); + Expression having = plainSelect.getHaving(); + if (Objects.isNull(having)) { + plainSelect.setHaving(waitingForAdd); + } else { + plainSelect.setHaving(new AndExpression(having, waitingForAdd)); + } } } return selectStatement.toString(); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java index 7fe0399f1..fdcd5b982 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java @@ -100,7 +100,7 @@ public class QueryReqConverter { Map fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp); String sql = databaseReq.getSql(); log.info("convert name to bizName before:{}", sql); - String replaceFields = SqlParserUpdateHelper.replaceFields(sql, fieldNameToBizNameMap, true); + String replaceFields = SqlParserUpdateHelper.replaceFields(sql, fieldNameToBizNameMap, false); log.info("convert name to bizName after:{}", replaceFields); databaseReq.setSql(replaceFields); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java index f76fc2c97..5e87c7a54 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java @@ -3,11 +3,11 @@ package com.tencent.supersonic.semantic.query.utils; import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; import static com.tencent.supersonic.common.pojo.Constants.UNIONALL; -import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.PostConstruct; @@ -36,10 +37,13 @@ import org.springframework.util.CollectionUtils; @Component public class QueryUtils { + private static final String pattern = "\\(`(.*?)`\\)"; + + private static final String no_quotation_pattern = "\\((.*?)\\)"; + private final Set patterns = new HashSet<>(); @Value("${query.cache.enable:true}") private Boolean cacheEnable; - private final CacheUtils cacheUtils; private final StatUtils statUtils; @@ -86,6 +90,15 @@ public class QueryUtils { } if (namePair.containsKey(nameEn)) { column.setName(namePair.get(nameEn)); + } else { + String nameEnByRegex = getNameEnByRegex(nameEn, pattern); + if (StringUtils.isEmpty(nameEnByRegex)) { + nameEnByRegex = getNameEnByRegex(nameEn, no_quotation_pattern); + } + if (StringUtils.isNotEmpty(nameEnByRegex) && StringUtils.isNotEmpty(namePair.get(nameEnByRegex))) { + String filedName = namePair.get(nameEnByRegex); + column.setName(nameEn.replaceAll(nameEnByRegex, filedName)); + } } if (nameTypePair.containsKey(nameEn)) { column.setShowType(nameTypePair.get(nameEn)); @@ -103,6 +116,7 @@ public class QueryUtils { }); } + public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, QueryMultiStructReq queryMultiStructCmd) { List aggregators = queryMultiStructCmd.getQueryStructReqs().stream() @@ -142,6 +156,17 @@ public class QueryUtils { }); } + private String getNameEnByRegex(String nameEn, String pattern) { + Pattern p = Pattern.compile(pattern); + Matcher m = p.matcher(nameEn); + + if (m.find()) { + String result = m.group(1); + return result; + } + return null; + } + private boolean isNumberType(String type) { if (StringUtils.isBlank(type)) { return false;