From 491c76368cd91dccca145a7a5c13b4865b33813a Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:29:07 +0800 Subject: [PATCH] [improvement][Chat] Add TimeCorrector and rename the associated SqlParserHelper. (#707) --- .../core/corrector/BaseSemanticCorrector.java | 20 +-- .../chat/core/corrector/GroupByCorrector.java | 16 +- .../chat/core/corrector/HavingCorrector.java | 14 +- .../chat/core/corrector/SchemaCorrector.java | 14 +- .../chat/core/corrector/SelectCorrector.java | 6 +- .../chat/core/corrector/TimeCorrector.java | 57 +++++++ .../chat/core/corrector/WhereCorrector.java | 24 +-- .../chat/core/parser/QueryTypeParser.java | 6 +- .../parser/sql/llm/LLMResponseService.java | 4 +- .../core/corrector/TimeCorrectorTest.java | 100 ++++++++++++ .../processor/parse/MetricCheckProcessor.java | 24 +-- .../processor/parse/ParseInfoProcessor.java | 10 +- .../server/service/impl/QueryServiceImpl.java | 28 ++-- .../common/util/jsqlparser/DateVisitor.java | 90 +++++++++++ .../jsqlparser/FieldlValueReplaceVisitor.java | 2 +- .../FunctionAliasReplaceVisitor.java | 2 +- ...ParserAddHelper.java => SqlAddHelper.java} | 38 ++--- .../util/jsqlparser/SqlDateSelectHelper.java | 35 +++++ ...erEqualHelper.java => SqlEqualHelper.java} | 26 +-- ...RemoveHelper.java => SqlRemoveHelper.java} | 42 ++--- ...placeHelper.java => SqlReplaceHelper.java} | 42 ++--- ...lper.java => SqlSelectFunctionHelper.java} | 6 +- ...SelectHelper.java => SqlSelectHelper.java} | 51 ++---- .../supersonic/common/util/DateUtilsTest.java | 14 +- ...dHelperTest.java => SqlAddHelperTest.java} | 148 +++++++++--------- .../jsqlparser/SqlDateSelectHelperTest.java | 46 ++++++ ...elperTest.java => SqlEqualHelperTest.java} | 14 +- ...lperTest.java => SqlRemoveHelperTest.java} | 24 +-- ...perTest.java => SqlReplaceHelperTest.java} | 138 ++++++++-------- ....java => SqlSelectFunctionHelperTest.java} | 28 ++-- ...lperTest.java => SqlSelectHelperTest.java} | 88 +++++------ .../api/pojo/request/QueryStructReq.java | 8 +- .../core/adaptor/db/ClickHouseAdaptor.java | 4 +- .../core/adaptor/db/PostgresqlAdaptor.java | 4 +- .../headless/core/utils/SqlGenerateUtils.java | 8 +- .../server/aspect/DimValueAspect.java | 8 +- .../server/aspect/S2DataPermissionAspect.java | 4 +- .../server/utils/MetricCheckUtils.java | 4 +- .../server/utils/QueryReqConverter.java | 22 +-- .../server/utils/QueryStructUtils.java | 8 +- .../headless/server/utils/StatUtils.java | 6 +- .../main/resources/META-INF/spring.factories | 1 + .../main/resources/META-INF/spring.factories | 1 + 43 files changed, 749 insertions(+), 486 deletions(-) create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/TimeCorrector.java create mode 100644 chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/DateVisitor.java rename common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserAddHelper.java => SqlAddHelper.java} (90%) create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlDateSelectHelper.java rename common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserEqualHelper.java => SqlEqualHelper.java} (51%) rename common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserRemoveHelper.java => SqlRemoveHelper.java} (89%) rename common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserReplaceHelper.java => SqlReplaceHelper.java} (93%) rename common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserSelectFunctionHelper.java => SqlSelectFunctionHelper.java} (94%) rename common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserSelectHelper.java => SqlSelectHelper.java} (91%) rename common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserAddHelperTest.java => SqlAddHelperTest.java} (73%) create mode 100644 common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlDateSelectHelperTest.java rename common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserEqualHelperTest.java => SqlEqualHelperTest.java} (82%) rename common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserRemoveHelperTest.java => SqlRemoveHelperTest.java} (86%) rename common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserReplaceHelperTest.java => SqlReplaceHelperTest.java} (83%) rename common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserSelectFunctionHelperTest.java => SqlSelectFunctionHelperTest.java} (72%) rename common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/{SqlParserSelectHelperTest.java => SqlSelectHelperTest.java} (78%) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java index 2215efb58..80c701b19 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java @@ -6,9 +6,9 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -75,14 +75,14 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { } protected void addFieldsToSelect(SemanticParseInfo semanticParseInfo, String correctS2SQL) { - Set selectFields = new HashSet<>(SqlParserSelectHelper.getSelectFields(correctS2SQL)); - Set needAddFields = new HashSet<>(SqlParserSelectHelper.getGroupByFields(correctS2SQL)); - needAddFields.addAll(SqlParserSelectHelper.getOrderByFields(correctS2SQL)); + Set selectFields = new HashSet<>(SqlSelectHelper.getSelectFields(correctS2SQL)); + Set needAddFields = new HashSet<>(SqlSelectHelper.getGroupByFields(correctS2SQL)); + needAddFields.addAll(SqlSelectHelper.getOrderByFields(correctS2SQL)); // If there is no aggregate function in the S2SQL statement and // there is a data field in 'WHERE' statement, add the field to the 'SELECT' statement. - if (!SqlParserSelectFunctionHelper.hasAggregateFunction(correctS2SQL)) { - List whereFields = SqlParserSelectHelper.getWhereFields(correctS2SQL); + if (!SqlSelectFunctionHelper.hasAggregateFunction(correctS2SQL)) { + List whereFields = SqlSelectHelper.getWhereFields(correctS2SQL); List timeChNameList = TimeDimensionEnum.getChNameList(); Set timeFields = whereFields.stream().filter(field -> timeChNameList.contains(field)) .collect(Collectors.toSet()); @@ -94,7 +94,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { } needAddFields.removeAll(selectFields); - String replaceFields = SqlParserAddHelper.addFieldsToSelect(correctS2SQL, new ArrayList<>(needAddFields)); + String replaceFields = SqlAddHelper.addFieldsToSelect(correctS2SQL, new ArrayList<>(needAddFields)); semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceFields); } @@ -123,7 +123,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { if (CollectionUtils.isEmpty(metricToAggregate)) { return; } - String aggregateSql = SqlParserAddHelper.addAggregateToField(correctS2SQL, metricToAggregate); + String aggregateSql = SqlAddHelper.addAggregateToField(correctS2SQL, metricToAggregate); semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java index 7fd95e1ef..9b9f0681b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java @@ -5,8 +5,8 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -35,7 +35,7 @@ public class GroupByCorrector extends BaseSemanticCorrector { String correctS2SQL = sqlInfo.getCorrectS2SQL(); SemanticSchema semanticSchema = queryContext.getSemanticSchema(); // check if has distinct - boolean hasDistinct = SqlParserSelectHelper.hasDistinct(correctS2SQL); + boolean hasDistinct = SqlSelectHelper.hasDistinct(correctS2SQL); if (hasDistinct) { log.info("not add group by ,exist distinct in correctS2SQL:{}", correctS2SQL); return; @@ -54,7 +54,7 @@ public class GroupByCorrector extends BaseSemanticCorrector { ).collect(Collectors.toSet()); dimensions.add(TimeDimensionEnum.DAY.getChName()); - List selectFields = SqlParserSelectHelper.getSelectFields(correctS2SQL); + List selectFields = SqlSelectHelper.getSelectFields(correctS2SQL); if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) { return; @@ -63,12 +63,12 @@ public class GroupByCorrector extends BaseSemanticCorrector { if (selectFields.size() == 1 && selectFields.contains(TimeDimensionEnum.DAY.getChName())) { return; } - if (SqlParserSelectHelper.hasGroupBy(correctS2SQL)) { + if (SqlSelectHelper.hasGroupBy(correctS2SQL)) { log.info("not add group by ,exist group by in correctS2SQL:{}", correctS2SQL); return; } - List aggregateFields = SqlParserSelectHelper.getAggregateFields(correctS2SQL); + List aggregateFields = SqlSelectHelper.getAggregateFields(correctS2SQL); Set groupByFields = selectFields.stream() .filter(field -> dimensions.contains(field)) .filter(field -> { @@ -78,13 +78,13 @@ public class GroupByCorrector extends BaseSemanticCorrector { return true; }) .collect(Collectors.toSet()); - semanticParseInfo.getSqlInfo().setCorrectS2SQL(SqlParserAddHelper.addGroupBy(correctS2SQL, groupByFields)); + semanticParseInfo.getSqlInfo().setCorrectS2SQL(SqlAddHelper.addGroupBy(correctS2SQL, groupByFields)); addAggregate(queryContext, semanticParseInfo); } private void addAggregate(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - List sqlGroupByFields = SqlParserSelectHelper.getGroupByFields( + List sqlGroupByFields = SqlSelectHelper.getGroupByFields( semanticParseInfo.getSqlInfo().getCorrectS2SQL()); if (CollectionUtils.isEmpty(sqlGroupByFields)) { return; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java index 89e57f0ca..b4e40aa95 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/HavingCorrector.java @@ -3,9 +3,9 @@ package com.tencent.supersonic.chat.core.corrector; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import org.springframework.util.CollectionUtils; @@ -42,18 +42,18 @@ public class HavingCorrector extends BaseSemanticCorrector { if (CollectionUtils.isEmpty(metrics)) { return; } - String havingSql = SqlParserAddHelper.addHaving(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), metrics); + String havingSql = SqlAddHelper.addHaving(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), metrics); semanticParseInfo.getSqlInfo().setCorrectS2SQL(havingSql); } private void addHavingToSelect(SemanticParseInfo semanticParseInfo) { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - if (!SqlParserSelectFunctionHelper.hasAggregateFunction(correctS2SQL)) { + if (!SqlSelectFunctionHelper.hasAggregateFunction(correctS2SQL)) { return; } - List havingExpressionList = SqlParserSelectHelper.getHavingExpression(correctS2SQL); + List havingExpressionList = SqlSelectHelper.getHavingExpression(correctS2SQL); if (!CollectionUtils.isEmpty(havingExpressionList)) { - String replaceSql = SqlParserAddHelper.addFunctionToSelect(correctS2SQL, havingExpressionList); + String replaceSql = SqlAddHelper.addFunctionToSelect(correctS2SQL, havingExpressionList); semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceSql); } return; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java index 85f3ef730..3b657f792 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SchemaCorrector.java @@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.jsqlparser.AggregateEnum; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -19,7 +19,7 @@ import java.util.Set; import java.util.stream.Collectors; /** - * Perform schema corrections on the Schema information in S2QL. + * Perform schema corrections on the Schema information in S2SQL. */ @Slf4j public class SchemaCorrector extends BaseSemanticCorrector { @@ -41,20 +41,20 @@ public class SchemaCorrector extends BaseSemanticCorrector { private void correctAggFunction(SemanticParseInfo semanticParseInfo) { Map aggregateEnum = AggregateEnum.getAggregateEnum(); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); - String sql = SqlParserReplaceHelper.replaceFunction(sqlInfo.getCorrectS2SQL(), aggregateEnum); + String sql = SqlReplaceHelper.replaceFunction(sqlInfo.getCorrectS2SQL(), aggregateEnum); sqlInfo.setCorrectS2SQL(sql); } private void replaceAlias(SemanticParseInfo semanticParseInfo) { SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); - String replaceAlias = SqlParserReplaceHelper.replaceAlias(sqlInfo.getCorrectS2SQL()); + String replaceAlias = SqlReplaceHelper.replaceAlias(sqlInfo.getCorrectS2SQL()); sqlInfo.setCorrectS2SQL(replaceAlias); } private void correctFieldName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { Map fieldNameMap = getFieldNameMap(queryContext, semanticParseInfo.getViewId()); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); - String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap); + String sql = SqlReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap); sqlInfo.setCorrectS2SQL(sql); } @@ -70,7 +70,7 @@ public class SchemaCorrector extends BaseSemanticCorrector { SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); - String sql = SqlParserReplaceHelper.replaceFieldNameByValue(sqlInfo.getCorrectS2SQL(), fieldValueToFieldNames); + String sql = SqlReplaceHelper.replaceFieldNameByValue(sqlInfo.getCorrectS2SQL(), fieldValueToFieldNames); sqlInfo.setCorrectS2SQL(sql); } @@ -102,7 +102,7 @@ public class SchemaCorrector extends BaseSemanticCorrector { ))); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); - String sql = SqlParserReplaceHelper.replaceValue(sqlInfo.getCorrectS2SQL(), filedNameToValueMap, false); + String sql = SqlReplaceHelper.replaceValue(sqlInfo.getCorrectS2SQL(), filedNameToValueMap, false); sqlInfo.setCorrectS2SQL(sql); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SelectCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SelectCorrector.java index 56fbcf2af..6e088d508 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SelectCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/SelectCorrector.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.corrector; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -16,8 +16,8 @@ public class SelectCorrector extends BaseSemanticCorrector { @Override public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - List aggregateFields = SqlParserSelectHelper.getAggregateFields(correctS2SQL); - List selectFields = SqlParserSelectHelper.getSelectFields(correctS2SQL); + List aggregateFields = SqlSelectHelper.getAggregateFields(correctS2SQL); + List selectFields = SqlSelectHelper.getSelectFields(correctS2SQL); // If the number of aggregated fields is equal to the number of queried fields, do not add fields to select. if (!CollectionUtils.isEmpty(aggregateFields) && !CollectionUtils.isEmpty(selectFields) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/TimeCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/TimeCorrector.java new file mode 100644 index 000000000..f39860601 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/TimeCorrector.java @@ -0,0 +1,57 @@ +package com.tencent.supersonic.chat.core.corrector; + +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.common.util.jsqlparser.DateVisitor.DateBoundInfo; +import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlDateSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import org.apache.commons.lang3.StringUtils; + +/** + * Perform SQL corrections on the time in S2SQL. + */ +@Slf4j +public class TimeCorrector extends BaseSemanticCorrector { + + @Override + public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { + + parserDateDiffFunction(semanticParseInfo); + + addLowerBoundDate(semanticParseInfo); + + } + + private void addLowerBoundDate(SemanticParseInfo semanticParseInfo) { + String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); + DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(correctS2SQL); + if (Objects.isNull(dateBoundInfo)) { + return; + } + if (StringUtils.isBlank(dateBoundInfo.getLowerBound()) + && StringUtils.isNotBlank(dateBoundInfo.getUpperBound()) + && StringUtils.isNotBlank(dateBoundInfo.getUpperDate())) { + String upperDate = dateBoundInfo.getUpperDate(); + try { + correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL); + String condExpr = dateBoundInfo.getColumName() + " >= '" + upperDate + "'"; + correctS2SQL = SqlAddHelper.addWhere(correctS2SQL, CCJSqlParserUtil.parseCondExpression(condExpr)); + } catch (JSQLParserException e) { + log.error("parseCondExpression", e); + } + semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL); + } + } + + private void parserDateDiffFunction(SemanticParseInfo semanticParseInfo) { + String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); + correctS2SQL = SqlReplaceHelper.replaceFunction(correctS2SQL); + semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL); + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java index 41604054d..dc9fb9775 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java @@ -10,9 +10,9 @@ import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.StringUtil; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -38,8 +38,6 @@ public class WhereCorrector extends BaseSemanticCorrector { addDateIfNotExist(queryContext, semanticParseInfo); - parserDateDiffFunction(semanticParseInfo); - addQueryFilter(queryContext, semanticParseInfo); updateFieldValueByTechName(queryContext, semanticParseInfo); @@ -58,25 +56,19 @@ public class WhereCorrector extends BaseSemanticCorrector { } catch (JSQLParserException e) { log.error("parseCondExpression", e); } - correctS2SQL = SqlParserAddHelper.addWhere(correctS2SQL, expression); + correctS2SQL = SqlAddHelper.addWhere(correctS2SQL, expression); semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL); } } - private void parserDateDiffFunction(SemanticParseInfo semanticParseInfo) { - String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - correctS2SQL = SqlParserReplaceHelper.replaceFunction(correctS2SQL); - semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL); - } - private void addDateIfNotExist(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - List whereFields = SqlParserSelectHelper.getWhereFields(correctS2SQL); + List whereFields = SqlSelectHelper.getWhereFields(correctS2SQL); if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) { String currentDate = S2SqlDateHelper.getReferenceDate(queryContext, semanticParseInfo.getViewId()); if (StringUtils.isNotBlank(currentDate)) { - correctS2SQL = SqlParserAddHelper.addParenthesisToWhere(correctS2SQL); - correctS2SQL = SqlParserAddHelper.addWhere( + correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL); + correctS2SQL = SqlAddHelper.addWhere( correctS2SQL, TimeDimensionEnum.DAY.getChName(), currentDate); } } @@ -107,7 +99,7 @@ public class WhereCorrector extends BaseSemanticCorrector { } Map> aliasAndBizNameToTechName = getAliasAndBizNameToTechName(dimensions); - String correctS2SQL = SqlParserReplaceHelper.replaceValue(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), + String correctS2SQL = SqlReplaceHelper.replaceValue(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), aliasAndBizNameToTechName); semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java index 3c1893faf..b716a0e1a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java @@ -12,7 +12,7 @@ import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -54,7 +54,7 @@ public class QueryTypeParser implements SemanticParser { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) { //If all the fields in the SELECT statement are of tag type. - List whereFields = SqlParserSelectHelper.getWhereFields(sqlInfo.getS2SQL()) + List whereFields = SqlSelectHelper.getWhereFields(sqlInfo.getS2SQL()) .stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) .collect(Collectors.toList()); @@ -72,7 +72,7 @@ public class QueryTypeParser implements SemanticParser { } } //2. metric queryType - List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getS2SQL()); + List selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getS2SQL()); List metrics = semanticSchema.getMetrics(viewId); if (CollectionUtils.isNotEmpty(metrics)) { Set metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java index 15f4c5714..bb2dc2afe 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMResponseService.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery; import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlResp; import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserEqualHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlEqualHelper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; import org.springframework.stereotype.Service; @@ -52,7 +52,7 @@ public class LLMResponseService { Map result = new HashMap<>(); for (Map.Entry entry : llmResp.getSqlRespMap().entrySet()) { String key = entry.getKey(); - if (result.keySet().stream().anyMatch(existKey -> SqlParserEqualHelper.equals(existKey, key))) { + if (result.keySet().stream().anyMatch(existKey -> SqlEqualHelper.equals(existKey, key))) { continue; } result.put(key, entry.getValue()); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java new file mode 100644 index 000000000..11fcb5250 --- /dev/null +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/core/corrector/TimeCorrectorTest.java @@ -0,0 +1,100 @@ +package com.tencent.supersonic.chat.core.corrector; + +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +class TimeCorrectorTest { + + @Test + void testDoCorrect() { + TimeCorrector corrector = new TimeCorrector(); + QueryContext queryContext = new QueryContext(); + SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); + SqlInfo sqlInfo = new SqlInfo(); + //1.数据日期 <= + String sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; + sqlInfo.setCorrectS2SQL(sql); + semanticParseInfo.setSqlInfo(sqlInfo); + corrector.doCorrect(queryContext, semanticParseInfo); + + Assert.assertEquals( + "SELECT 维度1, SUM(播放量) FROM 数据库 WHERE ((歌手名 = '张三') AND 数据日期 <= '2023-11-17') " + + "AND 数据日期 >= '2023-11-17' GROUP BY 维度1", + sqlInfo.getCorrectS2SQL()); + + //2.数据日期 < + sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1"; + sqlInfo.setCorrectS2SQL(sql); + corrector.doCorrect(queryContext, semanticParseInfo); + + Assert.assertEquals( + "SELECT 维度1, SUM(播放量) FROM 数据库 WHERE ((歌手名 = '张三') AND 数据日期 < '2023-11-17') " + + "AND 数据日期 >= '2023-11-17' GROUP BY 维度1", + sqlInfo.getCorrectS2SQL()); + + //3.数据日期 >= + sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1"; + sqlInfo.setCorrectS2SQL(sql); + corrector.doCorrect(queryContext, semanticParseInfo); + + Assert.assertEquals( + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1", + sqlInfo.getCorrectS2SQL()); + + //4.数据日期 > + sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1"; + sqlInfo.setCorrectS2SQL(sql); + corrector.doCorrect(queryContext, semanticParseInfo); + + Assert.assertEquals( + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1", + sqlInfo.getCorrectS2SQL()); + + //5.no 数据日期 + sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE 歌手名 = '张三' GROUP BY 维度1"; + sqlInfo.setCorrectS2SQL(sql); + corrector.doCorrect(queryContext, semanticParseInfo); + + Assert.assertEquals( + "SELECT 维度1, SUM(播放量) FROM 数据库 WHERE 歌手名 = '张三' GROUP BY 维度1", + sqlInfo.getCorrectS2SQL()); + + //6. 数据日期-月 <= + sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE 歌手名 = '张三' AND 数据日期_月 <= '2024-01' GROUP BY 维度1"; + sqlInfo.setCorrectS2SQL(sql); + corrector.doCorrect(queryContext, semanticParseInfo); + + Assert.assertEquals( + "SELECT 维度1, SUM(播放量) FROM 数据库 WHERE (歌手名 = '张三' AND 数据日期_月 <= '2024-01') " + + "AND 数据日期_月 >= '2024-01' GROUP BY 维度1", + sqlInfo.getCorrectS2SQL()); + + //7. 数据日期-月 > + sql = "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE 歌手名 = '张三' AND 数据日期_月 > '2024-01' GROUP BY 维度1"; + sqlInfo.setCorrectS2SQL(sql); + corrector.doCorrect(queryContext, semanticParseInfo); + + Assert.assertEquals( + "SELECT 维度1, SUM(播放量) FROM 数据库 " + + "WHERE 歌手名 = '张三' AND 数据日期_月 > '2024-01' GROUP BY 维度1", + sqlInfo.getCorrectS2SQL()); + + //8. no where + sql = "SELECT COUNT(1) FROM 数据库"; + sqlInfo.setCorrectS2SQL(sql); + corrector.doCorrect(queryContext, semanticParseInfo); + Assert.assertEquals("SELECT COUNT(1) FROM 数据库", sqlInfo.getCorrectS2SQL()); + } +} \ No newline at end of file diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/MetricCheckProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/MetricCheckProcessor.java index 6e1003c04..ec957bcf5 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/MetricCheckProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/MetricCheckProcessor.java @@ -15,8 +15,8 @@ import com.tencent.supersonic.chat.server.service.SemanticService; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlRemoveHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -61,9 +61,9 @@ public class MetricCheckProcessor implements ParseResultProcessor { public String processCorrectSql(SemanticParseInfo parseInfo, SemanticSchema semanticSchema) { String correctSql = parseInfo.getSqlInfo().getCorrectS2SQL(); - List groupByFields = SqlParserSelectHelper.getGroupByFields(correctSql); - List metricFields = SqlParserSelectHelper.getAggregateFields(correctSql); - List whereFields = SqlParserSelectHelper.getWhereFields(correctSql); + List groupByFields = SqlSelectHelper.getGroupByFields(correctSql); + List metricFields = SqlSelectHelper.getAggregateFields(correctSql); + List whereFields = SqlSelectHelper.getWhereFields(correctSql); List dimensionFields = getDimensionFields(groupByFields, whereFields); if (CollectionUtils.isEmpty(metricFields) || StringUtils.isBlank(correctSql)) { return correctSql; @@ -195,8 +195,8 @@ public class MetricCheckProcessor implements ParseResultProcessor { } private boolean checkHasMetric(String correctSql, SemanticSchema semanticSchema) { - List selectFields = SqlParserSelectHelper.getSelectFields(correctSql); - List aggFields = SqlParserSelectHelper.getAggregateFields(correctSql); + List selectFields = SqlSelectHelper.getSelectFields(correctSql); + List aggFields = SqlSelectHelper.getAggregateFields(correctSql); List collect = semanticSchema.getMetrics().stream() .map(SchemaElement::getName).collect(Collectors.toList()); for (String field : selectFields) { @@ -209,11 +209,11 @@ public class MetricCheckProcessor implements ParseResultProcessor { private static String removeFieldInSql(String sql, Set metricToRemove, Set dimensionByToRemove, Set whereFieldsToRemove) { - sql = SqlParserRemoveHelper.removeWhereCondition(sql, whereFieldsToRemove); - sql = SqlParserRemoveHelper.removeSelect(sql, metricToRemove); - sql = SqlParserRemoveHelper.removeSelect(sql, dimensionByToRemove); - sql = SqlParserRemoveHelper.removeGroupBy(sql, dimensionByToRemove); - sql = SqlParserRemoveHelper.removeNumberFilter(sql); + sql = SqlRemoveHelper.removeWhereCondition(sql, whereFieldsToRemove); + sql = SqlRemoveHelper.removeSelect(sql, metricToRemove); + sql = SqlRemoveHelper.removeSelect(sql, dimensionByToRemove); + sql = SqlRemoveHelper.removeGroupBy(sql, dimensionByToRemove); + sql = SqlRemoveHelper.removeNumberFilter(sql); return sql; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java index 13b37cd23..db52b0d46 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java @@ -17,7 +17,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -60,7 +60,7 @@ public class ParseInfoProcessor implements ParseResultProcessor { if (correctS2SQL.equals(sqlInfo.getS2SQL())) { return; } - List expressions = SqlParserSelectHelper.getFilterExpression(correctS2SQL); + List expressions = SqlSelectHelper.getFilterExpression(correctS2SQL); //set dataInfo try { if (!org.apache.commons.collections.CollectionUtils.isEmpty(expressions)) { @@ -87,15 +87,15 @@ public class ParseInfoProcessor implements ParseResultProcessor { if (Objects.isNull(semanticSchema)) { return; } - List allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL())); + List allFields = getFieldsExceptDate(SqlSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL())); Set metrics = getElements(viewId, allFields, semanticSchema.getMetrics()); parseInfo.setMetrics(metrics); if (QueryType.METRIC.equals(parseInfo.getQueryType())) { - List groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL()); + List groupByFields = SqlSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL()); List groupByDimensions = getFieldsExceptDate(groupByFields); parseInfo.setDimensions(getElements(viewId, groupByDimensions, semanticSchema.getDimensions())); } else if (QueryType.TAG.equals(parseInfo.getQueryType())) { - List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL()); + List selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL()); List selectDimensions = getFieldsExceptDate(selectFields); parseInfo.setDimensions(getElements(viewId, selectDimensions, semanticSchema.getDimensions())); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java index 014638f89..7198e6d82 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java @@ -60,10 +60,10 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlRemoveHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import lombok.extern.slf4j.Slf4j; @@ -318,7 +318,7 @@ public class QueryServiceImpl implements QueryService { if (Objects.nonNull(parseInfo.getSqlInfo()) && StringUtils.isNotBlank(parseInfo.getSqlInfo().getCorrectS2SQL())) { String correctorSql = parseInfo.getSqlInfo().getCorrectS2SQL(); - fields = SqlParserSelectHelper.getAllFields(correctorSql); + fields = SqlSelectHelper.getAllFields(correctorSql); } if (LLMSqlQuery.QUERY_MODE.equalsIgnoreCase(parseInfo.getQueryMode()) && checkMetricReplace(fields, queryData.getMetrics())) { @@ -373,8 +373,8 @@ public class QueryServiceImpl implements QueryService { String correctorSql = parseInfo.getSqlInfo().getCorrectS2SQL(); log.info("correctorSql before replacing:{}", correctorSql); // get where filter and having filter - List whereExpressionList = SqlParserSelectHelper.getWhereExpressions(correctorSql); - List havingExpressionList = SqlParserSelectHelper.getHavingExpressions(correctorSql); + List whereExpressionList = SqlSelectHelper.getWhereExpressions(correctorSql); + List havingExpressionList = SqlSelectHelper.getHavingExpressions(correctorSql); List addWhereConditions = new ArrayList<>(); List addHavingConditions = new ArrayList<>(); Set removeWhereFieldNames = new HashSet<>(); @@ -384,16 +384,16 @@ public class QueryServiceImpl implements QueryService { parseInfo.getDimensionFilters(), addWhereConditions, removeWhereFieldNames); updateDateInfo(queryData, parseInfo, filedNameToValueMap, whereExpressionList, addWhereConditions, removeWhereFieldNames); - correctorSql = SqlParserReplaceHelper.replaceValue(correctorSql, filedNameToValueMap); - correctorSql = SqlParserRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames); + correctorSql = SqlReplaceHelper.replaceValue(correctorSql, filedNameToValueMap); + correctorSql = SqlRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames); // replace having filter updateFilters(havingExpressionList, queryData.getDimensionFilters(), parseInfo.getDimensionFilters(), addHavingConditions, removeHavingFieldNames); - correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap); - correctorSql = SqlParserRemoveHelper.removeHavingCondition(correctorSql, removeHavingFieldNames); + correctorSql = SqlReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap); + correctorSql = SqlRemoveHelper.removeHavingCondition(correctorSql, removeHavingFieldNames); - correctorSql = SqlParserAddHelper.addWhere(correctorSql, addWhereConditions); - correctorSql = SqlParserAddHelper.addHaving(correctorSql, addHavingConditions); + correctorSql = SqlAddHelper.addWhere(correctorSql, addWhereConditions); + correctorSql = SqlAddHelper.addHaving(correctorSql, addHavingConditions); log.info("correctorSql after replacing:{}", correctorSql); return correctorSql; } @@ -407,7 +407,7 @@ public class QueryServiceImpl implements QueryService { Map> fieldMap = new HashMap<>(); if (CollectionUtils.isNotEmpty(oriMetrics) && !oriMetrics.contains(metric.getName())) { fieldMap.put(oriMetrics.get(0), Pair.of(metric.getName(), metric.getDefaultAgg())); - correctorSql = SqlParserReplaceHelper.replaceAggFields(correctorSql, fieldMap); + correctorSql = SqlReplaceHelper.replaceAggFields(correctorSql, fieldMap); } log.info("after replaceMetrics:{}", correctorSql); parseInfo.getSqlInfo().setCorrectS2SQL(correctorSql); diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/DateVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/DateVisitor.java new file mode 100644 index 000000000..5eeed1b7b --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/DateVisitor.java @@ -0,0 +1,90 @@ +package com.tencent.supersonic.common.util.jsqlparser; + +import java.util.List; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.operators.relational.GreaterThan; +import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; +import net.sf.jsqlparser.expression.operators.relational.MinorThan; +import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; +import net.sf.jsqlparser.schema.Column; +import org.springframework.util.CollectionUtils; + +@Slf4j +public class DateVisitor extends ExpressionVisitorAdapter { + + private List filedNames; + + private DateBoundInfo dateBoundInfo = new DateBoundInfo(); + + public DateVisitor(List filedNames) { + this.filedNames = filedNames; + } + + @Override + public void visit(GreaterThan expr) { + if (containsField(expr.getLeftExpression())) { + dateBoundInfo.setColumName(SqlSelectHelper.getColumnName(expr.getLeftExpression())); + dateBoundInfo.setLowerBound(expr.getStringExpression()); + String columnValue = SqlSelectHelper.getColumValue(expr.getRightExpression()); + dateBoundInfo.setLowerDate(columnValue); + } + } + + @Override + public void visit(GreaterThanEquals expr) { + if (containsField(expr.getLeftExpression())) { + dateBoundInfo.setColumName(SqlSelectHelper.getColumnName(expr.getLeftExpression())); + dateBoundInfo.setLowerBound(expr.getStringExpression()); + String columnValue = SqlSelectHelper.getColumValue(expr.getRightExpression()); + dateBoundInfo.setLowerDate(columnValue); + } + } + + @Override + public void visit(MinorThanEquals expr) { + if (containsField(expr.getLeftExpression())) { + dateBoundInfo.setColumName(SqlSelectHelper.getColumnName(expr.getLeftExpression())); + dateBoundInfo.setUpperBound(expr.getStringExpression()); + String columnValue = SqlSelectHelper.getColumValue(expr.getRightExpression()); + dateBoundInfo.setUpperDate(columnValue); + } + } + + @Override + public void visit(MinorThan expr) { + if (containsField(expr.getLeftExpression())) { + dateBoundInfo.setColumName(SqlSelectHelper.getColumnName(expr.getLeftExpression())); + dateBoundInfo.setUpperBound(expr.getStringExpression()); + String columnValue = SqlSelectHelper.getColumValue(expr.getRightExpression()); + dateBoundInfo.setUpperDate(columnValue); + } + } + + private boolean containsField(Expression expr) { + if (expr instanceof Column) { + Column column = (Column) expr; + if (!CollectionUtils.isEmpty(filedNames) + && filedNames.contains(column.getColumnName())) { + return true; + } + } + return false; + } + + public DateBoundInfo getDateBoundInfo() { + return dateBoundInfo; + } + + @Data + public class DateBoundInfo { + + private String columName; + private String lowerBound; + private String lowerDate; + private String upperBound; + private String upperDate; + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java index c333bcdca..382ae621d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java @@ -102,7 +102,7 @@ public class FieldlValueReplaceVisitor extends ExpressionVisitorAdapter { if (Objects.isNull(rightExpression) || Objects.isNull(leftExpression)) { return; } - String columnName = SqlParserSelectHelper.getColumnName(leftExpression); + String columnName = SqlSelectHelper.getColumnName(leftExpression); if (StringUtils.isEmpty(columnName)) { return; } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionAliasReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionAliasReplaceVisitor.java index 3d719bbf1..45810d677 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionAliasReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionAliasReplaceVisitor.java @@ -15,7 +15,7 @@ public class FunctionAliasReplaceVisitor extends SelectItemVisitorAdapter { public void visit(SelectExpressionItem selectExpressionItem) { if (selectExpressionItem.getExpression() instanceof Function) { Function function = (Function) selectExpressionItem.getExpression(); - String columnName = SqlParserSelectHelper.getColumnName(function); + String columnName = SqlSelectHelper.getColumnName(function); //1.exist alias. as //2.alias's fieldName not equal. "sum(pv) as pv" cannot be replaced. if (Objects.nonNull(selectExpressionItem.getAlias()) && !selectExpressionItem.getAlias().getName() diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserAddHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelper.java similarity index 90% rename from common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserAddHelper.java rename to common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelper.java index c5723074f..ea2d26d55 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserAddHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelper.java @@ -31,10 +31,10 @@ import org.springframework.util.CollectionUtils; * Sql Parser add Helper */ @Slf4j -public class SqlParserAddHelper { +public class SqlAddHelper { public static String addFieldsToSelect(String sql, List fields) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); // add fields to select if (selectStatement == null) { return null; @@ -63,7 +63,7 @@ public class SqlParserAddHelper { } public static String addFunctionToSelect(String sql, List expressionList) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); if (selectStatement == null) { return null; } @@ -116,7 +116,7 @@ public class SqlParserAddHelper { if (StringUtils.isEmpty(column) || Objects.isNull(value)) { return sql; } - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -139,7 +139,7 @@ public class SqlParserAddHelper { } public static String addWhere(String sql, Expression expression) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -157,7 +157,7 @@ public class SqlParserAddHelper { } public static String addWhere(String sql, List expressionList) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -182,7 +182,7 @@ public class SqlParserAddHelper { } public static String addAggregateToField(String sql, Map fieldNameToAggregate) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -204,7 +204,7 @@ public class SqlParserAddHelper { if (CollectionUtils.isEmpty(groupByFields)) { return sql; } - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -213,7 +213,7 @@ public class SqlParserAddHelper { PlainSelect plainSelect = (PlainSelect) selectBody; GroupByElement groupByElement = new GroupByElement(); - List originalGroupByFields = SqlParserSelectHelper.getGroupByFields(sql); + List originalGroupByFields = SqlSelectHelper.getGroupByFields(sql); if (!CollectionUtils.isEmpty(originalGroupByFields)) { groupByFields.addAll(originalGroupByFields); } @@ -230,7 +230,7 @@ public class SqlParserAddHelper { if (selectItem instanceof SelectExpressionItem) { SelectExpressionItem selectExpressionItem = (SelectExpressionItem) selectItem; Expression expression = selectExpressionItem.getExpression(); - Function function = SqlParserSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); + Function function = SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); if (function == null) { continue; } @@ -246,7 +246,7 @@ public class SqlParserAddHelper { } for (OrderByElement orderByElement : orderByElements) { Expression expression = orderByElement.getExpression(); - Function function = SqlParserSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); + Function function = SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); if (function == null) { continue; } @@ -260,7 +260,7 @@ public class SqlParserAddHelper { return; } for (Expression expression : groupByElement.getGroupByExpressions()) { - Function function = SqlParserSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); + Function function = SqlSelectFunctionHelper.getFunction(expression, fieldNameToAggregate); if (function == null) { continue; } @@ -277,7 +277,7 @@ public class SqlParserAddHelper { private static void modifyWhereExpression(Expression whereExpression, Map fieldNameToAggregate) { - if (SqlParserSelectHelper.isLogicExpression(whereExpression)) { + if (SqlSelectHelper.isLogicExpression(whereExpression)) { AndExpression andExpression = (AndExpression) whereExpression; Expression leftExpression = andExpression.getLeftExpression(); Expression rightExpression = andExpression.getRightExpression(); @@ -297,7 +297,7 @@ public class SqlParserAddHelper { ComparisonOperator comparisonOperator = (ComparisonOperator) expression; if (comparisonOperator.getRightExpression() instanceof Column) { String columnName = ((Column) (comparisonOperator).getRightExpression()).getColumnName(); - Function function = SqlParserSelectFunctionHelper.getFunction(comparisonOperator.getRightExpression(), + Function function = SqlSelectFunctionHelper.getFunction(comparisonOperator.getRightExpression(), fieldNameToAggregate.get(columnName)); if (Objects.nonNull(function)) { comparisonOperator.setRightExpression(function); @@ -305,7 +305,7 @@ public class SqlParserAddHelper { } if (comparisonOperator.getLeftExpression() instanceof Column) { String columnName = ((Column) (comparisonOperator).getLeftExpression()).getColumnName(); - Function function = SqlParserSelectFunctionHelper.getFunction(comparisonOperator.getLeftExpression(), + Function function = SqlSelectFunctionHelper.getFunction(comparisonOperator.getLeftExpression(), fieldNameToAggregate.get(columnName)); if (Objects.nonNull(function)) { comparisonOperator.setLeftExpression(function); @@ -314,7 +314,7 @@ public class SqlParserAddHelper { } public static String addHaving(String sql, Set fieldNames) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -339,11 +339,11 @@ public class SqlParserAddHelper { } } } - return SqlParserRemoveHelper.removeNumberFilter(selectStatement.toString()); + return SqlRemoveHelper.removeNumberFilter(selectStatement.toString()); } public static String addHaving(String sql, List expressionList) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -368,7 +368,7 @@ public class SqlParserAddHelper { } public static String addParenthesisToWhere(String sql) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlDateSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlDateSelectHelper.java new file mode 100644 index 000000000..ee1819647 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlDateSelectHelper.java @@ -0,0 +1,35 @@ +package com.tencent.supersonic.common.util.jsqlparser; + +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.common.util.jsqlparser.DateVisitor.DateBoundInfo; +import java.util.List; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.statement.select.PlainSelect; + +/** + * Date field parsing helper class + */ +@Slf4j +public class SqlDateSelectHelper { + + public static DateBoundInfo getDateBoundInfo(String sql) { + List plainSelectList = SqlSelectHelper.getPlainSelect(sql); + if (plainSelectList.size() != 1) { + return null; + } + PlainSelect plainSelect = plainSelectList.get(0); + if (Objects.isNull(plainSelect)) { + return null; + } + Expression where = plainSelect.getWhere(); + if (Objects.isNull(where)) { + return null; + } + DateVisitor dateVisitor = new DateVisitor(TimeDimensionEnum.getChNameList()); + where.accept(dateVisitor); + return dateVisitor.getDateBoundInfo(); + } +} + diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserEqualHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlEqualHelper.java similarity index 51% rename from common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserEqualHelper.java rename to common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlEqualHelper.java index 928996512..c8f31242b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserEqualHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlEqualHelper.java @@ -8,7 +8,7 @@ import org.apache.commons.collections.CollectionUtils; * Sql Parser equal Helper */ @Slf4j -public class SqlParserEqualHelper { +public class SqlEqualHelper { /** * determine if two SQL statements are equal. @@ -19,43 +19,43 @@ public class SqlParserEqualHelper { */ public static boolean equals(String thisSql, String otherSql) { //1. select fields - List thisSelectFields = SqlParserSelectHelper.getSelectFields(thisSql); - List otherSelectFields = SqlParserSelectHelper.getSelectFields(otherSql); + List thisSelectFields = SqlSelectHelper.getSelectFields(thisSql); + List otherSelectFields = SqlSelectHelper.getSelectFields(otherSql); if (!CollectionUtils.isEqualCollection(thisSelectFields, otherSelectFields)) { return false; } //2. all fields - List thisAllFields = SqlParserSelectHelper.getAllFields(thisSql); - List otherAllFields = SqlParserSelectHelper.getAllFields(otherSql); + List thisAllFields = SqlSelectHelper.getAllFields(thisSql); + List otherAllFields = SqlSelectHelper.getAllFields(otherSql); if (!CollectionUtils.isEqualCollection(thisAllFields, otherAllFields)) { return false; } //3. where - List thisFieldExpressions = SqlParserSelectHelper.getFilterExpression(thisSql); - List otherFieldExpressions = SqlParserSelectHelper.getFilterExpression(otherSql); + List thisFieldExpressions = SqlSelectHelper.getFilterExpression(thisSql); + List otherFieldExpressions = SqlSelectHelper.getFilterExpression(otherSql); if (!CollectionUtils.isEqualCollection(thisFieldExpressions, otherFieldExpressions)) { return false; } //4. tableName - if (!SqlParserSelectHelper.getDbTableName(thisSql) - .equalsIgnoreCase(SqlParserSelectHelper.getDbTableName(otherSql))) { + if (!SqlSelectHelper.getDbTableName(thisSql) + .equalsIgnoreCase(SqlSelectHelper.getDbTableName(otherSql))) { return false; } //5. having - List thisHavingExpressions = SqlParserSelectHelper.getHavingExpressions(thisSql); - List otherHavingExpressions = SqlParserSelectHelper.getHavingExpressions(otherSql); + List thisHavingExpressions = SqlSelectHelper.getHavingExpressions(thisSql); + List otherHavingExpressions = SqlSelectHelper.getHavingExpressions(otherSql); if (!CollectionUtils.isEqualCollection(thisHavingExpressions, otherHavingExpressions)) { return false; } //6. orderBy - List thisOrderByExpressions = SqlParserSelectHelper.getOrderByExpressions(thisSql); - List otherOrderByExpressions = SqlParserSelectHelper.getOrderByExpressions(otherSql); + List thisOrderByExpressions = SqlSelectHelper.getOrderByExpressions(thisSql); + List otherOrderByExpressions = SqlSelectHelper.getOrderByExpressions(otherSql); if (!CollectionUtils.isEqualCollection(thisOrderByExpressions, otherOrderByExpressions)) { return false; diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlRemoveHelper.java similarity index 89% rename from common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java rename to common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlRemoveHelper.java index 413b45cd5..12c736263 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlRemoveHelper.java @@ -37,10 +37,10 @@ import java.util.Objects; * Sql Parser remove Helper */ @Slf4j -public class SqlParserRemoveHelper { +public class SqlRemoveHelper { public static String removeSelect(String sql, Set fields) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); if (selectStatement == null) { return sql; } @@ -52,7 +52,7 @@ public class SqlParserRemoveHelper { selectItems.removeIf(selectItem -> { if (selectItem instanceof SelectExpressionItem) { SelectExpressionItem selectExpressionItem = (SelectExpressionItem) selectItem; - String columnName = SqlParserSelectHelper.getColumnName(selectExpressionItem.getExpression()); + String columnName = SqlSelectHelper.getColumnName(selectExpressionItem.getExpression()); return fields.contains(columnName); } return false; @@ -61,7 +61,7 @@ public class SqlParserRemoveHelper { } public static String removeWhereCondition(String sql, Set removeFieldNames) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -84,7 +84,7 @@ public class SqlParserRemoveHelper { } public static String removeNumberFilter(String sql) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); if (selectStatement == null) { return sql; } @@ -105,7 +105,7 @@ public class SqlParserRemoveHelper { } private static void removeWhereExpression(Expression whereExpression, Set removeFieldNames) { - if (SqlParserSelectHelper.isLogicExpression(whereExpression)) { + if (SqlSelectHelper.isLogicExpression(whereExpression)) { BinaryExpression binaryExpression = (BinaryExpression) whereExpression; Expression leftExpression = binaryExpression.getLeftExpression(); Expression rightExpression = binaryExpression.getRightExpression(); @@ -140,7 +140,7 @@ public class SqlParserRemoveHelper { || expression instanceof MinorThanEquals || expression instanceof MinorThan) { ComparisonOperator comparisonOperator = (ComparisonOperator) expression; - String columnName = SqlParserSelectHelper.getColumnName(comparisonOperator.getLeftExpression(), + String columnName = SqlSelectHelper.getColumnName(comparisonOperator.getLeftExpression(), comparisonOperator.getRightExpression()); if (!removeFieldNames.contains(columnName)) { return; @@ -158,7 +158,7 @@ public class SqlParserRemoveHelper { } if (expression instanceof InExpression) { InExpression inExpression = (InExpression) expression; - String columnName = SqlParserSelectHelper.getColumnName(inExpression.getLeftExpression(), + String columnName = SqlSelectHelper.getColumnName(inExpression.getLeftExpression(), inExpression.getRightExpression()); if (!removeFieldNames.contains(columnName)) { return; @@ -175,7 +175,7 @@ public class SqlParserRemoveHelper { } if (expression instanceof LikeExpression) { LikeExpression likeExpression = (LikeExpression) expression; - String columnName = SqlParserSelectHelper.getColumnName(likeExpression.getLeftExpression(), + String columnName = SqlSelectHelper.getColumnName(likeExpression.getLeftExpression(), likeExpression.getRightExpression()); if (!removeFieldNames.contains(columnName)) { return; @@ -192,7 +192,7 @@ public class SqlParserRemoveHelper { } public static String removeHavingCondition(String sql, Set removeFieldNames) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -207,26 +207,8 @@ public class SqlParserRemoveHelper { return removeNumberFilter(selectStatement.toString()); } - public static String removeWhere(String sql, List fields) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); - SelectBody selectBody = selectStatement.getSelectBody(); - if (!(selectBody instanceof PlainSelect)) { - return sql; - } - PlainSelect plainSelect = (PlainSelect) selectBody; - Expression where = plainSelect.getWhere(); - - if (where == null) { - return sql; - } else { - where.accept(new FilterRemoveVisitor(fields)); - plainSelect.setWhere(where); - } - return selectStatement.toString(); - } - public static String removeGroupBy(String sql, Set fields) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); if (selectStatement == null) { return sql; } @@ -328,7 +310,7 @@ public class SqlParserRemoveHelper { return distinguishNumberFilter(leftExpression, expression); } if (sqlEditEnum.equals(SqlEditEnum.DATEDIFF)) { - return SqlParserReplaceHelper.distinguishDateDiffFilter(leftExpression, expression); + return SqlReplaceHelper.distinguishDateDiffFilter(leftExpression, expression); } return expression; } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelper.java similarity index 93% rename from common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelper.java rename to common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelper.java index 7f665c5a0..f7a221048 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelper.java @@ -43,10 +43,10 @@ import org.springframework.util.CollectionUtils; * Sql Parser replace Helper */ @Slf4j -public class SqlParserReplaceHelper { +public class SqlReplaceHelper { public static String replaceSelectFields(String sql, Map fieldNameMap) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { return sql; @@ -84,7 +84,7 @@ public class SqlParserReplaceHelper { } public static String replaceAggFields(String sql, Map> fieldNameToAggMap) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { return sql; @@ -122,14 +122,14 @@ public class SqlParserReplaceHelper { public static String replaceValue(String sql, Map> filedNameToValueMap, boolean exactReplace) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { return sql; } List plainSelectList = new ArrayList<>(); plainSelectList.add((PlainSelect) selectBody); - List plainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList); + List plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect plainSelect : plainSelects) { Expression where = plainSelect.getWhere(); FieldlValueReplaceVisitor visitor = new FieldlValueReplaceVisitor(exactReplace, filedNameToValueMap); @@ -141,14 +141,14 @@ public class SqlParserReplaceHelper { } public static String replaceFieldNameByValue(String sql, Map> fieldValueToFieldNames) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { return sql; } List plainSelectList = new ArrayList<>(); plainSelectList.add((PlainSelect) selectBody); - List plainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList); + List plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect plainSelect : plainSelects) { Expression where = plainSelect.getWhere(); FiledNameReplaceVisitor visitor = new FiledNameReplaceVisitor(fieldValueToFieldNames); @@ -164,7 +164,7 @@ public class SqlParserReplaceHelper { } public static String replaceFields(String sql, Map fieldNameMap, boolean exactReplace) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); System.out.println(selectStatement.getSelectBody()); SelectBody selectBody = selectStatement.getSelectBody(); List plainSelectList = new ArrayList<>(); @@ -189,7 +189,7 @@ public class SqlParserReplaceHelper { } else { return sql; } - List plainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList); + List plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect plainSelect : plainSelects) { replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, plainSelect); } @@ -238,7 +238,7 @@ public class SqlParserReplaceHelper { SelectBody subSelectBody = ((SubSelect) join.getRightItem()).getSelectBody(); List plainSelectList = new ArrayList<>(); plainSelectList.add((PlainSelect) subSelectBody); - List subPlainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList); + List subPlainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect subPlainSelect : subPlainSelects) { replaceFieldsInPlainOneSelect(fieldNameMap, exactReplace, subPlainSelect); } @@ -262,14 +262,14 @@ public class SqlParserReplaceHelper { } public static String replaceFunction(String sql, Map functionMap) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { return sql; } List plainSelectList = new ArrayList<>(); plainSelectList.add((PlainSelect) selectBody); - List plainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList); + List plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect plainSelect : plainSelects) { replaceFunction(functionMap, plainSelect); } @@ -303,14 +303,14 @@ public class SqlParserReplaceHelper { } public static String replaceFunction(String sql) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { return sql; } Expression where = ((PlainSelect) selectBody).getWhere(); try { - Expression expression = SqlParserRemoveHelper.filteredExpression(where, SqlEditEnum.DATEDIFF); + Expression expression = SqlRemoveHelper.filteredExpression(where, SqlEditEnum.DATEDIFF); ((PlainSelect) selectBody).setWhere(expression); } catch (Exception e) { log.info("replaceFunction has an exception:{}", e.toString()); @@ -384,7 +384,7 @@ public class SqlParserReplaceHelper { if (StringUtils.isEmpty(tableName)) { return sql; } - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (selectBody instanceof PlainSelect) { PlainSelect plainSelect = (PlainSelect) selectBody; @@ -406,7 +406,7 @@ public class SqlParserReplaceHelper { // replace table name List plainSelects = new ArrayList<>(); plainSelects.add(plainSelect); - List painSelects = SqlParserSelectHelper.getPlainSelects(plainSelects); + List painSelects = SqlSelectHelper.getPlainSelects(plainSelects); for (PlainSelect painSelect : painSelects) { painSelect.accept( new SelectVisitorAdapter() { @@ -422,7 +422,7 @@ public class SqlParserReplaceHelper { SelectBody subSelectBody = ((SubSelect) join.getRightItem()).getSelectBody(); List plainSelectList = new ArrayList<>(); plainSelectList.add((PlainSelect) subSelectBody); - List subPlainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList); + List subPlainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect subPlainSelect : subPlainSelects) { subPlainSelect.getFromItem().accept(new TableNameReplaceVisitor(tableName)); } @@ -436,7 +436,7 @@ public class SqlParserReplaceHelper { } public static String replaceAlias(String sql) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { return sql; @@ -454,7 +454,7 @@ public class SqlParserReplaceHelper { } public static String replaceHavingValue(String sql, Map> filedNameToValueMap) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { return sql; @@ -523,7 +523,7 @@ public class SqlParserReplaceHelper { } public static String replaceSqlByExpression(String sql, Map replace) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); List plainSelectList = new ArrayList<>(); if (selectBody instanceof PlainSelect) { @@ -539,7 +539,7 @@ public class SqlParserReplaceHelper { } else { return sql; } - List plainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList); + List plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect plainSelect : plainSelects) { replacePlainSelectByExpr(plainSelect, replace); } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectFunctionHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectFunctionHelper.java similarity index 94% rename from common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectFunctionHelper.java rename to common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectFunctionHelper.java index 14504259f..db11e094e 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectFunctionHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectFunctionHelper.java @@ -23,13 +23,13 @@ import java.util.Set; * Sql Parser Select function Helper */ @Slf4j -public class SqlParserSelectFunctionHelper { +public class SqlSelectFunctionHelper { public static boolean hasAggregateFunction(String sql) { if (!CollectionUtils.isEmpty(getFunctions(sql))) { return true; } - return SqlParserSelectHelper.hasGroupBy(sql); + return SqlSelectHelper.hasGroupBy(sql); } public static void main(String[] args) { @@ -46,7 +46,7 @@ public class SqlParserSelectFunctionHelper { } public static Set getFunctions(String sql) { - Select selectStatement = SqlParserSelectHelper.getSelect(sql); + Select selectStatement = SqlSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java similarity index 91% rename from common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java rename to common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java index e5af76ebc..d798e7e10 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java @@ -13,15 +13,13 @@ import net.sf.jsqlparser.expression.CaseExpression; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.LongValue; import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.WhenClause; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; import net.sf.jsqlparser.expression.operators.conditional.OrExpression; import net.sf.jsqlparser.expression.operators.conditional.XorExpression; import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; -import net.sf.jsqlparser.expression.operators.relational.GreaterThan; -import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; -import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; @@ -38,14 +36,13 @@ import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; import net.sf.jsqlparser.statement.select.SetOperationList; import net.sf.jsqlparser.statement.select.SubSelect; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.springframework.util.CollectionUtils; /** * Sql Parser Select Helper */ @Slf4j -public class SqlParserSelectHelper { +public class SqlSelectHelper { public static List getFilterExpression(String sql) { List plainSelectList = getPlainSelect(sql); @@ -435,34 +432,16 @@ public class SqlParserSelectHelper { return ""; } - public static Expression getTimeFilter(List> times, String columnBegin, - String columnEnd) { - Expression expression = null; - for (ImmutablePair t : times) { - Expression expr = null; - ComparisonOperator left = new MinorThanEquals(); - if (t.left.equals(t.right)) { - left.setLeftExpression(new Column(columnBegin)); - left.setRightExpression(new StringValue(t.left)); - ComparisonOperator right = new GreaterThan(); - right.setLeftExpression(new Column(columnEnd)); - right.setRightExpression(new StringValue(t.right)); - expr = new AndExpression(left, right); - } else { - left.setLeftExpression(new StringValue(t.left)); - left.setRightExpression(new Column(columnEnd)); - ComparisonOperator right = new GreaterThanEquals(); - right.setLeftExpression(new StringValue(t.right)); - right.setRightExpression(new Column(columnBegin)); - expr = new AndExpression(left, right); - } - if (expression == null) { - expression = expr; - continue; - } - expression = new OrExpression(expression, expr); + public static String getColumValue(Expression expression) { + if (expression instanceof StringValue) { + StringValue value = (StringValue) expression; + return value.getValue(); } - return expression; + if (expression instanceof LongValue) { + LongValue value = (LongValue) expression; + return String.valueOf(value.getValue()); + } + return ""; } public static Table getTable(String sql) { @@ -488,14 +467,6 @@ public class SqlParserSelectHelper { return table.getFullyQualifiedName(); } - public static String getNormalizedSql(String sql) { - Select selectStatement = getSelect(sql); - if (selectStatement == null) { - return null; - } - return selectStatement.toString(); - } - public static Set getColumnFromExpr(String expr) { Expression expression = QueryExpressionReplaceVisitor.getExpression(expr); Set columns = new HashSet<>(); diff --git a/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java b/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java index fd9703533..bbef9221b 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java @@ -10,7 +10,7 @@ import java.util.List; class DateUtilsTest { @Test - void getBeforeDate() { + void testGetBeforeDate() { String dateStr = DateUtils.getBeforeDate("2023-08-10", 1, DatePeriodEnum.DAY); Assert.assertEquals(dateStr, "2023-08-09"); @@ -38,18 +38,6 @@ class DateUtilsTest { dateStr = DateUtils.getBeforeDate("2023-08-10", 0, DatePeriodEnum.YEAR); Assert.assertEquals(dateStr, "2023-01-01"); - - dateStr = DateUtils.getBeforeDate(0, DatePeriodEnum.DAY); - //Assert.assertEquals(dateStr, "2023-09-08"); - - dateStr = DateUtils.getBeforeDate(1, DatePeriodEnum.DAY); - //Assert.assertEquals(dateStr, "2023-09-07"); - - dateStr = DateUtils.getBeforeDate(1, DatePeriodEnum.WEEK); - //Assert.assertEquals(dateStr, "2023-09-01"); - - dateStr = DateUtils.getBeforeDate(1, DatePeriodEnum.MONTH); - //Assert.assertEquals(dateStr, "2023-08-08"); } @Test diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserAddHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelperTest.java similarity index 73% rename from common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserAddHelperTest.java rename to common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelperTest.java index 0648de68f..842bbbeb2 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserAddHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlAddHelperTest.java @@ -15,26 +15,26 @@ import org.junit.jupiter.api.Test; /** * SqlParserAddHelperTest Test */ -class SqlParserAddHelperTest { +class SqlAddHelperTest { @Test - void addWhere() throws JSQLParserException { + void testAddWhere() throws JSQLParserException { String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - sql = SqlParserAddHelper.addWhere(sql, "column_a", 123444555); - List selectFields = SqlParserSelectHelper.getAllFields(sql); + sql = SqlAddHelper.addWhere(sql, "column_a", 123444555); + List selectFields = SqlSelectHelper.getAllFields(sql); Assert.assertEquals(selectFields.contains("column_a"), true); - sql = SqlParserAddHelper.addWhere(sql, "column_b", "123456666"); - selectFields = SqlParserSelectHelper.getAllFields(sql); + sql = SqlAddHelper.addWhere(sql, "column_b", "123456666"); + selectFields = SqlSelectHelper.getAllFields(sql); Assert.assertEquals(selectFields.contains("column_b"), true); Expression expression = CCJSqlParserUtil.parseCondExpression(" ( column_c = 111 or column_d = 1111)"); - sql = SqlParserAddHelper.addWhere( + sql = SqlAddHelper.addWhere( "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1", expression); @@ -42,20 +42,20 @@ class SqlParserAddHelperTest { Assert.assertEquals(sql.contains("column_c = 111"), true); sql = "select 部门,sum (访问次数) from 超音数 where 用户 = alice or 发布日期 ='2023-07-03' group by 部门 limit 1"; - sql = SqlParserAddHelper.addParenthesisToWhere(sql); - sql = SqlParserAddHelper.addWhere(sql, "数据日期", "2023-08-08"); + sql = SqlAddHelper.addParenthesisToWhere(sql); + sql = SqlAddHelper.addWhere(sql, "数据日期", "2023-08-08"); Assert.assertEquals(sql, "SELECT 部门, sum(访问次数) FROM 超音数 WHERE " + "(用户 = alice OR 发布日期 = '2023-07-03') AND 数据日期 = '2023-08-08' GROUP BY 部门 LIMIT 1"); } @Test - void addFunctionToSelect() { + void testAddFunctionToSelect() { String sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; - List havingExpressionList = SqlParserSelectHelper.getHavingExpression(sql); + List havingExpressionList = SqlSelectHelper.getHavingExpression(sql); - String replaceSql = SqlParserAddHelper.addFunctionToSelect(sql, havingExpressionList); + String replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", @@ -63,9 +63,9 @@ class SqlParserAddHelperTest { sql = "SELECT user_name,sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; - havingExpressionList = SqlParserSelectHelper.getHavingExpression(sql); + havingExpressionList = SqlSelectHelper.getHavingExpression(sql); - replaceSql = SqlParserAddHelper.addFunctionToSelect(sql, havingExpressionList); + replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", @@ -73,9 +73,9 @@ class SqlParserAddHelperTest { sql = "SELECT user_name,sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') AND " + "sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; - havingExpressionList = SqlParserSelectHelper.getHavingExpression(sql); + havingExpressionList = SqlSelectHelper.getHavingExpression(sql); - replaceSql = SqlParserAddHelper.addFunctionToSelect(sql, havingExpressionList); + replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') " + "AND sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", @@ -84,12 +84,12 @@ class SqlParserAddHelperTest { } @Test - void addAggregateToField() { + void testAddAggregateToField() { String sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; - List havingExpressionList = SqlParserSelectHelper.getHavingExpression(sql); + List havingExpressionList = SqlSelectHelper.getHavingExpression(sql); - String replaceSql = SqlParserAddHelper.addFunctionToSelect(sql, havingExpressionList); + String replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", @@ -97,9 +97,9 @@ class SqlParserAddHelperTest { sql = "SELECT user_name,sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; - havingExpressionList = SqlParserSelectHelper.getHavingExpression(sql); + havingExpressionList = SqlSelectHelper.getHavingExpression(sql); - replaceSql = SqlParserAddHelper.addFunctionToSelect(sql, havingExpressionList); + replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", @@ -107,9 +107,9 @@ class SqlParserAddHelperTest { sql = "SELECT user_name,sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') AND " + "sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; - havingExpressionList = SqlParserSelectHelper.getHavingExpression(sql); + havingExpressionList = SqlSelectHelper.getHavingExpression(sql); - replaceSql = SqlParserAddHelper.addFunctionToSelect(sql, havingExpressionList); + replaceSql = SqlAddHelper.addFunctionToSelect(sql, havingExpressionList); System.out.println(replaceSql); Assert.assertEquals("SELECT user_name, sum(pv) FROM 超音数 WHERE (sys_imp_date <= '2023-09-03') " + "AND sys_imp_date = '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000", @@ -117,7 +117,7 @@ class SqlParserAddHelperTest { } @Test - void addAggregateToMetricField() { + void testAddAggregateToMetricField() { String sql = "select department, pv from t_1 where sys_imp_date = '2023-09-11' order by pv desc limit 10"; Map filedNameToAggregate = new HashMap<>(); @@ -126,8 +126,8 @@ class SqlParserAddHelperTest { Set groupByFields = new HashSet<>(); groupByFields.add("department"); - String replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + String replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -136,8 +136,8 @@ class SqlParserAddHelperTest { sql = "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 " + "order by pv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -145,8 +145,8 @@ class SqlParserAddHelperTest { replaceSql); sql = "select department, pv from t_1 where pv >1 order by pv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sum(pv) > 1 " @@ -154,8 +154,8 @@ class SqlParserAddHelperTest { replaceSql); sql = "select department, pv from t_1 where sum(pv) >1 order by pv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sum(pv) > 1 " @@ -164,8 +164,8 @@ class SqlParserAddHelperTest { sql = "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' and sum(pv) >1 " + "GROUP BY department order by pv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -174,8 +174,8 @@ class SqlParserAddHelperTest { sql = "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 " + "GROUP BY department order by pv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -184,8 +184,8 @@ class SqlParserAddHelperTest { sql = "select department, pv from t_1 where sys_imp_date = '2023-09-11' and pv >1 and department = 'HR' " + "GROUP BY department order by pv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sys_imp_date = '2023-09-11' AND sum(pv) > 1 " @@ -194,8 +194,8 @@ class SqlParserAddHelperTest { sql = "select department, pv from t_1 where (pv >1 and department = 'HR') " + " and sys_imp_date = '2023-09-11' GROUP BY department order by pv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE (sum(pv) > 1 AND department = 'HR') AND " @@ -204,9 +204,9 @@ class SqlParserAddHelperTest { sql = "select department, sum(pv) as pv from t_1 where sys_imp_date = '2023-09-11' GROUP BY " + "department order by pv desc limit 10"; - replaceSql = SqlParserReplaceHelper.replaceAlias(sql); - replaceSql = SqlParserAddHelper.addAggregateToField(replaceSql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlReplaceHelper.replaceAlias(sql); + replaceSql = SqlAddHelper.addAggregateToField(replaceSql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) AS pv " @@ -216,7 +216,7 @@ class SqlParserAddHelperTest { } @Test - void addAggregateToCountDiscountMetricField() { + void testAddAggregateToCountDiscountMetricField() { String sql = "select department, uv from t_1 where sys_imp_date = '2023-09-11' order by uv desc limit 10"; Map filedNameToAggregate = new HashMap<>(); @@ -225,8 +225,8 @@ class SqlParserAddHelperTest { Set groupByFields = new HashSet<>(); groupByFields.add("department"); - String replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + String replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -235,8 +235,8 @@ class SqlParserAddHelperTest { sql = "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 " + "order by uv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -244,8 +244,8 @@ class SqlParserAddHelperTest { replaceSql); sql = "select department, uv from t_1 where uv >1 order by uv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) FROM t_1 WHERE count(DISTINCT uv) > 1 " @@ -253,8 +253,8 @@ class SqlParserAddHelperTest { replaceSql); sql = "select department, uv from t_1 where count(DISTINCT uv) >1 order by uv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) FROM t_1 WHERE count(DISTINCT uv) > 1 " @@ -264,8 +264,8 @@ class SqlParserAddHelperTest { sql = "select department, count(DISTINCT uv) from t_1 where sys_imp_date = '2023-09-11'" + " and count(DISTINCT uv) >1 " + "GROUP BY department order by count(DISTINCT uv) desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -274,8 +274,8 @@ class SqlParserAddHelperTest { sql = "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 " + "GROUP BY department order by count(DISTINCT uv) desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -284,8 +284,8 @@ class SqlParserAddHelperTest { sql = "select department, uv from t_1 where sys_imp_date = '2023-09-11' and uv >1 and department = 'HR' " + "GROUP BY department order by uv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) FROM t_1 WHERE sys_imp_date = " @@ -295,8 +295,8 @@ class SqlParserAddHelperTest { sql = "select department, uv from t_1 where (uv >1 and department = 'HR') " + " and sys_imp_date = '2023-09-11' GROUP BY department order by uv desc limit 10"; - replaceSql = SqlParserAddHelper.addAggregateToField(sql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlAddHelper.addAggregateToField(sql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) FROM t_1 WHERE (count(DISTINCT uv) > " @@ -307,9 +307,9 @@ class SqlParserAddHelperTest { sql = "select department, count(DISTINCT uv) as uv from t_1 where sys_imp_date = '2023-09-11' GROUP BY " + "department order by uv desc limit 10"; - replaceSql = SqlParserReplaceHelper.replaceAlias(sql); - replaceSql = SqlParserAddHelper.addAggregateToField(replaceSql, filedNameToAggregate); - replaceSql = SqlParserAddHelper.addGroupBy(replaceSql, groupByFields); + replaceSql = SqlReplaceHelper.replaceAlias(sql); + replaceSql = SqlAddHelper.addAggregateToField(replaceSql, filedNameToAggregate); + replaceSql = SqlAddHelper.addGroupBy(replaceSql, groupByFields); Assert.assertEquals( "SELECT department, count(DISTINCT uv) AS uv " @@ -319,14 +319,14 @@ class SqlParserAddHelperTest { } @Test - void addGroupBy() { + void testAddGroupBy() { String sql = "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' " + "order by sum(pv) desc limit 10"; Set groupByFields = new HashSet<>(); groupByFields.add("department"); - String replaceSql = SqlParserAddHelper.addGroupBy(sql, groupByFields); + String replaceSql = SqlAddHelper.addGroupBy(sql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -336,7 +336,7 @@ class SqlParserAddHelperTest { sql = "select department, sum(pv) from t_1 where (department = 'HR') and sys_imp_date = '2023-09-11' " + "order by sum(pv) desc limit 10"; - replaceSql = SqlParserAddHelper.addGroupBy(sql, groupByFields); + replaceSql = SqlAddHelper.addGroupBy(sql, groupByFields); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE (department = 'HR') AND sys_imp_date " @@ -345,7 +345,7 @@ class SqlParserAddHelperTest { } @Test - void addHaving() { + void testAddHaving() { String sql = "select department, sum(pv) from t_1 where sys_imp_date = '2023-09-11' and " + "sum(pv) > 2000 group by department order by sum(pv) desc limit 10"; List groupByFields = new ArrayList<>(); @@ -354,7 +354,7 @@ class SqlParserAddHelperTest { Set fieldNames = new HashSet<>(); fieldNames.add("pv"); - String replaceSql = SqlParserAddHelper.addHaving(sql, fieldNames); + String replaceSql = SqlAddHelper.addHaving(sql, fieldNames); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -364,7 +364,7 @@ class SqlParserAddHelperTest { sql = "select department, sum(pv) from t_1 where (sum(pv) > 2000) and sys_imp_date = '2023-09-11' " + "group by department order by sum(pv) desc limit 10"; - replaceSql = SqlParserAddHelper.addHaving(sql, fieldNames); + replaceSql = SqlAddHelper.addHaving(sql, fieldNames); Assert.assertEquals( "SELECT department, sum(pv) FROM t_1 WHERE sys_imp_date = '2023-09-11' " @@ -373,12 +373,12 @@ class SqlParserAddHelperTest { } @Test - void addParenthesisToWhere() { + void testAddParenthesisToWhere() { String sql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + " order by 播放量 desc limit 11"; - String replaceSql = SqlParserAddHelper.addParenthesisToWhere(sql); + String replaceSql = SqlAddHelper.addParenthesisToWhere(sql); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1 " @@ -388,10 +388,10 @@ class SqlParserAddHelperTest { } @Test - void addFieldsToSelect() { + void testAddFieldsToSelect() { String correctS2SQL = "SELECT 用户, 页面 FROM 超音数用户部门 GROUP BY 用户, 页面 ORDER BY count(*) DESC"; - String replaceFields = SqlParserAddHelper.addFieldsToSelect(correctS2SQL, - SqlParserSelectHelper.getOrderByFields(correctS2SQL)); + String replaceFields = SqlAddHelper.addFieldsToSelect(correctS2SQL, + SqlSelectHelper.getOrderByFields(correctS2SQL)); Assert.assertEquals( "SELECT 用户, 页面 FROM 超音数用户部门 GROUP BY 用户, 页面 ORDER BY count(*) DESC", replaceFields); diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlDateSelectHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlDateSelectHelperTest.java new file mode 100644 index 000000000..bb4fbab5a --- /dev/null +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlDateSelectHelperTest.java @@ -0,0 +1,46 @@ +package com.tencent.supersonic.common.util.jsqlparser; + + +import com.tencent.supersonic.common.util.jsqlparser.DateVisitor.DateBoundInfo; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +class SqlDateSelectHelperTest { + + @Test + void testGetDateBoundInfo() { + + String sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1"; + DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + Assert.assertEquals(dateBoundInfo.getLowerBound(), ">="); + Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17"); + + sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1"; + dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + Assert.assertEquals(dateBoundInfo.getLowerBound(), ">"); + Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17"); + + sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; + dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + Assert.assertEquals(dateBoundInfo.getUpperBound(), "<="); + Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); + + sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1"; + dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + Assert.assertEquals(dateBoundInfo.getUpperBound(), "<"); + Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); + + sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-10-17' " + + "AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; + dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + Assert.assertEquals(dateBoundInfo.getUpperBound(), "<="); + Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); + Assert.assertEquals(dateBoundInfo.getLowerBound(), ">="); + Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-10-17"); + } +} \ No newline at end of file diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserEqualHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlEqualHelperTest.java similarity index 82% rename from common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserEqualHelperTest.java rename to common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlEqualHelperTest.java index 104c87e29..295bb3fb7 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserEqualHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlEqualHelperTest.java @@ -8,33 +8,33 @@ import org.junit.jupiter.api.Test; * @author lex luo * @date 2023/11/15 15:04 */ -class SqlParserEqualHelperTest { +class SqlEqualHelperTest { @Test void testEquals() { String sql1 = "SELECT * FROM table1 WHERE column1 = 1 AND column2 = 2"; String sql2 = "SELECT * FROM table1 WHERE column2 = 2 AND column1 = 1"; - Assert.equals(SqlParserEqualHelper.equals(sql1, sql2), true); + Assert.equals(SqlEqualHelper.equals(sql1, sql2), true); sql1 = "SELECT a,b,c,d FROM table1 WHERE column1 = 1 AND column2 = 2 order by a"; sql2 = "SELECT d,c,b,a FROM table1 WHERE column2 = 2 AND column1 = 1 order by a"; - Assert.equals(SqlParserEqualHelper.equals(sql1, sql2), true); + Assert.equals(SqlEqualHelper.equals(sql1, sql2), true); sql1 = "SELECT a,sum(b),sum(c),sum(d) FROM table1 WHERE column1 = 1 AND column2 = 2 group by a order by a"; sql2 = "SELECT sum(d),sum(c),sum(b),a FROM table1 WHERE column2 = 2 AND column1 = 1 group by a order by a"; - Assert.equals(SqlParserEqualHelper.equals(sql1, sql2), true); + Assert.equals(SqlEqualHelper.equals(sql1, sql2), true); sql1 = "SELECT a,sum(b),sum(c),sum(d) FROM table1 WHERE column1 = 1 AND column2 = 2 group by a order by a"; sql2 = "SELECT sum(d),sum(c),sum(b),a FROM table1 WHERE column2 = 2 AND column1 = 1 group by a order by a"; - Assert.equals(SqlParserEqualHelper.equals(sql1, sql2), true); + Assert.equals(SqlEqualHelper.equals(sql1, sql2), true); sql1 = "SELECT a,b,c,d FROM table1 WHERE column1 = 1 AND column2 = 2 order by a"; sql2 = "SELECT d,c,b,f FROM table1 WHERE column2 = 2 AND column1 = 1 order by a"; - Assert.equals(SqlParserEqualHelper.equals(sql1, sql2), false); + Assert.equals(SqlEqualHelper.equals(sql1, sql2), false); sql1 = "SELECT\n" + "页面,\n" @@ -64,6 +64,6 @@ class SqlParserEqualHelperTest { + "LIMIT\n" + "365"; - Assert.equals(SqlParserEqualHelper.equals(sql1, sql2), true); + Assert.equals(SqlEqualHelper.equals(sql1, sql2), true); } } \ No newline at end of file diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlRemoveHelperTest.java similarity index 86% rename from common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelperTest.java rename to common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlRemoveHelperTest.java index 60da626e3..ab06c81d3 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserRemoveHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlRemoveHelperTest.java @@ -8,13 +8,13 @@ import org.junit.jupiter.api.Test; /** * SqlParser Remove Helper Test */ -class SqlParserRemoveHelperTest { +class SqlRemoveHelperTest { @Test - void removeWhereHavingCondition() { + void testRemoveWhereHavingCondition() { String sql = "select 歌曲名 from 歌曲库 where sum(粉丝数) > 20000 and 2>1 and " + "sum(播放量) > 20000 and 1=1 HAVING sum(播放量) > 20000 and 3>1"; - sql = SqlParserRemoveHelper.removeNumberFilter(sql); + sql = SqlRemoveHelper.removeNumberFilter(sql); System.out.println(sql); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE sum(粉丝数) > 20000 AND sum(播放量) > 20000 HAVING sum(播放量) > 20000", @@ -22,7 +22,7 @@ class SqlParserRemoveHelperTest { sql = "SELECT 歌曲,sum(播放量) FROM 歌曲库\n" + "WHERE (歌手名 = '张三' AND 2 > 1) AND 数据日期 = '2023-11-07'\n" + "GROUP BY 歌曲名 HAVING sum(播放量) > 100000"; - sql = SqlParserRemoveHelper.removeNumberFilter(sql); + sql = SqlRemoveHelper.removeNumberFilter(sql); System.out.println(sql); Assert.assertEquals( "SELECT 歌曲, sum(播放量) FROM 歌曲库 WHERE (歌手名 = '张三') " @@ -31,7 +31,7 @@ class SqlParserRemoveHelperTest { sql = "SELECT 歌曲名,sum(播放量) FROM 歌曲库 WHERE (1 = 1 AND 1 = 1 AND 2 > 1 )" + "AND 1 = 1 AND 歌曲类型 IN ('类型一', '类型二') AND 歌手名 IN ('林俊杰', '周杰伦')" + "AND 数据日期 = '2023-11-07' GROUP BY 歌曲名 HAVING 2 > 1 AND SUM(播放量) >= 1000"; - sql = SqlParserRemoveHelper.removeNumberFilter(sql); + sql = SqlRemoveHelper.removeNumberFilter(sql); System.out.println(sql); Assert.assertEquals( "SELECT 歌曲名, sum(播放量) FROM 歌曲库 WHERE 歌曲类型 IN ('类型一', '类型二') " @@ -41,7 +41,7 @@ class SqlParserRemoveHelperTest { sql = "SELECT 品牌名称,法人 FROM 互联网企业 WHERE (2 > 1 AND 1 = 1) AND 数据日期 = '2023-10-31'" + "GROUP BY 品牌名称, 法人 HAVING 2 > 1 AND sum(注册资本) > 100000000 AND sum(营收占比) = 0.5 and 1 = 1"; - sql = SqlParserRemoveHelper.removeNumberFilter(sql); + sql = SqlRemoveHelper.removeNumberFilter(sql); System.out.println(sql); Assert.assertEquals( "SELECT 品牌名称, 法人 FROM 互联网企业 WHERE 数据日期 = '2023-10-31' GROUP BY " @@ -50,11 +50,11 @@ class SqlParserRemoveHelperTest { } @Test - void removeHavingCondition() { + void testRemoveHavingCondition() { String sql = "select 歌曲名 from 歌曲库 where 歌手名 = '周杰伦' HAVING sum(播放量) > 20000"; Set removeFieldNames = new HashSet<>(); removeFieldNames.add("播放量"); - String replaceSql = SqlParserRemoveHelper.removeHavingCondition(sql, removeFieldNames); + String replaceSql = SqlRemoveHelper.removeHavingCondition(sql, removeFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE 歌手名 = '周杰伦'", replaceSql); @@ -62,7 +62,7 @@ class SqlParserRemoveHelperTest { } @Test - void removeWhereCondition() { + void testRemoveWhereCondition() { String sql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + " order by 播放量 desc limit 11"; @@ -70,7 +70,7 @@ class SqlParserRemoveHelperTest { Set removeFieldNames = new HashSet<>(); removeFieldNames.add("歌曲名"); - String replaceSql = SqlParserRemoveHelper.removeWhereCondition(sql, removeFieldNames); + String replaceSql = SqlRemoveHelper.removeWhereCondition(sql, removeFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 " @@ -81,7 +81,7 @@ class SqlParserRemoveHelperTest { sql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + "and 歌曲名 in ('邓紫棋','周杰伦') and 歌曲名 in ('邓紫棋') and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + " order by 播放量 desc limit 11"; - replaceSql = SqlParserRemoveHelper.removeWhereCondition(sql, removeFieldNames); + replaceSql = SqlRemoveHelper.removeWhereCondition(sql, removeFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 " + "AND 数据日期 = '2023-08-09' AND " @@ -91,7 +91,7 @@ class SqlParserRemoveHelperTest { sql = "select 歌曲名 from 歌曲库 where (datediff('day', 发布日期, '2023-08-09') <= 1 " + "and 歌曲名 in ('邓紫棋','周杰伦') and 歌曲名 in ('邓紫棋')) and 数据日期 = '2023-08-09' " + " order by 播放量 desc limit 11"; - replaceSql = SqlParserRemoveHelper.removeWhereCondition(sql, removeFieldNames); + replaceSql = SqlRemoveHelper.removeWhereCondition(sql, removeFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1) " + "AND 数据日期 = '2023-08-09' ORDER BY 播放量 DESC LIMIT 11", diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelperTest.java similarity index 83% rename from common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelperTest.java rename to common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelperTest.java index d3872992b..607c584dd 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlReplaceHelperTest.java @@ -14,16 +14,16 @@ import java.util.Set; /** * SqlParserReplaceHelperTest */ -class SqlParserReplaceHelperTest { +class SqlReplaceHelperTest { @Test - void replaceSelectField() { + void testReplaceSelectField() { String sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; Map fieldMap = new HashMap<>(); fieldMap.put("播放量", "播放量1"); - sql = SqlParserReplaceHelper.replaceSelectFields(sql, fieldMap); + sql = SqlReplaceHelper.replaceSelectFields(sql, fieldMap); System.out.println(sql); Assert.assertEquals("SELECT 维度1, sum(播放量1) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", sql); @@ -32,25 +32,25 @@ class SqlParserReplaceHelperTest { + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; fieldMap = new HashMap<>(); fieldMap.put("播放量", "播放量1"); - sql = SqlParserReplaceHelper.replaceSelectFields(sql, fieldMap); + sql = SqlReplaceHelper.replaceSelectFields(sql, fieldMap); System.out.println(sql); Assert.assertEquals("SELECT 维度1, 播放量1 FROM 数据库 WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", sql); } @Test - void replaceAggField() { + void testReplaceAggField() { String sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1"; Map> fieldMap = new HashMap<>(); fieldMap.put("播放量", Pair.of("收听用户数", AggOperatorEnum.COUNT_DISTINCT.name())); - sql = SqlParserReplaceHelper.replaceAggFields(sql, fieldMap); + sql = SqlReplaceHelper.replaceAggFields(sql, fieldMap); System.out.println(sql); Assert.assertEquals("SELECT 维度1, count(DISTINCT 收听用户数) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 = '2023-11-17' GROUP BY 维度1", sql); } @Test - void replaceValue() { + void testReplaceValue() { String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + "and 歌手名 = '杰伦' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" @@ -62,7 +62,7 @@ class SqlParserReplaceHelperTest { valueMap.put("杰伦", "周杰伦"); filedNameToValueMap.put("歌手名", valueMap); - replaceSql = SqlParserReplaceHelper.replaceValue(replaceSql, filedNameToValueMap); + replaceSql = SqlReplaceHelper.replaceValue(replaceSql, filedNameToValueMap); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 AND " @@ -81,7 +81,7 @@ class SqlParserReplaceHelperTest { valueMap2.put("陈奕迅", "陈奕迅"); filedNameToValueMap2.put("歌手名", valueMap2); - replaceSql = SqlParserReplaceHelper.replaceValue(replaceSql, filedNameToValueMap2, false); + replaceSql = SqlReplaceHelper.replaceValue(replaceSql, filedNameToValueMap2, false); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 AND 歌手名 = '周杰伦' " @@ -92,7 +92,7 @@ class SqlParserReplaceHelperTest { + "and 歌手名 = '周杰' and 歌手名 = '林俊' and 歌手名 = '陈' and 歌曲发布时 = '2023-08-01') and 数据日期 = '2023-08-09' " + " order by 播放量 desc limit 11"; - replaceSql = SqlParserReplaceHelper.replaceValue(replaceSql, filedNameToValueMap2, false); + replaceSql = SqlReplaceHelper.replaceValue(replaceSql, filedNameToValueMap2, false); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1 AND 歌手名 = '周杰伦' " @@ -105,7 +105,7 @@ class SqlParserReplaceHelperTest { + ") ) and 数据日期 = '2023-08-09' " + " order by 播放量 desc limit 11"; - replaceSql = SqlParserReplaceHelper.replaceValue(replaceSql, filedNameToValueMap2, false); + replaceSql = SqlReplaceHelper.replaceValue(replaceSql, filedNameToValueMap2, false); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1 AND " @@ -115,7 +115,7 @@ class SqlParserReplaceHelperTest { } @Test - void replaceFieldNameByValue() { + void testReplaceFieldNameByValue() { String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + "and 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" @@ -124,7 +124,7 @@ class SqlParserReplaceHelperTest { Map> fieldValueToFieldNames = new HashMap<>(); fieldValueToFieldNames.put("邓紫棋", Collections.singleton("歌手名")); - replaceSql = SqlParserReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); + replaceSql = SqlReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 AND " @@ -135,7 +135,7 @@ class SqlParserReplaceHelperTest { + "and 歌曲名 like '%邓紫棋%' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + " order by 播放量 desc limit 11"; - replaceSql = SqlParserReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); + replaceSql = SqlReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 " @@ -152,7 +152,7 @@ class SqlParserReplaceHelperTest { + "and 歌手名 = '林俊杰' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + " order by 播放量 desc limit 11"; - replaceSql = SqlParserReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); + replaceSql = SqlReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE datediff('day', 发布日期, '2023-08-09') <= 1 " @@ -163,7 +163,7 @@ class SqlParserReplaceHelperTest { + "and 歌手名 = '林俊杰' and 歌曲发布时 = '2023-08-01') and 数据日期 = '2023-08-09'" + " order by 播放量 desc limit 11"; - replaceSql = SqlParserReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); + replaceSql = SqlReplaceHelper.replaceFieldNameByValue(replaceSql, fieldValueToFieldNames); Assert.assertEquals( "SELECT 歌曲名 FROM 歌曲库 WHERE (datediff('day', 发布日期, '2023-08-09') <= 1 AND " @@ -173,37 +173,37 @@ class SqlParserReplaceHelperTest { } @Test - void replaceUnionFields() { + void testReplaceUnionFields() { Map fieldToBizName1 = new HashMap<>(); fieldToBizName1.put("公司成立时间", "company_established_time"); fieldToBizName1.put("年营业额", "annual_turnover"); String replaceSql = "SELECT * FROM 互联网企业 ORDER BY 公司成立时间 DESC LIMIT 3 " + "UNION SELECT * FROM 互联网企业 ORDER BY 年营业额 DESC LIMIT 5"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName1); - replaceSql = SqlParserReplaceHelper.replaceTable(replaceSql, "internet"); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName1); + replaceSql = SqlReplaceHelper.replaceTable(replaceSql, "internet"); Assert.assertEquals( "SELECT * FROM internet ORDER BY company_established_time DESC LIMIT 3 " + "UNION SELECT * FROM internet ORDER BY annual_turnover DESC LIMIT 5", replaceSql); } @Test - void replaceFields() { + void testReplaceFields() { Map fieldToBizName = initParams(); String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'" + " order by 播放量 desc limit 11"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT song_name FROM 歌曲库 WHERE (publish_date >= '2023-08-08' AND publish_date <= '2023-08-09')" + " AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND song_publis_date = '2023-08-01'" + " ORDER BY play_count DESC LIMIT 11", replaceSql); replaceSql = "select 品牌名称 from 互联网企业 where datediff('year', 品牌成立时间, '2023-11-04') > 17 and 注册资本 = 50000000"; - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); - replaceSql = SqlParserRemoveHelper.removeNumberFilter(replaceSql); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlRemoveHelper.removeNumberFilter(replaceSql); Assert.assertEquals( "SELECT 品牌名称 FROM 互联网企业 WHERE 品牌成立时间 < '2006-11-04' AND 注册资本 = 50000000", replaceSql); @@ -211,8 +211,8 @@ class SqlParserReplaceHelperTest { + "where datediff('year', 数据日期, '2023-09-03') <= 0.5 " + "group by MONTH(数据日期) order by sum(访问次数) desc limit 1"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT MONTH(sys_imp_date), sum(pv) FROM 内容库产品 WHERE (sys_imp_date >= '2023-03-03' " @@ -223,8 +223,8 @@ class SqlParserReplaceHelperTest { + "where datediff('year', 数据日期, '2023-09-03') <= 0.5 " + "group by MONTH(数据日期) HAVING sum(访问次数) > 1000"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT MONTH(sys_imp_date), sum(pv) FROM 内容库产品 WHERE (sys_imp_date >= '2023-03-03' AND" @@ -234,8 +234,8 @@ class SqlParserReplaceHelperTest { replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) " + "in (2022, 2023) and 数据日期 = '2023-08-14' group by YEAR(发行日期)"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT YEAR(publish_date), count(song_name) FROM 歌曲库 " @@ -247,8 +247,8 @@ class SqlParserReplaceHelperTest { + "where YEAR(发行日期) in (2022, 2023) and 数据日期 = '2023-08-14' " + "group by 发行日期"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT YEAR(publish_date), count(song_name) FROM 歌曲库 " @@ -256,11 +256,11 @@ class SqlParserReplaceHelperTest { + " GROUP BY publish_date", replaceSql); - replaceSql = SqlParserReplaceHelper.replaceFields( + replaceSql = SqlReplaceHelper.replaceFields( "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-11') <= 1 " + "and 结算播放量 > 1000000 and datediff('day', 数据日期, '2023-08-11') <= 30", fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT song_name FROM 歌曲库 WHERE (publish_date >= '2022-08-11' " @@ -268,64 +268,64 @@ class SqlParserReplaceHelperTest { + "(sys_imp_date >= '2023-07-12' AND sys_imp_date <= '2023-08-11')", replaceSql); - replaceSql = SqlParserReplaceHelper.replaceFields( + replaceSql = SqlReplaceHelper.replaceFields( "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 " + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT song_name FROM 歌曲库 WHERE (publish_date >= '2023-08-08' AND publish_date <= '2023-08-09')" + " AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' ORDER BY play_count DESC LIMIT 11", replaceSql); - replaceSql = SqlParserReplaceHelper.replaceFields( + replaceSql = SqlReplaceHelper.replaceFields( "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') = 0 " + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT song_name FROM 歌曲库 WHERE (publish_date >= '2023-01-01' AND publish_date <= '2023-08-09')" + " AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' ORDER BY play_count DESC LIMIT 11", replaceSql); - replaceSql = SqlParserReplaceHelper.replaceFields( + replaceSql = SqlReplaceHelper.replaceFields( "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') <= 0.5 " + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT song_name FROM 歌曲库 WHERE (publish_date >= '2023-02-09' AND publish_date <= '2023-08-09')" + " AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' ORDER BY play_count DESC LIMIT 11", replaceSql); - replaceSql = SqlParserReplaceHelper.replaceFields( + replaceSql = SqlReplaceHelper.replaceFields( "select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') >= 0.5 " + "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); - replaceSql = SqlParserRemoveHelper.removeNumberFilter(replaceSql); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlRemoveHelper.removeNumberFilter(replaceSql); Assert.assertEquals( "SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-02-09' AND" + " singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09'" + " ORDER BY play_count DESC LIMIT 11", replaceSql); - replaceSql = SqlParserReplaceHelper.replaceFields( + replaceSql = SqlReplaceHelper.replaceFields( "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08' and 用户 ='alice'" + " and 发布日期 ='11' order by 访问次数 desc limit 1", fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT department, user_id FROM 超音数 WHERE sys_imp_date = '2023-08-08'" + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1", replaceSql); - replaceSql = SqlParserReplaceHelper.replaceTable(replaceSql, "s2"); + replaceSql = SqlReplaceHelper.replaceTable(replaceSql, "s2"); - replaceSql = SqlParserAddHelper.addFieldsToSelect(replaceSql, Collections.singletonList("field_a")); + replaceSql = SqlAddHelper.addFieldsToSelect(replaceSql, Collections.singletonList("field_a")); - replaceSql = SqlParserReplaceHelper.replaceFields( + replaceSql = SqlReplaceHelper.replaceFields( "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 ='alice' and 发布日期 ='11' group by 部门 limit 1", fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT department, sum(pv) FROM 超音数 WHERE sys_imp_date = '2023-08-08'" @@ -333,8 +333,8 @@ class SqlParserReplaceHelperTest { replaceSql = "select sum(访问次数) from 超音数 where 数据日期 >= '2023-08-06' " + "and 数据日期 <= '2023-08-06' and 部门 = 'hr'"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT sum(pv) FROM 超音数 WHERE sys_imp_date >= '2023-08-06' " @@ -342,8 +342,8 @@ class SqlParserReplaceHelperTest { replaceSql = "SELECT 歌曲名称, sum(评分) FROM CSpider WHERE(1 < 2) AND 数据日期 = '2023-10-15' " + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider WHERE 语种 = '英文')"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT song_name, sum(user_id) FROM CSpider WHERE (1 < 2) AND " @@ -353,8 +353,8 @@ class SqlParserReplaceHelperTest { replaceSql = "SELECT sum(评分)/ (SELECT sum(评分) FROM CSpider WHERE 数据日期 = '2023-10-15')" + " FROM CSpider WHERE 数据日期 = '2023-10-15' " + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider WHERE 语种 = '英文')"; - replaceSql = SqlParserReplaceHelper.replaceFields(replaceSql, fieldToBizName); - replaceSql = SqlParserReplaceHelper.replaceFunction(replaceSql); + replaceSql = SqlReplaceHelper.replaceFields(replaceSql, fieldToBizName); + replaceSql = SqlReplaceHelper.replaceFunction(replaceSql); Assert.assertEquals( "SELECT sum(user_id) / (SELECT sum(user_id) FROM CSpider WHERE sys_imp_date = '2023-10-15') " @@ -363,25 +363,25 @@ class SqlParserReplaceHelperTest { } @Test - void replaceTable() { + void testReplaceTable() { String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - String replaceSql = SqlParserReplaceHelper.replaceTable(sql, "s2"); + String replaceSql = SqlReplaceHelper.replaceTable(sql, "s2"); Assert.assertEquals( "SELECT 部门, sum(访问次数) FROM s2 WHERE 数据日期 = '2023-08-08' " + "AND 用户 = alice AND 发布日期 = '11' GROUP BY 部门 LIMIT 1", replaceSql); sql = "select * from 互联网企业 order by 公司成立时间 desc limit 3 union select * from 互联网企业 order by 年营业额 desc limit 5"; - replaceSql = SqlParserReplaceHelper.replaceTable(sql, "internet"); + replaceSql = SqlReplaceHelper.replaceTable(sql, "internet"); Assert.assertEquals( "SELECT * FROM internet ORDER BY 公司成立时间 DESC LIMIT 3 " + "UNION SELECT * FROM internet ORDER BY 年营业额 DESC LIMIT 5", replaceSql); sql = "SELECT * FROM CSpider音乐 WHERE (评分 < (SELECT min(评分) " + "FROM CSpider音乐 WHERE 语种 = '英文')) AND 数据日期 = '2023-10-11'"; - replaceSql = SqlParserReplaceHelper.replaceTable(sql, "cspider"); + replaceSql = SqlReplaceHelper.replaceTable(sql, "cspider"); Assert.assertEquals( "SELECT * FROM cspider WHERE (评分 < (SELECT min(评分) FROM " @@ -390,7 +390,7 @@ class SqlParserReplaceHelperTest { sql = "SELECT 歌曲名称, sum(评分) FROM CSpider音乐 WHERE(1 < 2) AND 数据日期 = '2023-10-15' " + "GROUP BY 歌曲名称 HAVING sum(评分) < ( SELECT min(评分) FROM CSpider音乐 WHERE 语种 = '英文')"; - replaceSql = SqlParserReplaceHelper.replaceTable(sql, "cspider"); + replaceSql = SqlReplaceHelper.replaceTable(sql, "cspider"); Assert.assertEquals( "SELECT 歌曲名称, sum(评分) FROM cspider WHERE (1 < 2) AND 数据日期 = " @@ -400,7 +400,7 @@ class SqlParserReplaceHelperTest { } @Test - void replaceFunctionName() { + void testReplaceFunctionName() { String sql = "select 公司名称,平均(注册资本),总部地点 from 互联网企业 where\n" + "年营业额 >= 28800000000 and 最大(注册资本)>10000 \n" @@ -409,7 +409,7 @@ class SqlParserReplaceHelperTest { Map map = new HashMap<>(); map.put("平均", "avg"); map.put("最大", "max"); - sql = SqlParserReplaceHelper.replaceFunction(sql, map); + sql = SqlReplaceHelper.replaceFunction(sql, map); System.out.println(sql); Assert.assertEquals("SELECT 公司名称, avg(注册资本), 总部地点 FROM 互联网企业 WHERE 年营业额 >= 28800000000 AND " + "max(注册资本) > 10000 GROUP BY 公司名称 HAVING avg(注册资本) > 10000 ORDER BY avg(注册资本) DESC LIMIT 5", sql); @@ -418,7 +418,7 @@ class SqlParserReplaceHelperTest { + " datediff('month', 数据日期, '2023-09-02') <= 6 group by MONTH(数据日期)"; Map functionMap = new HashMap<>(); functionMap.put("MONTH".toLowerCase(), "toMonth"); - String replaceSql = SqlParserReplaceHelper.replaceFunction(sql, functionMap); + String replaceSql = SqlReplaceHelper.replaceFunction(sql, functionMap); Assert.assertEquals( "SELECT toMonth(数据日期) AS 月份, avg(访问次数) AS 平均访问次数 FROM 内容库产品 WHERE" @@ -427,7 +427,7 @@ class SqlParserReplaceHelperTest { sql = "select month(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" + " datediff('month', 数据日期, '2023-09-02') <= 6 group by MONTH(数据日期)"; - replaceSql = SqlParserReplaceHelper.replaceFunction(sql, functionMap); + replaceSql = SqlReplaceHelper.replaceFunction(sql, functionMap); Assert.assertEquals( "SELECT toMonth(数据日期) AS 月份, avg(访问次数) AS 平均访问次数 FROM 内容库产品 WHERE" @@ -436,7 +436,7 @@ class SqlParserReplaceHelperTest { sql = "select month(数据日期) as 月份, avg(访问次数) as 平均访问次数 from 内容库产品 where" + " (datediff('month', 数据日期, '2023-09-02') <= 6) and 数据日期 = '2023-10-10' group by MONTH(数据日期)"; - replaceSql = SqlParserReplaceHelper.replaceFunction(sql, functionMap); + replaceSql = SqlReplaceHelper.replaceFunction(sql, functionMap); Assert.assertEquals( "SELECT toMonth(数据日期) AS 月份, avg(访问次数) AS 平均访问次数 FROM 内容库产品 WHERE" @@ -446,10 +446,10 @@ class SqlParserReplaceHelperTest { } @Test - void replaceAlias() { + void testReplaceAlias() { String sql = "select 部门, sum(访问次数) as 总访问次数 from 超音数 where " + "datediff('day', 数据日期, '2023-09-05') <= 3 group by 部门 order by 总访问次数 desc limit 10"; - String replaceSql = SqlParserReplaceHelper.replaceAlias(sql); + String replaceSql = SqlReplaceHelper.replaceAlias(sql); System.out.println(replaceSql); Assert.assertEquals( "SELECT 部门, sum(访问次数) FROM 超音数 WHERE " @@ -459,7 +459,7 @@ class SqlParserReplaceHelperTest { sql = "select 部门, sum(访问次数) as 总访问次数 from 超音数 where " + "(datediff('day', 数据日期, '2023-09-05') <= 3) and 数据日期 = '2023-10-10' " + "group by 部门 order by 总访问次数 desc limit 10"; - replaceSql = SqlParserReplaceHelper.replaceAlias(sql); + replaceSql = SqlReplaceHelper.replaceAlias(sql); System.out.println(replaceSql); Assert.assertEquals( "SELECT 部门, sum(访问次数) FROM 超音数 WHERE " @@ -470,7 +470,7 @@ class SqlParserReplaceHelperTest { sql = "select 部门, sum(访问次数) as 访问次数 from 超音数 where " + "(datediff('day', 数据日期, '2023-09-05') <= 3) and 数据日期 = '2023-10-10' " + "group by 部门 order by 访问次数 desc limit 10"; - replaceSql = SqlParserReplaceHelper.replaceAlias(sql); + replaceSql = SqlReplaceHelper.replaceAlias(sql); System.out.println(replaceSql); Assert.assertEquals( "SELECT 部门, sum(访问次数) AS 访问次数 FROM 超音数 WHERE (datediff('day', 数据日期, " diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectFunctionHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectFunctionHelperTest.java similarity index 72% rename from common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectFunctionHelperTest.java rename to common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectFunctionHelperTest.java index 0cc921e66..52b550f4e 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectFunctionHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectFunctionHelperTest.java @@ -7,68 +7,68 @@ import org.junit.jupiter.api.Test; /** * SqlParserSelectHelper Test */ -class SqlParserSelectFunctionHelperTest { +class SqlSelectFunctionHelperTest { @Test - void hasAggregateFunction() throws JSQLParserException { + void testHasAggregateFunction() throws JSQLParserException { String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - boolean hasAggregateFunction = SqlParserSelectFunctionHelper.hasAggregateFunction(sql); + boolean hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, true); sql = "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - hasAggregateFunction = SqlParserSelectFunctionHelper.hasAggregateFunction(sql); + hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, true); sql = "SELECT count(1) FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice'" + " AND publish_date = '11' ORDER BY pv DESC LIMIT 1"; - hasAggregateFunction = SqlParserSelectFunctionHelper.hasAggregateFunction(sql); + hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, true); sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"; - hasAggregateFunction = SqlParserSelectFunctionHelper.hasAggregateFunction(sql); + hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, false); sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" + " AND user_id = 'alice' AND publish_date = '11'"; - hasAggregateFunction = SqlParserSelectFunctionHelper.hasAggregateFunction(sql); + hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, false); sql = "SELECT user_name, pv FROM t_34 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10"; - hasAggregateFunction = SqlParserSelectFunctionHelper.hasAggregateFunction(sql); + hasAggregateFunction = SqlSelectFunctionHelper.hasAggregateFunction(sql); Assert.assertEquals(hasAggregateFunction, true); } @Test - void hasFunction() throws JSQLParserException { + void testHasFunction() throws JSQLParserException { String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - boolean hasFunction = SqlParserSelectFunctionHelper.hasFunction(sql, "sum"); + boolean hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "sum"); Assert.assertEquals(hasFunction, true); sql = "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - hasFunction = SqlParserSelectFunctionHelper.hasFunction(sql, "count"); + hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "count"); Assert.assertEquals(hasFunction, true); sql = "select 部门,count (*) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - hasFunction = SqlParserSelectFunctionHelper.hasFunction(sql, "count"); + hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "count"); Assert.assertEquals(hasFunction, true); sql = "SELECT user_name, pv FROM t_34 WHERE sys_imp_date <= '2023-09-03' " + "AND sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10"; - hasFunction = SqlParserSelectFunctionHelper.hasFunction(sql, "sum"); + hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "sum"); Assert.assertEquals(hasFunction, false); sql = "select 部门,min (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - hasFunction = SqlParserSelectFunctionHelper.hasFunction(sql, "min"); + hasFunction = SqlSelectFunctionHelper.hasFunction(sql, "min"); Assert.assertEquals(hasFunction, true); } diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelperTest.java similarity index 78% rename from common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java rename to common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelperTest.java index f82025f23..6693850e5 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelperTest.java @@ -9,23 +9,23 @@ import org.junit.jupiter.api.Test; /** * SqlParserSelectHelper Test */ -class SqlParserSelectHelperTest { +class SqlSelectHelperTest { @Test - void getWhereFilterExpression() { + void testGetWhereFilterExpression() { - Select selectStatement = SqlParserSelectHelper.getSelect( + Select selectStatement = SqlSelectHelper.getSelect( "select 用户名, 访问次数 from 超音数 where 用户名 in ('alice', 'lucy')"); System.out.println(selectStatement); - List fieldExpression = SqlParserSelectHelper.getFilterExpression( + List fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE " + "sys_imp_date = '2023-08-08' AND YEAR(publish_date) = 2023 " + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + " AND YEAR(publish_date) = 2023 " + " AND MONTH(publish_date) = 8" @@ -33,64 +33,64 @@ class SqlParserSelectHelperTest { System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" + " AND YEAR(publish_date) = 2023 " + " AND MONTH(publish_date) = 8 AND DAY(publish_date) =20 " + " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' " + "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE " + "user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE " + "user_id = 'alice' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, user_id, field_a FROM s2 WHERE " + "user_id like '%alice%' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, pv FROM s2 WHERE " + "user_id like '%alice%' AND publish_date > 10000 " + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, pv FROM s2 WHERE " + "(user_id like '%alice%' AND publish_date > 10000) and sys_imp_date = '2023-08-08' " + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, pv FROM s2 WHERE " + "(user_id like '%alice%' AND publish_date > 10000) and song_name in " + "('七里香','晴天') and sys_imp_date = '2023-08-08' " @@ -98,7 +98,7 @@ class SqlParserSelectHelperTest { System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, pv FROM s2 WHERE " + "(user_id like '%alice%' AND publish_date > 10000) and song_name in (1,2) " + "and sys_imp_date = '2023-08-08' " @@ -106,7 +106,7 @@ class SqlParserSelectHelperTest { System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "SELECT department, pv FROM s2 WHERE " + "(user_id like '%alice%' AND publish_date > 10000) and 1 in (1) " + "and sys_imp_date = '2023-08-08' " @@ -114,12 +114,12 @@ class SqlParserSelectHelperTest { System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression("SELECT sum(销量) / (SELECT sum(销量) FROM 营销月模型 " + fieldExpression = SqlSelectHelper.getFilterExpression("SELECT sum(销量) / (SELECT sum(销量) FROM 营销月模型 " + "WHERE MONTH(数据日期) = 9) FROM 营销月模型 WHERE 国家中文名 = '肯尼亚' AND MONTH(数据日期) = 9"); System.out.println(fieldExpression); - fieldExpression = SqlParserSelectHelper.getFilterExpression( + fieldExpression = SqlSelectHelper.getFilterExpression( "select 等级, count(*) from 歌手 where 别名 = '港台' or 活跃区域 = '港台' and" + " datediff('day', 数据日期, '2023-12-24') <= 0 group by 等级"); @@ -128,79 +128,79 @@ class SqlParserSelectHelperTest { } @Test - void getAllFields() { + void testGetAllFields() { - List allFields = SqlParserSelectHelper.getAllFields( + List allFields = SqlSelectHelper.getAllFields( "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'" + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); Assert.assertEquals(allFields.size(), 6); - allFields = SqlParserSelectHelper.getAllFields( + allFields = SqlSelectHelper.getAllFields( "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date >= '2023-08-08'" + " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"); Assert.assertEquals(allFields.size(), 6); - allFields = SqlParserSelectHelper.getAllFields( + allFields = SqlSelectHelper.getAllFields( "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 = 'alice'" + " and 发布日期 ='11' group by 部门 limit 1"); Assert.assertEquals(allFields.size(), 5); - allFields = SqlParserSelectHelper.getAllFields( + allFields = SqlSelectHelper.getAllFields( "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + "sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10 "); Assert.assertEquals(allFields.size(), 3); - allFields = SqlParserSelectHelper.getAllFields( + allFields = SqlSelectHelper.getAllFields( "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"); Assert.assertEquals(allFields.size(), 3); - allFields = SqlParserSelectHelper.getAllFields( + allFields = SqlSelectHelper.getAllFields( "SELECT department, user_id, field_a FROM s2 WHERE " + "(user_id = 'alice' AND publish_date = '11') and sys_imp_date " + "= '2023-08-08' ORDER BY pv DESC LIMIT 1"); Assert.assertEquals(allFields.size(), 6); - allFields = SqlParserSelectHelper.getAllFields( + allFields = SqlSelectHelper.getAllFields( "SELECT * FROM CSpider WHERE (评分 < (SELECT min(评分) FROM CSpider WHERE 语种 = '英文' ))" + " AND 数据日期 = '2023-10-12'"); Assert.assertEquals(allFields.size(), 3); - allFields = SqlParserSelectHelper.getAllFields("SELECT sum(销量) / (SELECT sum(销量) FROM 营销 " + allFields = SqlSelectHelper.getAllFields("SELECT sum(销量) / (SELECT sum(销量) FROM 营销 " + "WHERE MONTH(数据日期) = 9) FROM 营销 WHERE 国家中文名 = '中国' AND MONTH(数据日期) = 9"); Assert.assertEquals(allFields.size(), 3); - allFields = SqlParserSelectHelper.getAllFields( + allFields = SqlSelectHelper.getAllFields( "SELECT 用户, 页面 FROM 超音数用户部门 GROUP BY 用户, 页面 ORDER BY count(*) DESC"); Assert.assertEquals(allFields.size(), 2); } @Test - void getSelectFields() { + void testGetSelectFields() { String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' " + "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1"; - List selectFields = SqlParserSelectHelper.getSelectFields(sql); + List selectFields = SqlSelectHelper.getSelectFields(sql); Assert.assertEquals(selectFields.contains("访问次数"), true); Assert.assertEquals(selectFields.contains("部门"), true); } @Test - void getWhereFields() { + void testGetWhereFields() { String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; - List selectFields = SqlParserSelectHelper.getWhereFields(sql); + List selectFields = SqlSelectHelper.getWhereFields(sql); Assert.assertEquals(selectFields.contains("发布日期"), true); Assert.assertEquals(selectFields.contains("数据日期"), true); @@ -208,7 +208,7 @@ class SqlParserSelectHelperTest { sql = "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08'" + " and 用户 = 'alice' and 发布日期 ='11' order by 访问次数 limit 1"; - selectFields = SqlParserSelectHelper.getWhereFields(sql); + selectFields = SqlSelectHelper.getWhereFields(sql); Assert.assertEquals(selectFields.contains("发布日期"), true); Assert.assertEquals(selectFields.contains("数据日期"), true); @@ -217,7 +217,7 @@ class SqlParserSelectHelperTest { sql = "select 部门,用户 from 超音数 where" + " (用户 = 'alice' and 发布日期 ='11') and 数据日期 = '2023-08-08' " + "order by 访问次数 limit 1"; - selectFields = SqlParserSelectHelper.getWhereFields(sql); + selectFields = SqlSelectHelper.getWhereFields(sql); Assert.assertEquals(selectFields.contains("发布日期"), true); Assert.assertEquals(selectFields.contains("数据日期"), true); @@ -225,49 +225,49 @@ class SqlParserSelectHelperTest { } @Test - void getOrderByFields() { + void testGetOrderByFields() { String sql = "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08'" + " and 用户 = 'alice' and 发布日期 ='11' order by 访问次数 limit 1"; - List selectFields = SqlParserSelectHelper.getOrderByFields(sql); + List selectFields = SqlSelectHelper.getOrderByFields(sql); Assert.assertEquals(selectFields.contains("访问次数"), true); sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + "sys_imp_date >= '2023-08-04' GROUP BY user_name ORDER BY sum(pv) DESC LIMIT 10 "; - selectFields = SqlParserSelectHelper.getOrderByFields(sql); + selectFields = SqlSelectHelper.getOrderByFields(sql); Assert.assertEquals(selectFields.contains("pv"), true); } @Test - void getGroupByFields() { + void testGetGroupByFields() { String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; - List selectFields = SqlParserSelectHelper.getGroupByFields(sql); + List selectFields = SqlSelectHelper.getGroupByFields(sql); Assert.assertEquals(selectFields.contains("部门"), true); } @Test - void getHavingExpression() { + void testGetHavingExpression() { String sql = "SELECT user_name FROM 超音数 WHERE sys_imp_date <= '2023-09-03' AND " + "sys_imp_date >= '2023-08-04' GROUP BY user_name HAVING sum(pv) > 1000"; - List leftExpressionList = SqlParserSelectHelper.getHavingExpression(sql); + List leftExpressionList = SqlSelectHelper.getHavingExpression(sql); Assert.assertEquals(leftExpressionList.get(0).toString(), "sum(pv)"); } @Test - void getAggregateFields() { + void testGetAggregateFields() { String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'" + " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1"; - List selectFields = SqlParserSelectHelper.getAggregateFields(sql); + List selectFields = SqlSelectHelper.getAggregateFields(sql); Assert.assertEquals(selectFields.contains("访问次数"), true); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index a2cdb275b..679563c9d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import lombok.Data; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; @@ -156,7 +156,7 @@ public class QueryStructReq extends SemanticQueryReq { } /** - * convert queryStructReq to QueryS2QLReq + * convert queryStructReq to QueryS2SQLReq * * @param queryStructReq * @return @@ -261,7 +261,7 @@ public class QueryStructReq extends SemanticQueryReq { String sql = select.toString(); if (StringUtils.isNotBlank(whereClause)) { Expression expression = CCJSqlParserUtil.parseCondExpression(whereClause); - sql = SqlParserAddHelper.addWhere(sql, expression); + sql = SqlAddHelper.addWhere(sql, expression); } //7.Set DateInfo @@ -269,7 +269,7 @@ public class QueryStructReq extends SemanticQueryReq { String dateWhereStr = dateModeUtils.getDateWhereStr(queryStructReq.getDateInfo()); if (StringUtils.isNotBlank(dateWhereStr)) { Expression expression = CCJSqlParserUtil.parseCondExpression(dateWhereStr); - sql = SqlParserAddHelper.addWhere(sql, expression); + sql = SqlAddHelper.addWhere(sql, expression); } return sql; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java index fd9524313..7e6fa0349 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.headless.core.adaptor.db; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.Constants; import java.util.HashMap; @@ -49,7 +49,7 @@ public class ClickHouseAdaptor extends DbAdaptor { functionMap.put("MONTH".toLowerCase(), "toMonth"); functionMap.put("DAY".toLowerCase(), "toDayOfMonth"); functionMap.put("YEAR".toLowerCase(), "toYear"); - return SqlParserReplaceHelper.replaceFunction(sql, functionMap); + return SqlReplaceHelper.replaceFunction(sql, functionMap); } @Override diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java index f9cd19f00..0524c3dc1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.adaptor.db; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; import java.util.HashMap; import java.util.Map; @@ -47,7 +47,7 @@ public class PostgresqlAdaptor extends DbAdaptor { functionMap.put("MONTH".toLowerCase(), "toMonth"); functionMap.put("DAY".toLowerCase(), "toDayOfMonth"); functionMap.put("YEAR".toLowerCase(), "toYear"); - return SqlParserReplaceHelper.replaceFunction(sql, functionMap); + return SqlReplaceHelper.replaceFunction(sql, functionMap); } @Override diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 581277d76..77333cf05 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -8,8 +8,8 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.StringUtil; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; @@ -279,7 +279,7 @@ public class SqlGenerateUtils { Set visitedMetric, Set measures, Set dimensions) { - Set fields = SqlParserSelectHelper.getColumnFromExpr(expression); + Set fields = SqlSelectHelper.getColumnFromExpr(expression); if (!CollectionUtils.isEmpty(fields)) { Map replace = new HashMap<>(); for (String field : fields) { @@ -321,7 +321,7 @@ public class SqlGenerateUtils { } } if (!CollectionUtils.isEmpty(replace)) { - String expr = SqlParserReplaceHelper.replaceExpression(expression, replace); + String expr = SqlReplaceHelper.replaceExpression(expression, replace); log.info("derived measure {}->{}", expression, expr); return expr; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index 2d856b1ca..afc48d73d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -7,8 +7,8 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; @@ -90,7 +90,7 @@ public class DimValueAspect { MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySqlReq.getModelIds())); String sql = querySqlReq.getSql(); log.info("correctorSql before replacing:{}", sql); - List fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql); + List fieldExpressionList = SqlSelectHelper.getWhereExpressions(sql); List dimensions = dimensionService.getDimensions(metaFilter); Set fieldNames = dimensions.stream().map(SchemaItem::getName).collect(Collectors.toSet()); Map> filedNameToValueMap = new HashMap<>(); @@ -117,7 +117,7 @@ public class DimValueAspect { replaceInCondition(expression, dimension, filedNameToValueMap); } } - sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap); + sql = SqlReplaceHelper.replaceValue(sql, filedNameToValueMap); log.info("correctorSql after replacing:{}", sql); querySqlReq.setSql(sql); Map> techNameToBizName = getTechNameToBizName(dimensions); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java index dc872a6f3..f6c73e5b2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; @@ -283,7 +283,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { try { Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) "); if (StringUtils.isNotEmpty(joiner.toString())) { - String sql = SqlParserAddHelper.addWhere(querySqlReq.getSql(), expression); + String sql = SqlAddHelper.addWhere(querySqlReq.getSql(), expression); log.info("before doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql()); querySqlReq.setSql(sql); log.info("after doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java index 2a1788fc7..4c1f3d64a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.headless.server.utils; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; @@ -63,7 +63,7 @@ public class MetricCheckUtils { private static boolean hasAggregateFunction(String expr) { String sql = String.format("select %s from table", expr); - return SqlParserSelectFunctionHelper.hasAggregateFunction(sql); + return SqlSelectFunctionHelper.hasAggregateFunction(sql); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index acba99c5c..e55be5371 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -6,9 +6,9 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.SchemaItem; @@ -72,12 +72,12 @@ public class QueryReqConverter { //3.correct tableName correctTableName(querySQLReq); - String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql()); + String tableName = SqlSelectHelper.getTableName(querySQLReq.getSql()); if (StringUtils.isEmpty(tableName)) { return new QueryStatement(); } //4.build MetricTables - List allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()); + List allFields = SqlSelectHelper.getAllFields(querySQLReq.getSql()); List metrics = getMetrics(semanticSchemaResp, allFields); QueryStructReq queryStructReq = new QueryStructReq(); MetricTable metricTable = new MetricTable(); @@ -135,9 +135,9 @@ public class QueryReqConverter { // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" String sql = databaseReq.getSql(); - if (!SqlParserSelectHelper.hasGroupBy(sql) - || SqlParserSelectFunctionHelper.hasFunction(sql, "count") - || SqlParserSelectFunctionHelper.hasFunction(sql, "count_distinct")) { + if (!SqlSelectHelper.hasGroupBy(sql) + || SqlSelectFunctionHelper.hasFunction(sql, "count") + || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) { return AggOption.NATIVE; } return AggOption.DEFAULT; @@ -147,7 +147,7 @@ public class QueryReqConverter { Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); String sql = databaseReq.getSql(); log.info("convert name to bizName before:{}", sql); - String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); + String replaceFields = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); log.info("convert name to bizName after:{}", replaceFields); databaseReq.setSql(replaceFields); } @@ -214,7 +214,7 @@ public class QueryReqConverter { public void correctTableName(QuerySqlReq querySqlReq) { String sql = querySqlReq.getSql(); - sql = SqlParserReplaceHelper.replaceTable(sql, + sql = SqlReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + querySqlReq.getViewId()); querySqlReq.setSql(sql); } @@ -239,7 +239,7 @@ public class QueryReqConverter { measures, replaces); if (!CollectionUtils.isEmpty(replaces)) { // metricTable sql use measures replace metric - sql = SqlParserReplaceHelper.replaceSqlByExpression(sql, replaces); + sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces); metricTable.setAggOption(AggOption.NATIVE); // metricTable use measures replace metric if (!CollectionUtils.isEmpty(measures)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index 4cd3ea92a..f1671ff74 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -8,7 +8,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; @@ -126,7 +126,7 @@ public class QueryStructUtils { } public Set getResName(QuerySqlReq querySqlReq) { - Set resNameSet = SqlParserSelectHelper.getAllFields(querySqlReq.getSql()) + Set resNameSet = SqlSelectHelper.getAllFields(querySqlReq.getSql()) .stream().collect(Collectors.toSet()); return resNameSet; } @@ -170,7 +170,7 @@ public class QueryStructUtils { public Set getFilterResNameEnExceptInternalCol(QuerySqlReq querySqlReq) { String sql = querySqlReq.getSql(); - Set resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); + Set resNameEnSet = SqlSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } @@ -235,7 +235,7 @@ public class QueryStructUtils { } public DateConf getDateConfBySql(String sql) { - List fieldExpressions = SqlParserSelectHelper.getFilterExpression(sql); + List fieldExpressions = SqlSelectHelper.getFilterExpression(sql); if (!CollectionUtils.isEmpty(fieldExpressions)) { Set dateList = new HashSet<>(); String startDate = ""; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index 97a12d37a..8f65682b6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.util.SqlFilterUtils; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.enums.QueryOptMode; @@ -100,8 +100,8 @@ public class StatUtils { public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) { QueryStat queryStatInfo = new QueryStat(); - List aggFields = SqlParserSelectHelper.getAggregateFields(querySqlReq.getSql()); - List allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql()); + List aggFields = SqlSelectHelper.getAggregateFields(querySqlReq.getSql()); + List allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql()); List dimensions = allFields.stream().filter(aggFields::contains).collect(Collectors.toList()); String userName = getUserName(facadeUser); diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index c9ff38195..a0a46b0ec 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -13,6 +13,7 @@ com.tencent.supersonic.chat.core.parser.SemanticParser=\ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\ com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \ + com.tencent.supersonic.chat.core.corrector.TimeCorrector, \ com.tencent.supersonic.chat.core.corrector.SelectCorrector, \ com.tencent.supersonic.chat.core.corrector.WhereCorrector, \ com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \ diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 80b5b0431..c9f4240c3 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -13,6 +13,7 @@ com.tencent.supersonic.chat.core.parser.SemanticParser=\ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\ com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \ + com.tencent.supersonic.chat.core.corrector.TimeCorrector, \ com.tencent.supersonic.chat.core.corrector.SelectCorrector, \ com.tencent.supersonic.chat.core.corrector.WhereCorrector, \ com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \