From abc590d15a33794c398be3fb83951660a0d95aad Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:26:53 +0800 Subject: [PATCH] (improvement)(Headless) queryBySql supports Time Correct. (#915) --- .../core/chat/corrector/TimeCorrector.java | 34 +++++++++++++++++++ .../core/chat/corrector/WhereCorrector.java | 31 ----------------- .../rest/api/SqlQueryApiController.java | 2 ++ .../service/impl/DimensionServiceImpl.java | 2 +- 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/TimeCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/TimeCorrector.java index 83ad22ff6..eb0964e38 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/TimeCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/TimeCorrector.java @@ -1,17 +1,24 @@ package com.tencent.supersonic.headless.core.chat.corrector; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.jsqlparser.DateVisitor.DateBoundInfo; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlDateSelectHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.core.pojo.QueryContext; +import com.tencent.supersonic.headless.core.utils.S2SqlDateHelper; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.util.CollectionUtils; +import java.util.List; import java.util.Objects; /** @@ -23,12 +30,39 @@ public class TimeCorrector extends BaseSemanticCorrector { @Override public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { + addDateIfNotExist(queryContext, semanticParseInfo); + parserDateDiffFunction(semanticParseInfo); addLowerBoundDate(semanticParseInfo); } + private void addDateIfNotExist(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { + String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); + List whereFields = SqlSelectHelper.getWhereFields(correctS2SQL); + if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) { + + Pair startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, + semanticParseInfo.getDataSetId(), semanticParseInfo.getQueryType()); + + if (StringUtils.isNotBlank(startEndDate.getLeft()) + && StringUtils.isNotBlank(startEndDate.getRight())) { + correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL); + String dateChName = TimeDimensionEnum.DAY.getChName(); + String condExpr = String.format(" ( %s >= '%s' and %s <= '%s' )", dateChName, + startEndDate.getLeft(), dateChName, startEndDate.getRight()); + try { + Expression expression = CCJSqlParserUtil.parseCondExpression(condExpr); + correctS2SQL = SqlAddHelper.addWhere(correctS2SQL, expression); + } catch (JSQLParserException e) { + log.error("parseCondExpression:{}", e); + } + } + } + semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL); + } + private void addLowerBoundDate(SemanticParseInfo semanticParseInfo) { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(correctS2SQL); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/WhereCorrector.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/WhereCorrector.java index 48cb1110e..5e5a3d489 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/WhereCorrector.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/WhereCorrector.java @@ -2,24 +2,20 @@ package com.tencent.supersonic.headless.core.chat.corrector; import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.request.QueryFilters; import com.tencent.supersonic.headless.core.pojo.QueryContext; -import com.tencent.supersonic.headless.core.utils.S2SqlDateHelper; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; @@ -38,8 +34,6 @@ public class WhereCorrector extends BaseSemanticCorrector { @Override public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - addDateIfNotExist(queryContext, semanticParseInfo); - addQueryFilter(queryContext, semanticParseInfo); updateFieldValueByTechName(queryContext, semanticParseInfo); @@ -63,31 +57,6 @@ public class WhereCorrector extends BaseSemanticCorrector { } } - private void addDateIfNotExist(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - List whereFields = SqlSelectHelper.getWhereFields(correctS2SQL); - if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) { - - Pair startEndDate = S2SqlDateHelper.getStartEndDate(queryContext, - semanticParseInfo.getDataSetId(), semanticParseInfo.getQueryType()); - - if (StringUtils.isNotBlank(startEndDate.getLeft()) - && StringUtils.isNotBlank(startEndDate.getRight())) { - correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL); - String dateChName = TimeDimensionEnum.DAY.getChName(); - String condExpr = String.format(" ( %s >= '%s' and %s <= '%s' )", dateChName, - startEndDate.getLeft(), dateChName, startEndDate.getRight()); - try { - Expression expression = CCJSqlParserUtil.parseCondExpression(condExpr); - correctS2SQL = SqlAddHelper.addWhere(correctS2SQL, expression); - } catch (JSQLParserException e) { - log.error("parseCondExpression:{}", e); - } - } - } - semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL); - } - private String getQueryFilter(QueryFilters queryFilters) { if (Objects.isNull(queryFilters) || CollectionUtils.isEmpty(queryFilters.getFilters())) { return null; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java index c7f8a5918..d9f152ce8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.rest.api; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.SqlInfo; @@ -71,6 +72,7 @@ public class SqlQueryApiController { sqlInfo.setCorrectS2SQL(querySqlReq.getSql()); sqlInfo.setS2SQL(querySqlReq.getSql()); semanticParseInfo.setSqlInfo(sqlInfo); + semanticParseInfo.setQueryType(QueryType.TAG); ComponentFactory.getSemanticCorrectors().forEach(corrector -> { if (!(corrector instanceof GrammarCorrector)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 86332cfe6..1c3ac182c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -402,7 +402,7 @@ public class DimensionServiceImpl implements DimensionService { String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(dimensionReq.getName()); if (StringUtils.isNotBlank(forbiddenCharacters)) { throw new InvalidArgumentException(String.format("名称包含特殊字符, 请修改: %s，特殊字符: %s", - dimensionReq.getBizName(), forbiddenCharacters)); + dimensionReq.getName(), forbiddenCharacters)); } if (bizNameMap.containsKey(dimensionReq.getBizName())) { DimensionResp dimensionResp = bizNameMap.get(dimensionReq.getBizName());