topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][Chat] Add TimeCorrector and rename the associated SqlParserHelper. (#707)

Browse Source
This commit is contained in:
lexluo09
2024-02-01 15:29:07 +08:00
committed by GitHub
parent 2c1c443b3e
commit 491c76368c
43 changed files with 749 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
@@ -156,7 +156,7 @@ public class QueryStructReq extends SemanticQueryReq {
}
/**
* convert queryStructReq to QueryS2QLReq
* convert queryStructReq to QueryS2SQLReq
*
* @param queryStructReq
* @return
@@ -261,7 +261,7 @@ public class QueryStructReq extends SemanticQueryReq {
String sql = select.toString();
if (StringUtils.isNotBlank(whereClause)) {
Expression expression = CCJSqlParserUtil.parseCondExpression(whereClause);
sql = SqlParserAddHelper.addWhere(sql, expression);
sql = SqlAddHelper.addWhere(sql, expression);
}
//7.Set DateInfo
@@ -269,7 +269,7 @@ public class QueryStructReq extends SemanticQueryReq {
String dateWhereStr = dateModeUtils.getDateWhereStr(queryStructReq.getDateInfo());
if (StringUtils.isNotBlank(dateWhereStr)) {
Expression expression = CCJSqlParserUtil.parseCondExpression(dateWhereStr);
sql = SqlParserAddHelper.addWhere(sql, expression);
sql = SqlAddHelper.addWhere(sql, expression);
}
return sql;
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.HashMap;
@@ -49,7 +49,7 @@ public class ClickHouseAdaptor extends DbAdaptor {
functionMap.put("MONTH".toLowerCase(), "toMonth");
functionMap.put("DAY".toLowerCase(), "toDayOfMonth");
functionMap.put("YEAR".toLowerCase(), "toYear");
return SqlParserReplaceHelper.replaceFunction(sql, functionMap);
return SqlReplaceHelper.replaceFunction(sql, functionMap);
}
@Override

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PostgresqlAdaptor.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
import java.util.HashMap;
import java.util.Map;
@@ -47,7 +47,7 @@ public class PostgresqlAdaptor extends DbAdaptor {
functionMap.put("MONTH".toLowerCase(), "toMonth");
functionMap.put("DAY".toLowerCase(), "toDayOfMonth");
functionMap.put("YEAR".toLowerCase(), "toYear");
return SqlParserReplaceHelper.replaceFunction(sql, functionMap);
return SqlReplaceHelper.replaceFunction(sql, functionMap);
}
@Override

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -8,8 +8,8 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
@@ -279,7 +279,7 @@ public class SqlGenerateUtils {
Set<String> visitedMetric,
Set<String> measures,
Set<String> dimensions) {
Set<String> fields = SqlParserSelectHelper.getColumnFromExpr(expression);
Set<String> fields = SqlSelectHelper.getColumnFromExpr(expression);
if (!CollectionUtils.isEmpty(fields)) {
Map<String, String> replace = new HashMap<>();
for (String field : fields) {
@@ -321,7 +321,7 @@ public class SqlGenerateUtils {
}
}
if (!CollectionUtils.isEmpty(replace)) {
String expr = SqlParserReplaceHelper.replaceExpression(expression, replace);
String expr = SqlReplaceHelper.replaceExpression(expression, replace);
log.info("derived measure {}->{}", expression, expr);
return expr;
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -7,8 +7,8 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
@@ -90,7 +90,7 @@ public class DimValueAspect {
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySqlReq.getModelIds()));
String sql = querySqlReq.getSql();
log.info("correctorSql before replacing:{}", sql);
List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<FieldExpression> fieldExpressionList = SqlSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Set<String> fieldNames = dimensions.stream().map(SchemaItem::getName).collect(Collectors.toSet());
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
@@ -117,7 +117,7 @@ public class DimValueAspect {
replaceInCondition(expression, dimension, filedNameToValueMap);
}
}
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
sql = SqlReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql);
querySqlReq.setSql(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
@@ -283,7 +283,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(querySqlReq.getSql(), expression);
String sql = SqlAddHelper.addWhere(querySqlReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql());
querySqlReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql());

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
@@ -63,7 +63,7 @@ public class MetricCheckUtils {
private static boolean hasAggregateFunction(String expr) {
String sql = String.format("select %s from table", expr);
return SqlParserSelectFunctionHelper.hasAggregateFunction(sql);
return SqlSelectFunctionHelper.hasAggregateFunction(sql);
}
}

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -6,9 +6,9 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
@@ -72,12 +72,12 @@ public class QueryReqConverter {
//3.correct tableName
correctTableName(querySQLReq);
String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql());
String tableName = SqlSelectHelper.getTableName(querySQLReq.getSql());
if (StringUtils.isEmpty(tableName)) {
return new QueryStatement();
}
//4.build MetricTables
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql());
List<String> allFields = SqlSelectHelper.getAllFields(querySQLReq.getSql());
List<String> metrics = getMetrics(semanticSchemaResp, allFields);
QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable();
@@ -135,9 +135,9 @@ public class QueryReqConverter {
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql();
if (!SqlParserSelectHelper.hasGroupBy(sql)
|| SqlParserSelectFunctionHelper.hasFunction(sql, "count")
|| SqlParserSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
if (!SqlSelectHelper.hasGroupBy(sql)
|| SqlSelectFunctionHelper.hasFunction(sql, "count")
|| SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
return AggOption.NATIVE;
}
return AggOption.DEFAULT;
@@ -147,7 +147,7 @@ public class QueryReqConverter {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
String replaceFields = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
log.info("convert name to bizName after:{}", replaceFields);
databaseReq.setSql(replaceFields);
}
@@ -214,7 +214,7 @@ public class QueryReqConverter {
public void correctTableName(QuerySqlReq querySqlReq) {
String sql = querySqlReq.getSql();
sql = SqlParserReplaceHelper.replaceTable(sql,
sql = SqlReplaceHelper.replaceTable(sql,
Constants.TABLE_PREFIX + querySqlReq.getViewId());
querySqlReq.setSql(sql);
}
@@ -239,7 +239,7 @@ public class QueryReqConverter {
measures, replaces);
if (!CollectionUtils.isEmpty(replaces)) {
// metricTable sql use measures replace metric
sql = SqlParserReplaceHelper.replaceSqlByExpression(sql, replaces);
sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces);
metricTable.setAggOption(AggOption.NATIVE);
// metricTable use measures replace metric
if (!CollectionUtils.isEmpty(measures)) {

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
@@ -126,7 +126,7 @@ public class QueryStructUtils {
}
public Set<String> getResName(QuerySqlReq querySqlReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(querySqlReq.getSql())
Set<String> resNameSet = SqlSelectHelper.getAllFields(querySqlReq.getSql())
.stream().collect(Collectors.toSet());
return resNameSet;
}
@@ -170,7 +170,7 @@ public class QueryStructUtils {
public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySqlReq) {
String sql = querySqlReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
Set<String> resNameEnSet = SqlSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
@@ -235,7 +235,7 @@ public class QueryStructUtils {
}
public DateConf getDateConfBySql(String sql) {
List<FieldExpression> fieldExpressions = SqlParserSelectHelper.getFilterExpression(sql);
List<FieldExpression> fieldExpressions = SqlSelectHelper.getFilterExpression(sql);
if (!CollectionUtils.isEmpty(fieldExpressions)) {
Set<String> dateList = new HashSet<>();
String startDate = "";

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.QueryOptMode;
@@ -100,8 +100,8 @@ public class StatUtils {
public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
List<String> aggFields = SqlParserSelectHelper.getAggregateFields(querySqlReq.getSql());
List<String> allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql());
List<String> aggFields = SqlSelectHelper.getAggregateFields(querySqlReq.getSql());
List<String> allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql());
List<String> dimensions = allFields.stream().filter(aggFields::contains).collect(Collectors.toList());
String userName = getUserName(facadeUser);