topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:00:23 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(project) The Corrector is integrated into the Semantic module as an intermediate step for translating S2QL into SQL (#306)

Browse Source
This commit is contained in:
lexluo09
2023-11-01 22:34:55 +08:00
committed by GitHub
parent 372e4acc2c
commit b9f5e0a354
41 changed files with 483 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilter.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.google.common.base.Objects;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import lombok.Data;
import lombok.ToString;

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java
View File

@@ -25,7 +25,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMResponseService.java
View File

@@ -22,7 +22,7 @@ import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/PluginParser.java
View File

@@ -19,7 +19,7 @@ import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import org.springframework.util.CollectionUtils;
import java.util.HashMap;
import java.util.List;

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java
View File

@@ -23,7 +23,7 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;

4
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.chat.query.rule.metric;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.common.pojo.enums.FilterType;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import lombok.extern.slf4j.Slf4j;

4
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java
View File

@@ -7,8 +7,8 @@ import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNum
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.common.pojo.enums.FilterType;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.ArrayList;

2
chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java
View File

@@ -41,7 +41,7 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.text.DecimalFormat;
import java.time.DayOfWeek;

2
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -53,7 +53,7 @@ import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import com.tencent.supersonic.knowledge.utils.NatureHelper;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;

4
chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java
View File

@@ -6,8 +6,8 @@ import com.tencent.supersonic.chat.config.DefaultMetric;
import com.tencent.supersonic.chat.config.Dim4Dict;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;

116
chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java
View File

@@ -7,18 +7,20 @@ import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
@@ -29,7 +31,9 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
@@ -40,6 +44,7 @@ import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.SelectExpressionItem;
import net.sf.jsqlparser.statement.select.SelectItem;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
@@ -159,50 +164,93 @@ public class QueryReqBuilder {
* @param queryStructReq
* @return
*/
public static QueryS2QLReq buildS2QLReq(QueryStructReq queryStructReq) {
public static QueryS2QLReq buildS2QLReq(QueryStructReq queryStructReq) throws JSQLParserException {
Select select = new Select();
// Set the select items (columns)
//1.Set the select items (columns)
PlainSelect plainSelect = new PlainSelect();
List<SelectItem> selectItems = new ArrayList<>();
selectItems.add(new SelectExpressionItem(new Column("column1")));
selectItems.add(new SelectExpressionItem(new Column("column2")));
List<String> groups = queryStructReq.getGroups();
if (!CollectionUtils.isEmpty(groups)) {
for (String group : groups) {
selectItems.add(new SelectExpressionItem(new Column(group)));
}
}
List<Aggregator> aggregators = queryStructReq.getAggregators();
if (!CollectionUtils.isEmpty(aggregators)) {
for (Aggregator aggregator : aggregators) {
if (queryStructReq.getNativeQuery()) {
selectItems.add(new SelectExpressionItem(new Column(aggregator.getColumn())));
} else {
Function sumFunction = new Function();
AggOperatorEnum func = aggregator.getFunc();
if (AggOperatorEnum.UNKNOWN.equals(func)) {
func = AggOperatorEnum.SUM;
}
sumFunction.setName(func.getOperator());
sumFunction.setParameters(new ExpressionList(new Column(aggregator.getColumn())));
selectItems.add(new SelectExpressionItem(sumFunction));
}
}
}
plainSelect.setSelectItems(selectItems);
// Set the table name
Table table = new Table("table1");
//2.Set the table name
Table table = new Table(Constants.TABLE_PREFIX + queryStructReq.getModelId());
plainSelect.setFromItem(table);
// Set the order by clause
OrderByElement orderByElement = new OrderByElement();
orderByElement.setExpression(new Column("column1"));
plainSelect.setOrderByElements(Collections.singletonList(orderByElement));
// Set the group by clause
GroupByElement groupByElement = new GroupByElement();
groupByElement.addGroupByExpression(new Column("column1"));
plainSelect.setGroupByElement(groupByElement);
// Set the having clause
Expression havingExpression = null;
try {
havingExpression = CCJSqlParserUtil.parseCondExpression("condition2");
} catch (JSQLParserException e) {
log.error("");
//3.Set the order by clause
List<Order> orders = queryStructReq.getOrders();
if (!CollectionUtils.isEmpty(orders)) {
List<OrderByElement> orderByElements = new ArrayList<>();
for (Order order : orders) {
OrderByElement orderByElement = new OrderByElement();
orderByElement.setExpression(new Column(order.getColumn()));
orderByElement.setAsc(false);
if (Constants.ASC_UPPER.equalsIgnoreCase(order.getDirection())) {
orderByElement.setAsc(true);
}
orderByElements.add(orderByElement);
}
plainSelect.setOrderByElements(orderByElements);
}
plainSelect.setHaving(havingExpression);
// Set the limit clause
Limit limit = new Limit();
limit.setRowCount(new LongValue(10));
plainSelect.setLimit(limit);
//4.Set the group by clause
if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getNativeQuery()) {
GroupByElement groupByElement = new GroupByElement();
for (String group : groups) {
groupByElement.addGroupByExpression(new Column(group));
}
plainSelect.setGroupByElement(groupByElement);
}
//7.Set the limit clause
if (Objects.nonNull(queryStructReq.getLimit())) {
Limit limit = new Limit();
limit.setRowCount(new LongValue(queryStructReq.getLimit()));
plainSelect.setLimit(limit);
}
select.setSelectBody(plainSelect);
//5.Set where
List<Filter> dimensionFilters = queryStructReq.getDimensionFilters();
SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);
String whereClause = sqlFilterUtils.getWhereClause(dimensionFilters);
String sql = select.toString();
if (StringUtils.isNotBlank(whereClause)) {
Expression expression = CCJSqlParserUtil.parseCondExpression(whereClause);
sql = SqlParserAddHelper.addWhere(sql, expression);
}
//6.Set DateInfo
DateModeUtils dateModeUtils = ContextUtils.getBean(DateModeUtils.class);
String dateWhereStr = dateModeUtils.getDateWhereStr(queryStructReq.getDateInfo());
if (StringUtils.isNotBlank(dateWhereStr)) {
Expression expression = CCJSqlParserUtil.parseCondExpression(dateWhereStr);
sql = SqlParserAddHelper.addWhere(sql, expression);
}
QueryS2QLReq result = new QueryS2QLReq();
result.setSql(select.toString());
result.setSql(sql);
result.setModelId(queryStructReq.getModelId());
result.setVariables(new HashMap<>());
return result;

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java → chat/core/src/main/java/com/tencent/supersonic/chat/utils/SqlFilterUtils.java
View File

@@ -1,14 +1,14 @@
package com.tencent.supersonic.semantic.query.utils;
package com.tencent.supersonic.chat.utils;
import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_END;
import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Criterion;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Criterion;
import com.tencent.supersonic.common.pojo.Filter;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

2
chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java
View File

@@ -4,7 +4,7 @@ import com.google.gson.Gson;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.DateConf;

61
chat/core/src/test/java/com/tencent/supersonic/chat/utils/QueryReqBuilderTest.java
View File

@@ -1,7 +1,23 @@
package com.tencent.supersonic.chat.utils;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import net.sf.jsqlparser.JSQLParserException;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
/**
* QueryReqBuilderTest
@@ -9,9 +25,52 @@ import org.junit.jupiter.api.Test;
class QueryReqBuilderTest {
@Test
void buildS2QLReq() {
void buildS2QLReq() throws JSQLParserException {
init();
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setModelId(1L);
queryStructReq.setNativeQuery(false);
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.UNKNOWN);
aggregator.setColumn("pv");
queryStructReq.setAggregators(Arrays.asList(aggregator));
queryStructReq.setGroups(Arrays.asList("department"));
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateMode.LIST);
dateConf.setDateList(Arrays.asList("2023-08-01"));
queryStructReq.setDateInfo(dateConf);
List<Order> orders = new ArrayList<>();
Order order = new Order();
order.setColumn("uv");
orders.add(order);
queryStructReq.setOrders(orders);
QueryS2QLReq queryS2QLReq = QueryReqBuilder.buildS2QLReq(queryStructReq);
Assert.assertEquals(
"SELECT department, SUM(pv) FROM t_1 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "GROUP BY department ORDER BY uv LIMIT 2000", queryS2QLReq.getSql());
queryStructReq.setNativeQuery(true);
queryS2QLReq = QueryReqBuilder.buildS2QLReq(queryStructReq);
Assert.assertEquals(
"SELECT department, pv FROM t_1 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "ORDER BY uv LIMIT 2000",
queryS2QLReq.getSql());
}
private void init() {
MockedStatic<ContextUtils> mockContextUtils = Mockito.mockStatic(ContextUtils.class);
SqlFilterUtils sqlFilterUtils = new SqlFilterUtils();
mockContextUtils.when(() -> ContextUtils.getBean(SqlFilterUtils.class)).thenReturn(sqlFilterUtils);
DateModeUtils dateModeUtils = new DateModeUtils();
mockContextUtils.when(() -> ContextUtils.getBean(DateModeUtils.class)).thenReturn(dateModeUtils);
dateModeUtils.setSysDateCol("sys_imp_date");
dateModeUtils.setSysDateWeekCol("sys_imp_week");
dateModeUtils.setSysDateMonthCol("sys_imp_month");
}
}

2
common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java
View File

@@ -64,4 +64,6 @@ public class Constants {
public static final Long DEFAULT_FREQUENCY = 100000L;
public static final String TABLE_PREFIX = "t_";
}

5
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/Criterion.java → common/src/main/java/com/tencent/supersonic/common/pojo/Criterion.java
View File

@@ -1,10 +1,9 @@
package com.tencent.supersonic.semantic.api.query.pojo;
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import java.util.Arrays;
import java.util.List;
import lombok.Data;
@Data

5
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/pojo/Filter.java → common/src/main/java/com/tencent/supersonic/common/pojo/Filter.java
View File

@@ -1,9 +1,8 @@
package com.tencent.supersonic.semantic.api.query.pojo;
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

2
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ItemDateResp.java → common/src/main/java/com/tencent/supersonic/common/pojo/ItemDateResp.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.api.model.response;
package com.tencent.supersonic.common.pojo;
import java.util.ArrayList;
import java.util.List;

2
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/enums/FilterOperatorEnum.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterOperatorEnum.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.api.query.enums;
package com.tencent.supersonic.common.pojo.enums;
import com.fasterxml.jackson.annotation.JsonCreator;

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DateUtils.java → common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.utils;
package com.tencent.supersonic.common.util;
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
@@ -10,7 +10,7 @@ import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
@@ -21,6 +21,7 @@ import java.util.List;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.regex.Pattern;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.springframework.beans.factory.annotation.Value;
@@ -30,7 +31,8 @@ import org.springframework.util.CollectionUtils;
@Slf4j
@Component
public class DateUtils {
@Data
public class DateModeUtils {
@Value("${query.parameter.sys.date:sys_imp_date}")
private String sysDateCol;
@@ -227,6 +229,9 @@ public class DateUtils {
public String recentDateStr(ItemDateResp dateDate, DateConf dateInfo) {
if (Objects.isNull(dateDate)) {
return "";
}
if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) {
return recentDayStr(dateDate, dateInfo);
}
@@ -315,6 +320,11 @@ public class DateUtils {
LocalDate.now().minusDays(1));
}
public String getDateWhereStr(DateConf dateInfo) {
ItemDateResp dateDate = null;
return getDateWhereStr(dateInfo, dateDate);
}
public String getDateWhereStr(DateConf dateInfo, ItemDateResp dateDate) {
String dateStr = "";
switch (dateInfo.getDateMode()) {

245
common/src/main/java/com/tencent/supersonic/common/util/SqlFilterUtils.java Normal file
View File

@@ -0,0 +1,245 @@
package com.tencent.supersonic.common.util;
import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_END;
import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.Criterion;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
public class SqlFilterUtils {
private static String pattern = "^'.*?'$";
private static String numericPattern = "^[0-9]+$";
public List<String> getFiltersCol(List<Filter> filters) {
List<String> filterCols = new ArrayList<>();
if (CollectionUtils.isEmpty(filters)) {
return filterCols;
}
for (Filter filter : filters) {
filterCols.addAll(getFilterCol(filter));
}
return filterCols;
}
private List<String> getFilterCol(Filter filter) {
List<String> filterCols = new ArrayList<>();
if (Filter.Relation.FILTER.equals(filter.getRelation())) {
if (Strings.isNotEmpty(filter.getBizName())) {
filterCols.add(filter.getBizName());
}
}
List<Filter> children = filter.getChildren();
if (!CollectionUtils.isEmpty(children)) {
for (Filter child : children) {
filterCols.addAll(getFilterCol(child));
}
}
return filterCols;
}
public String getWhereClause(List<Filter> filters) {
StringJoiner joiner = new StringJoiner(Constants.AND_UPPER);
if (!CollectionUtils.isEmpty(filters)) {
filters.stream()
.forEach(filter -> {
if (Strings.isNotEmpty(dealFilter(filter))) {
joiner.add(SPACE + dealFilter(filter) + SPACE);
}
});
log.info("getWhereClause, where sql : {}", joiner.toString());
return joiner.toString();
}
return "";
}
public String dealFilter(Filter filter) {
if (Objects.isNull(filter)) {
return "";
}
if (Strings.isNotEmpty(filter.getBizName()) && filter.getBizName().endsWith(SYS_VAR)) {
return "";
}
StringBuilder condition = new StringBuilder();
if (Filter.Relation.FILTER.equals(filter.getRelation())) {
return dealSingleFilter(filter);
}
List<Filter> children = filter.getChildren();
condition.append(PARENTHESES_START);
StringJoiner joiner = new StringJoiner(SPACE + filter.getRelation().name() + SPACE);
for (Filter child : children) {
joiner.add(dealFilter(child));
}
condition.append(joiner.toString());
condition.append(PARENTHESES_END);
return condition.toString();
}
// todo deal metric filter
private String dealSingleFilter(Filter filter) {
String name = filter.getBizName();
Object value = filter.getValue();
FilterOperatorEnum operator = filter.getOperator();
String dataType = Criterion.StringDataType.STRING.name();
Criterion criterion = new Criterion(name, operator, value, dataType);
return generator(criterion);
}
private String generator(Criterion criterion) {
log.info("criterion :{}", criterion);
String sqlPart;
switch (criterion.getOperator()) {
case SQL_PART:
sqlPart = sqlPartLogic(criterion);
break;
case IS_NULL:
case IS_NOT_NULL:
sqlPart = judgeNullLogic(criterion);
break;
case EQUALS:
case NOT_EQUALS:
case GREATER_THAN:
case GREATER_THAN_EQUALS:
case MINOR_THAN:
case MINOR_THAN_EQUALS:
sqlPart = singleValueLogic(criterion);
break;
case BETWEEN:
sqlPart = betweenLogic(criterion);
break;
case IN:
case NOT_IN:
sqlPart = inLogic(criterion);
break;
case LIKE:
sqlPart = likeLogic(criterion);
break;
default:
throw new IllegalStateException("Unexpected value: " + criterion.getOperator());
}
return sqlPart;
}
private String likeLogic(Criterion criterion) {
if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
throw new RuntimeException("criterion.getValue() can not be null");
}
StringBuilder whereClause = new StringBuilder();
whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE);
String value = criterion.getValue().toString();
if (criterion.isNeedApostrophe() && !Pattern.matches(pattern, value)) {
// like click => 'like%'
whereClause.append(Constants.APOSTROPHE + value + Constants.PERCENT_SIGN + Constants.APOSTROPHE);
} else {
// like 'click' => 'like%'
whereClause.append(Constants.APOSTROPHE + value.replaceAll(Constants.APOSTROPHE, Constants.PERCENT_SIGN)
+ Constants.APOSTROPHE);
}
return whereClause.toString();
}
private String inLogic(Criterion criterion) {
if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
throw new RuntimeException("criterion.getValue() can not be null");
}
StringBuilder whereClause = new StringBuilder();
whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE);
List values = (List) criterion.getValue();
whereClause.append(PARENTHESES_START);
StringJoiner joiner = new StringJoiner(",");
if (criterion.isNeedApostrophe()) {
values.stream().forEach(value -> joiner.add(valueApostropheLogic(value.toString())));
} else {
values.stream().forEach(value -> joiner.add(value.toString()));
}
whereClause.append(joiner);
whereClause.append(PARENTHESES_END);
return whereClause.toString();
}
private String betweenLogic(Criterion criterion) {
if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
throw new RuntimeException("criterion.getValue() can not be null");
}
List values = (List) criterion.getValue();
if (values.size() != 2) {
throw new RuntimeException("between value size should be 2");
}
if (criterion.isNeedApostrophe()) {
return String.format("(%s >= %s and %s <= %s)", criterion.getColumn(),
valueApostropheLogic(values.get(0).toString()),
criterion.getColumn(), valueApostropheLogic(values.get(1).toString()));
}
return String.format("(%s >= %s and %s <= %s)", criterion.getColumn(), values.get(0).toString(),
criterion.getColumn(), values.get(1).toString());
}
private String singleValueLogic(Criterion criterion) {
if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
throw new RuntimeException("criterion.getValue() can not be null");
}
StringBuilder whereClause = new StringBuilder();
whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE);
String value = criterion.getValue().toString();
if (criterion.isNeedApostrophe()) {
value = valueApostropheLogic(value);
}
whereClause.append(value);
return whereClause.toString();
}
private String valueApostropheLogic(String value) {
if (Pattern.matches(pattern, value) || Pattern.matches(numericPattern, value)) {
return value;
}
return Constants.APOSTROPHE + value + Constants.APOSTROPHE;
}
private String judgeNullLogic(Criterion criterion) {
if (Objects.isNull(criterion) || Objects.isNull(criterion.getColumn())) {
throw new RuntimeException("criterion.getColumn() can not be null");
}
return String.format("( %s %s)", criterion.getColumn(), criterion.getOperator().getValue());
}
private String sqlPartLogic(Criterion criterion) {
if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
throw new RuntimeException("criterion.getValue() can not be null");
}
return PARENTHESES_START + SPACE + criterion.getValue().toString() + SPACE + PARENTHESES_END;
}
}

2
launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.rule.entity.EntityFilterQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import org.junit.Test;
import java.util.ArrayList;

2
launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java
View File

@@ -13,7 +13,7 @@ import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricTopNQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import org.junit.Assert;
import org.junit.Test;

2
launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import org.junit.Test;
import org.junit.jupiter.api.Order;

2
launchers/standalone/src/test/java/com/tencent/supersonic/integration/mapper/MapperTest.java
View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.integration.BaseQueryTest;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import org.junit.Test;

2
launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.llm.interpret.MetricOption;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import java.util.Set;

2
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.api.query.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.pojo.Param;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java
View File

@@ -4,7 +4,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java
View File

@@ -22,7 +22,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceRelaResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.model.domain.DatabaseService;

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java
View File

@@ -4,7 +4,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java
View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.semantic.api.model.request.DatasourceRelaReq;
import com.tencent.supersonic.semantic.api.model.request.DatasourceReq;
import com.tencent.supersonic.semantic.api.model.response.DatasourceRelaResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import java.util.List;
import java.util.Map;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java
View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.utils.DateUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import com.tencent.supersonic.semantic.query.utils.SqlGenerateUtils;
import java.util.ArrayList;
@@ -410,8 +410,8 @@ public class CalculateAggConverter implements SemanticConverter {
}
private static String getTimeDim(QueryStructReq queryStructCmd) {
DateUtils dateUtils = ContextUtils.getContext().getBean(DateUtils.class);
return dateUtils.getSysDateCol(queryStructCmd.getDateInfo());
DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class);
return dateModeUtils.getSysDateCol(queryStructCmd.getDateInfo());
}
private static String getLimit(QueryStructReq queryStructCmd) {

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;

7
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -46,7 +47,6 @@ import org.springframework.util.CollectionUtils;
@Slf4j
public class QueryReqConverter {
public static final String TABLE_PREFIX = "t_";
@Autowired
private ModelService domainService;
@Autowired
@@ -135,7 +135,7 @@ public class QueryReqConverter {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, false);
String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
log.info("convert name to bizName after:{}", replaceFields);
databaseReq.setSql(replaceFields);
}
@@ -202,7 +202,8 @@ public class QueryReqConverter {
}
public void correctTableName(QueryS2QLReq databaseReq) {
String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(), TABLE_PREFIX + databaseReq.getModelId());
String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(),
Constants.TABLE_PREFIX + databaseReq.getModelId());
databaseReq.setSql(sql);
}

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -17,9 +17,9 @@ import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -20,8 +20,8 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -8,8 +8,8 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;

56
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -11,6 +11,8 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
@@ -21,7 +23,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
@@ -67,7 +69,7 @@ public class QueryStructUtils {
internalCols.addAll(internalTimeCols);
}
private final DateUtils dateUtils;
private final DateModeUtils dateModeUtils;
private final SqlFilterUtils sqlFilterUtils;
private final Catalog catalog;
@Value("${internal.metric.cnt.suffix:internal_cnt}")
@@ -82,10 +84,10 @@ public class QueryStructUtils {
private String variablePrefix = "'${";
public QueryStructUtils(
DateUtils dateUtils,
DateModeUtils dateModeUtils,
SqlFilterUtils sqlFilterUtils, Catalog catalog) {
this.dateUtils = dateUtils;
this.dateModeUtils = dateModeUtils;
this.sqlFilterUtils = sqlFilterUtils;
this.catalog = catalog;
}
@@ -145,19 +147,19 @@ public class QueryStructUtils {
|| Strings.isEmpty(dateDate.getStartDate())
&& Strings.isEmpty(dateDate.getEndDate())) {
if (dateInfo.getDateMode().equals(DateMode.LIST)) {
return dateUtils.listDateStr(dateDate, dateInfo);
return dateModeUtils.listDateStr(dateDate, dateInfo);
}
if (dateInfo.getDateMode().equals(DateMode.BETWEEN)) {
return dateUtils.betweenDateStr(dateDate, dateInfo);
return dateModeUtils.betweenDateStr(dateDate, dateInfo);
}
if (dateUtils.hasAvailableDataMode(dateInfo)) {
return dateUtils.hasDataModeStr(dateDate, dateInfo);
if (dateModeUtils.hasAvailableDataMode(dateInfo)) {
return dateModeUtils.hasDataModeStr(dateDate, dateInfo);
}
return dateUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
}
log.info("dateDate:{}", dateDate);
return dateUtils.getDateWhereStr(dateInfo, dateDate);
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
}
@@ -177,7 +179,7 @@ public class QueryStructUtils {
return whereFromDate;
} else if (Strings.isEmpty(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
log.info("the current date information is empty, enter the date initialization logic");
return dateUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
}
return whereClauseFromFilter;
}
@@ -283,19 +285,19 @@ public class QueryStructUtils {
public String generateZipperWhere(QueryStatement queryStatement, QueryStructReq queryStructReq) {
if (Objects.nonNull(queryStatement.getParseSqlReq().getSql())) {
String sql = SqlParserRemoveHelper.removeWhere(queryStatement.getParseSqlReq().getSql(),
dateUtils.getDateCol());
dateModeUtils.getDateCol());
if (!CollectionUtils.isEmpty(queryStatement.getMetricReq().getDimensions())) {
List<String> dimension = queryStatement.getMetricReq().getDimensions().stream()
.filter(d -> !dateUtils.getDateCol().contains(d.toLowerCase())).collect(
.filter(d -> !dateModeUtils.getDateCol().contains(d.toLowerCase())).collect(
Collectors.toList());
dimension.add(dateUtils.getDateColBegin(queryStructReq.getDateInfo()));
dimension.add(dateUtils.getDateColEnd(queryStructReq.getDateInfo()));
dimension.add(dateModeUtils.getDateColBegin(queryStructReq.getDateInfo()));
dimension.add(dateModeUtils.getDateColEnd(queryStructReq.getDateInfo()));
queryStatement.getMetricReq().setDimensions(dimension);
}
return SqlParserAddHelper.addWhere(sql,
SqlParserSelectHelper.getTimeFilter(queryStatement.getTimeRanges(),
dateUtils.getDateColBegin(queryStructReq.getDateInfo()),
dateUtils.getDateColEnd(queryStructReq.getDateInfo())));
dateModeUtils.getDateColBegin(queryStructReq.getDateInfo()),
dateModeUtils.getDateColEnd(queryStructReq.getDateInfo())));
}
return queryStatement.getSql();
}
@@ -306,7 +308,7 @@ public class QueryStructUtils {
List<String> wheres = new ArrayList<>();
if (!CollectionUtils.isEmpty(timeRanges)) {
for (ImmutablePair<String, String> range : timeRanges) {
String strWhere = dateUtils.getDateWhereStr(queryStructCmd.getDateInfo(), range);
String strWhere = dateModeUtils.getDateWhereStr(queryStructCmd.getDateInfo(), range);
if (!strWhere.isEmpty()) {
wheres.add(strWhere);
}
@@ -337,11 +339,11 @@ public class QueryStructUtils {
}
switch (dateConf.getPeriod()) {
case DAY:
return dateUtils.recentDay(dateDate, dateConf);
return dateModeUtils.recentDay(dateDate, dateConf);
case WEEK:
return dateUtils.recentWeek(dateDate, dateConf);
return dateModeUtils.recentWeek(dateDate, dateConf);
case MONTH:
List<ImmutablePair<String, String>> rets = dateUtils.recentMonth(dateDate, dateConf);
List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dateDate, dateConf);
Optional<String> minBegins = rets.stream().map(i -> i.left).sorted().findFirst();
Optional<String> maxBegins = rets.stream().map(i -> i.right).sorted(Comparator.reverseOrder())
.findFirst();
@@ -387,13 +389,13 @@ public class QueryStructUtils {
}
switch (dateConf.getPeriod()) {
case DAY:
ret.add(dateUtils.recentDay(dateDate, dateConf));
ret.add(dateModeUtils.recentDay(dateDate, dateConf));
break;
case WEEK:
ret.add(dateUtils.recentWeek(dateDate, dateConf));
ret.add(dateModeUtils.recentWeek(dateDate, dateConf));
break;
case MONTH:
List<ImmutablePair<String, String>> rets = dateUtils.recentMonth(dateDate, dateConf);
List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dateDate, dateConf);
ret.addAll(rets);
break;
default:
@@ -426,10 +428,10 @@ public class QueryStructUtils {
if (Objects.isNull(f.getFieldName()) || !internalCols.contains(f.getFieldName().toLowerCase())) {
continue;
}
if (Objects.isNull(f.getFieldValue()) || !dateUtils.isDateStr(f.getFieldValue().toString())) {
if (Objects.isNull(f.getFieldValue()) || !dateModeUtils.isDateStr(f.getFieldValue().toString())) {
continue;
}
period = dateUtils.getPeriodByCol(f.getFieldName().toLowerCase());
period = dateModeUtils.getPeriodByCol(f.getFieldName().toLowerCase());
if ("".equals(period)) {
continue;
}
@@ -465,7 +467,7 @@ public class QueryStructUtils {
}
public List<String> getDateCol() {
return dateUtils.getDateCol();
return dateModeUtils.getDateCol();
}
public String getVariablePrefix() {

1
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java
View File

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeBackEnum;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;