topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Rules and large model queries should be queried using s2sql (#334)

Browse Source
This commit is contained in:
lexluo09
2023-11-07 16:23:31 +08:00
committed by GitHub
parent aa6c658a9a
commit 0365886270
37 changed files with 340 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/enums/TimeDimensionEnum.java
View File

@@ -1,28 +0,0 @@
package com.tencent.supersonic.semantic.api.model.enums;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
public enum TimeDimensionEnum {
DAY("sys_imp_date"),
WEEK("sys_imp_week"),
MONTH("sys_imp_month");
private String name;
TimeDimensionEnum(String name) {
this.name = name;
}
public static List<String> getNameList() {
return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getName).collect(Collectors.toList());
}
public String getName() {
return name;
}
}

31
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java
View File

@@ -1,25 +1,23 @@
package com.tencent.supersonic.semantic.api.query.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.pojo.Param;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
@@ -49,6 +47,7 @@ public class QueryStructReq {
private Long modelId;
private String modelName;
private List<String> groups = new ArrayList<>();
private List<Aggregator> aggregators = new ArrayList<>();
private List<Order> orders = new ArrayList<>();
@@ -62,6 +61,15 @@ public class QueryStructReq {
private boolean useS2qlSwitch;
/**
* Later deleted for compatibility only
*/
private String s2QL;
/**
* Later deleted for compatibility only
*/
private String logicSql;
public List<String> getGroups() {
if (!CollectionUtils.isEmpty(this.groups)) {
this.groups = groups.stream().filter(group -> !Strings.isEmpty(group)).collect(Collectors.toList());
@@ -203,6 +211,10 @@ public class QueryStructReq {
func = AggOperatorEnum.SUM;
}
sumFunction.setName(func.getOperator());
if (AggOperatorEnum.COUNT_DISTINCT.equals(func)) {
sumFunction.setName("count");
sumFunction.setDistinct(true);
}
sumFunction.setParameters(new ExpressionList(new Column(aggregator.getColumn())));
selectItems.add(new SelectExpressionItem(sumFunction));
}
@@ -210,7 +222,7 @@ public class QueryStructReq {
}
plainSelect.setSelectItems(selectItems);
//2.Set the table name
Table table = new Table(Constants.TABLE_PREFIX + queryStructReq.getModelId());
Table table = new Table(queryStructReq.getModelName());
plainSelect.setFromItem(table);
//3.Set the order by clause
@@ -218,6 +230,9 @@ public class QueryStructReq {
if (!CollectionUtils.isEmpty(orders)) {
List<OrderByElement> orderByElements = new ArrayList<>();
for (Order order : orders) {
if (StringUtils.isBlank(order.getColumn())) {
continue;
}
OrderByElement orderByElement = new OrderByElement();
orderByElement.setExpression(new Column(order.getColumn()));
orderByElement.setAsc(false);
@@ -249,7 +264,7 @@ public class QueryStructReq {
//6.Set where
List<Filter> dimensionFilters = queryStructReq.getDimensionFilters();
SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);
String whereClause = sqlFilterUtils.getWhereClause(dimensionFilters);
String whereClause = sqlFilterUtils.getWhereClause(dimensionFilters, false);
String sql = select.toString();
if (StringUtils.isNotBlank(whereClause)) {

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/ClickHouseAdaptor.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.HashMap;
import java.util.Map;

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/H2Adaptor.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
public class H2Adaptor extends EngineAdaptor {

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/adaptor/engineadapter/MysqlAdaptor.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.Constants;

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/SysTimeDimensionBuilder.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.model.domain.utils;
import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptor;

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -4,11 +4,10 @@ package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
@@ -185,7 +184,14 @@ public class QueryReqConverter {
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1));
dimensionResults.put(DateUtils.DATE_FIELD, TimeDimensionEnum.DAY.getName());
dimensionResults.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getName());
dimensionResults.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getName());
dimensionResults.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getName());
dimensionResults.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getName());
dimensionResults.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getName());
dimensionResults.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getName());
dimensionResults.putAll(metricResults);
return dimensionResults;
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlGenerateUtils.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;