topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project] supersonic 0.7.0 version backend update (#20)

Browse Source 
Co-authored-by: kanedai <kanedai@tencent.com>
This commit is contained in:
daikon
2023-07-31 11:09:58 +08:00
committed by GitHub
parent 078a81038f
commit e2b2d31429
675 changed files with 13089 additions and 13536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/executor/QueryExecutor.java
View File

@@ -1,12 +0,0 @@
package com.tencent.supersonic.semantic.query.application.executor;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
public interface QueryExecutor {
boolean accept(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(Catalog catalog,QueryStatement queryStatement);
}

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/optimizer/QueryOptimizer.java
View File

@@ -1,9 +0,0 @@
package com.tencent.supersonic.semantic.query.application.optimizer;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
public interface QueryOptimizer {
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
}

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SemanticItem.java
View File

@@ -1,6 +0,0 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
public interface SemanticItem {
public String getName();
}

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/Optimization.java
View File

@@ -1,9 +0,0 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
public interface Optimization {
public void visit(SemanticNode semanticNode);
}

344
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/convert/CalculateConverterAgg.java
View File

@@ -1,344 +0,0 @@
package com.tencent.supersonic.semantic.query.application.parser.convert;
import com.tencent.supersonic.common.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.utils.QueryStructUtils;
import com.tencent.supersonic.semantic.query.domain.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component("CalculateConverterAgg")
@Slf4j
public class CalculateConverterAgg implements SemanticConverter {
private final SemanticQueryEngine parserService;
private final QueryStructUtils queryStructUtils;
private final SqlGenerateUtils sqlGenerateUtils;
private final Catalog catalog;
@Value("${metricParser.agg.default:sum}")
private String metricAggDefault;
@Value("${metricParser.agg.ratio_roll.name:ratio_roll}")
private String metricAggRatioRollName;
@Value("${metricParser.agg.ratio_over.week:ratio_over_week}") // week over
private String metricAggRatioOverWeek;
@Value("${metricParser.agg.ratio_over.month:ratio_over_month}")
private String metricAggRatioOverMonth;
@Value("${metricParser.agg.ratio_over.quarter:ratio_over_quarter}")
private String metricAggRatioOverQuarter;
@Value("${metricParser.agg.ratio_over.year:ratio_over_year}")
private String metricAggRatioOverYear;
private List<String> dateGrain = new ArrayList<>(Arrays.asList("day", "week", "month", "year", "quarter"));
private Set<String> aggFunctionsOver = new HashSet<>(
Arrays.asList(metricAggRatioOverWeek, metricAggRatioOverMonth, metricAggRatioOverQuarter,
metricAggRatioOverYear));
public CalculateConverterAgg(
SemanticQueryEngine parserService,
@Lazy QueryStructUtils queryStructUtils,
SqlGenerateUtils sqlGenerateUtils, Catalog catalog) {
this.parserService = parserService;
this.queryStructUtils = queryStructUtils;
this.sqlGenerateUtils = sqlGenerateUtils;
this.catalog = catalog;
}
public ParseSqlReq generateSqlCommend(QueryStructReq queryStructCmd) throws Exception {
// 同环比
if (isRatioAccept(queryStructCmd)) {
return generateRatioSqlCommand(queryStructCmd);
}
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getDomainFullPath(queryStructCmd.getDomainId()));
String metricTableName = "metric_tb";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructCmd.getMetrics());
metricTable.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAgg(true);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
sqlGenerateUtils.getLimit(queryStructCmd));
sqlCommand.setSql(sql);
return sqlCommand;
}
@Override
public boolean accept(QueryStructReq queryStructCmd) {
if (queryStructCmd.getNativeQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {
return false;
}
//todo ck类型暂不拼with语句
if (queryStructCmd.getDomainId().equals(34L)) {
return false;
}
int nonSumFunction = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) {
return false;
}
if (agg.getFunc().equals(AggOperatorEnum.UNKNOWN)) {
return false;
}
if (agg.getFunc() != null
// && !agg.getFunc().equalsIgnoreCase(MetricAggDefault)
) {
nonSumFunction++;
}
}
return nonSumFunction > 0;
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd);
sqlCommend.setSql(parseSqlReq.getSql());
sqlCommend.setTables(parseSqlReq.getTables());
sqlCommend.setRootPath(parseSqlReq.getRootPath());
sqlCommend.setVariables(parseSqlReq.getVariables());
}
/**
* Ratio
*/
public boolean isRatioAccept(QueryStructReq queryStructCmd) {
Long ratioFuncNum = queryStructCmd.getAggregators().stream()
.filter(f -> f.getArgs() != null && f.getArgs().get(0) != null
&& metricAggRatioRollName.equalsIgnoreCase(f.getArgs().get(0))).count();
if (ratioFuncNum > 0) {
return true;
}
return false;
}
public ParseSqlReq generateRatioSqlCommand(QueryStructReq queryStructCmd) throws Exception {
check(queryStructCmd);
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getDomainFullPath(queryStructCmd.getDomainId()));
String metricTableName = "metric_tb";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructCmd.getMetrics());
metricTable.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
log.info("in generateSqlCommend, complete where:{}", where);
// metricTable.setWhere(queryStructCmd.getWhereClause());
metricTable.setWhere(where);
metricTable.setAgg(false);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sqlInner = String.format("select %s from %s %s ", getSelect(queryStructCmd), metricTableName,
getGroupBy(queryStructCmd));
String sql = String.format(
"select %s from ( select %s , %s from ( %s ) metric_tb_inner_1 ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd), getSelect(queryStructCmd, true), getLeadSelect(queryStructCmd), sqlInner,
getOrderBy(queryStructCmd), getLimit(queryStructCmd));
sqlCommand.setSql(sql);
return sqlCommand;
}
private String getOverSelect(QueryStructReq queryStructCmd) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getArgs() != null && f.getArgs().size() > 0) {
return String.format("if(%s = date_add(%s_roll,%s) and %s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s",
timeDim, timeDim, timeSpan, f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getColumn());
} else {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
}
private String getLeadSelect(QueryStructReq queryStructCmd) {
String timeDim = getTimeDim(queryStructCmd);
String groupDimWithOutTime = getGroupDimWithOutTime(queryStructCmd);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getArgs() != null && f.getArgs().size() > 0 && f.getArgs().get(0)
.equalsIgnoreCase(metricAggRatioRollName)) {
return String.format("lead(%s ,1,0) over ( %s order by %s desc) as %s_roll", f.getColumn(),
!groupDimWithOutTime.isEmpty() ? " partition by " + groupDimWithOutTime : "", timeDim,
f.getColumn());
} else {
return "";
}
}).filter(f -> !f.isEmpty()).collect(Collectors.joining(","));
String timeDimLead = String.format("lead(cast(%s as string) ,1,'') over ( %s order by %s desc) as %s_roll",
timeDim, !groupDimWithOutTime.isEmpty() ? " partition by " + groupDimWithOutTime : "", timeDim,
timeDim);
return timeDimLead + " , " + aggStr;
}
private String getTimeSpan(QueryStructReq queryStructCmd) {
String timeGrain = getTimeDimGrain(queryStructCmd).toLowerCase();
if ("week".equalsIgnoreCase(timeGrain)) {
return "7 day";
}
if ("quarter".equalsIgnoreCase(timeGrain)) {
return "3 month";
}
return "1 " + timeGrain;
}
private String getTimeDimGrain(QueryStructReq queryStructCmd) {
String grain = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getArgs() != null && f.getArgs().size() > 1 && f.getArgs().get(0)
.equalsIgnoreCase(metricAggRatioRollName)) {
return f.getArgs().get(1);
}
return "";
}).filter(f -> !f.isEmpty()).findFirst().orElse("");
return grain.isEmpty() ? "day" : grain;
}
private String getGroupDimWithOutTime(QueryStructReq queryStructCmd) {
String timeDim = getTimeDim(queryStructCmd);
return queryStructCmd.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(","));
}
private String getTimeDim(QueryStructReq queryStructCmd) {
String dsField = "";
String dsStart = "";
String dsEnd = "";
for (Filter filter : queryStructCmd.getOriginalFilter()) {
if (Filter.Relation.FILTER.equals(filter.getRelation())) {
// TODO get parameters from DateInfo
if ("DATE".equalsIgnoreCase(filter.getRelation().name())) {
if (FilterOperatorEnum.GREATER_THAN_EQUALS.getValue()
.equalsIgnoreCase(filter.getOperator().toString())
|| FilterOperatorEnum.GREATER_THAN.getValue()
.equalsIgnoreCase(filter.getOperator().toString())) {
dsField = filter.getBizName();
dsStart = filter.getValue().toString();
}
if (FilterOperatorEnum.MINOR_THAN_EQUALS.getValue()
.equalsIgnoreCase(filter.getOperator().toString())
|| FilterOperatorEnum.MINOR_THAN.getValue()
.equalsIgnoreCase(filter.getOperator().toString())) {
dsField = filter.getBizName();
dsEnd = filter.getValue().toString();
}
}
}
}
return dsField;
}
private String getLimit(QueryStructReq queryStructCmd) {
if (queryStructCmd.getLimit() > 0) {
return " limit " + String.valueOf(queryStructCmd.getLimit());
}
return "";
}
private String getSelect(QueryStructReq queryStructCmd) {
return getSelect(queryStructCmd, false);
}
private String getSelect(QueryStructReq queryStructCmd, boolean isRatio) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> getSelectField(f, isRatio))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
}
private String getSelectField(final Aggregator agg, boolean isRatio) {
if (!CollectionUtils.isEmpty(agg.getArgs()) && agg.getArgs().size() > 0) {
if (agg.getArgs().get(0).equalsIgnoreCase(metricAggRatioRollName)) {
if (isRatio) {
return agg.getColumn();
}
return agg.getFunc().name().isEmpty() ? agg.getColumn()
: agg.getFunc() + "( " + agg.getColumn() + " ) AS " + agg.getColumn() + " ";
}
}
if (CollectionUtils.isEmpty(agg.getArgs())) {
return agg.getFunc() + "( " + agg.getColumn() + " ) AS " + agg.getColumn() + " ";
}
return agg.getFunc() + "( " + agg.getArgs().stream().map(arg ->
arg.equals(agg.getColumn()) ? arg : (StringUtils.isNumeric(arg) ? arg : ("'" + arg + "'"))
).collect(Collectors.joining(",")) + " ) AS " + agg.getColumn() + " ";
}
private String getGroupBy(QueryStructReq queryStructCmd) {
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
return "";
}
return "group by " + String.join(",", queryStructCmd.getGroups());
}
private String getOrderBy(QueryStructReq queryStructCmd) {
if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) {
return "";
}
return "order by " + queryStructCmd.getOrders().stream()
.map(order -> " " + order.getColumn() + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
private void check(QueryStructReq queryStructCmd) throws Exception {
Set<String> aggFunctions = queryStructCmd.getAggregators().stream()
.filter(f -> f.getArgs() != null && f.getArgs().get(0) != null)
.map(agg -> agg.getArgs().get(0).toLowerCase()).collect(Collectors.toSet());
Long ratioOverNum = aggFunctions.stream().filter(aggFunctionsOver::contains).count();
if (ratioOverNum > 0) {
throw new Exception("not support over ratio");
}
if (aggFunctions.contains(metricAggRatioRollName)) {
if (ratioOverNum > 0) {
throw new Exception("not support over ratio and roll ratio together ");
}
}
if (getTimeDim(queryStructCmd).isEmpty()) {
throw new Exception("miss time filter");
}
String timeDimGrain = getTimeDimGrain(queryStructCmd).toLowerCase();
if (!dateGrain.contains(timeDimGrain)) {
throw new Exception("second arg must be [day week month year quarter] ");
}
}
}

26
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/SchemaService.java
View File

@@ -1,26 +0,0 @@
package com.tencent.supersonic.semantic.query.domain;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.core.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.core.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.core.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.DomainResp;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import java.util.List;
public interface SchemaService {
List<DomainSchemaResp> fetchDomainSchema(DomainSchemaFilterReq filter, User user);
List<DomainResp> getDomainListForAdmin(User user);
List<DomainResp> getDomainListForViewer(User user);
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq, User user);
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/executor/JdbcExecutor.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/executor/JdbcExecutor.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.semantic.query.application.executor;
package com.tencent.supersonic.semantic.query.executor;
import com.tencent.supersonic.semantic.api.core.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.core.domain.utils.SqlUtils;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.utils.SqlUtils;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/executor/QueryExecutor.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.semantic.query.executor;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
public interface QueryExecutor {
boolean accept(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(Catalog catalog,QueryStatement queryStatement);
}

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/optimizer/DetailQuery.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/DetailQuery.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.application.optimizer;
package com.tencent.supersonic.semantic.query.optimizer;
import com.google.common.base.Strings;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/QueryOptimizer.java Normal file
View File

@@ -0,0 +1,8 @@
package com.tencent.supersonic.semantic.query.optimizer;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
public interface QueryOptimizer {
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
}

25
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/QueryParser.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java
View File

@@ -1,14 +1,15 @@
package com.tencent.supersonic.semantic.query.application.parser;
package com.tencent.supersonic.semantic.query.parser;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.domain.utils.ComponentFactory;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
@@ -50,7 +51,7 @@ public class QueryParser {
QueryStatement queryStatement = new QueryStatement();
try {
if (!CollectionUtils.isEmpty(sqlCommend.getTables())) {
List<String> tables = new ArrayList<>();
List<String[]> tables = new ArrayList<>();
String sourceId = "";
for (MetricTable metricTable : sqlCommend.getTables()) {
MetricReq metricReq = new MetricReq();
@@ -64,12 +65,22 @@ public class QueryParser {
tableSql.getErrMsg()));
return queryStatement;
}
tables.add(String.format("%s as (%s)", metricTable.getAlias(), tableSql.getSql()));
tables.add(new String[]{metricTable.getAlias(), tableSql.getSql()});
sourceId = tableSql.getSourceId();
}
if (!tables.isEmpty()) {
String sql = "with " + String.join(",", tables) + "\n" + sqlCommend.getSql();
String sql = "";
if (sqlCommend.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + sqlCommend.getSql();
} else {
sql = sqlCommend.getSql();
for (String[] tb : tables) {
sql = sql.replaceAll(tb[0], "(" + tb[1] + ")");
}
}
queryStatement.setSql(sql);
queryStatement.setSourceId(sourceId);
return queryStatement;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticConverter.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/SemanticConverter.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.semantic.query.application.parser;
package com.tencent.supersonic.semantic.query.parser;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.Catalog;
public interface SemanticConverter {

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SqlParser.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/SqlParser.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.application.parser;
package com.tencent.supersonic.semantic.query.parser;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
public interface SqlParser {
QueryStatement explain(MetricReq metricReq, boolean isAgg, Catalog catalog) throws Exception;

15
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/CalciteSqlParser.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java
View File

@@ -1,14 +1,13 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite;
package com.tencent.supersonic.semantic.query.parser.calcite;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SqlParser;
import com.tencent.supersonic.semantic.query.application.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SqlParser;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
@Component("CalciteSqlParser")
public class CalciteSqlParser implements SqlParser {

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/Configuration.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/Configuration.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite;
package com.tencent.supersonic.semantic.query.parser.calcite;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.DSLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
import java.util.Properties;
import org.apache.calcite.avatica.util.Casing;
import org.apache.calcite.avatica.util.Quoting;

42
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/SemanticSchemaManager.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java
View File

@@ -1,25 +1,25 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite;
package com.tencent.supersonic.semantic.query.parser.calcite;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.MetricTypeParams;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MetricTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -68,7 +68,7 @@ public class SemanticSchemaManager {
catalog.getModelYamlTplByDomainIds(domainIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls);
if (!datasourceYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = datasourceYamlTpls.stream().map(d -> getDatasource(d))
.collect(Collectors.toMap(DataSource::getName, item -> item));
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
semanticModel.setDatasourceMap(dataSourceMap);
}
if (!dimensionYamlTpls.isEmpty()) {
@@ -267,8 +267,8 @@ public class SemanticSchemaManager {
@EnableCaching
public class GuavaCacheConfig {
@Value("${parser.cache.saveMinute:15}")
private Integer saveMinutes = 15;
@Value("${parser.cache.saveMinute:1}")
private Integer saveMinutes = 1;
@Value("${parser.cache.maximumSize:1000}")
private Integer maximumSize = 1000;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Constants.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
public class Constants {

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/DataSource.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
import java.util.List;
import lombok.Data;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Dimension.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Dimension.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticItem;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/DimensionTimeTypeParams.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DimensionTimeTypeParams.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Identify.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
import lombok.AllArgsConstructor;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Measure.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Measure.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
import lombok.AllArgsConstructor;
import lombok.Data;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Metric.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Metric.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticItem;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;
import java.util.List;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/MetricTypeParams.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MetricTypeParams.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
import java.util.List;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/SemanticModel.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/SemanticModel.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
import java.util.ArrayList;
import java.util.HashMap;

22
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/planner/AggPlanner.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java
View File

@@ -1,17 +1,17 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.planner;
package com.tencent.supersonic.semantic.query.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render.FilterRender;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render.OutputRender;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render.SourceRender;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.FilterRender;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.OutputRender;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.SourceRender;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/planner/Planner.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/Planner.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.planner;
package com.tencent.supersonic.semantic.query.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/DataSourceTable.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/DataSourceTable.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import java.util.ArrayList;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SchemaBuilder.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SchemaBuilder.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.DSLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticItem.java Normal file
View File

@@ -0,0 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.schema;
public interface SemanticItem {
public String getName();
}

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SemanticSchema.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SemanticSqlConformance.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSqlConformance.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import org.apache.calcite.sql.fun.SqlLibrary;
import org.apache.calcite.sql.validate.SqlConformance;

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SemanticSqlDialect.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSqlDialect.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.google.common.base.Preconditions;
import org.apache.calcite.avatica.util.Casing;
@@ -35,16 +35,11 @@ public class SemanticSqlDialect extends SqlDialect {
fetchFrame = writer.startList(SqlWriter.FrameTypeEnum.OFFSET);
writer.keyword("LIMIT");
if (offset != null) {
//writer.keyword("OFFSET");
offset.unparse(writer, -1, -1);
//writer.keyword("ROWS");
}
if (fetch != null) {
//writer.newlineAndIndent();
//fetchFrame = writer.startList(SqlWriter.FrameTypeEnum.FETCH);
writer.keyword(",");
//writer.keyword("NEXT");
fetch.unparse(writer, -1, -1);
}
@@ -92,4 +87,4 @@ public class SemanticSqlDialect extends SqlDialect {
public void unparseOffsetFetch(SqlWriter writer, @Nullable SqlNode offset, @Nullable SqlNode fetch) {
unparseFetchUsingAnsi(writer, offset, fetch);
}
}
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/DSLSqlValidatorImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/DSLSqlValidatorImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql;
package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.sql.SqlOperatorTable;

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Optimization.java Normal file
View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
public interface Optimization {
public void visit(SemanticNode semanticNode);
}

20
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/Renderer.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Renderer.java
View File

@@ -1,16 +1,16 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql;
package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/TableView.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql;
package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import java.util.ArrayList;
import java.util.List;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/AggFunctionNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/AggFunctionNode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

33
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/DataSourceNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -1,18 +1,19 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.application.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -20,6 +21,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
@Slf4j
public class DataSourceNode extends SemanticNode {
@@ -59,6 +61,24 @@ public class DataSourceNode extends SemanticNode {
metricCommand.getMetrics().stream().filter(m -> !schemaMetricName.contains(m)).forEach(m -> measures.add(m));
}
public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema, MetricReq metricCommand,
Set<String> queryDimension, List<String> measures,SqlValidatorScope scope) throws Exception {
if(Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
Set<String> filterConditions = new HashSet<>();
FilterNode.getFilterField(parse(metricCommand.getWhere(), scope), filterConditions);
Set<String> queryMeasures = new HashSet<>(measures);
Set<String> schemaMetricName = schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet());
for(String filterCondition : filterConditions) {
if(schemaMetricName.contains(filterCondition)) {
schema.getMetrics().stream().filter(m->m.getName().equalsIgnoreCase(filterCondition)).forEach(m -> m.getMetricTypeParams().getMeasures().stream().forEach(mm -> queryMeasures.add(mm.getName())));
continue;
}
queryDimension.add(filterCondition);
}
measures.clear();
measures.addAll(queryMeasures);
}
}
public static List<DataSource> getMatchDataSources(SqlValidatorScope scope, SemanticSchema schema,
MetricReq metricCommand) throws Exception {
@@ -98,6 +118,7 @@ public class DataSourceNode extends SemanticNode {
}
filterMeasure.addAll(sourceMeasure);
filterMeasure.addAll(dimension);
mergeQueryFilterDimensionMeasure(schema,metricCommand,queryDimension,measures,scope);
boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension, metricCommand, scope);
if (isAllMatch) {
log.info("baseDataSource match all ");

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/DimensionNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DimensionNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import java.util.List;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/FilterNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/FilterNode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import java.util.Set;
import org.apache.calcite.sql.SqlBasicCall;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/IdentifyNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/JoinNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/JoinNode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import lombok.Data;
import org.apache.calcite.sql.SqlNode;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/MeasureNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MeasureNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/MetricNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MetricNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/SemanticNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/SemanticNode.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.application.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Optimization;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Optimization;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;

20
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/render/FilterRender.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/FilterRender.java
View File

@@ -1,16 +1,16 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

33
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/render/JoinRender.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java
View File

@@ -1,19 +1,19 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.AggFunctionNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.AggFunctionNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -101,10 +101,7 @@ public class JoinRender extends Renderer {
left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope));
continue;
}
if (!left.getKind().equals(SqlKind.AS)) {
left = SemanticNode.buildAs(Constants.JOIN_TABLE_LEFT_PREFIX + tableView.getAlias(),
getTable(tableView, scope));
}
left = new SqlJoin(
SqlParserPos.ZERO,
left,

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/render/OutputRender.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/OutputRender.java
View File

@@ -1,13 +1,13 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.List;
import org.apache.calcite.sql.SqlNode;

41
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/render/SourceRender.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java
View File

@@ -1,24 +1,23 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render;
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.DimensionNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.IdentifyNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DimensionNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.IdentifyNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -252,8 +251,12 @@ public class SourceRender extends Renderer {
Optional<Metric> datasourceMetric = schema.getMetrics()
.stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst();
if (datasourceMetric.isPresent()) {
metrics.add(oriField);
return;
Set<String> measures = datasourceMetric.get().getMetricTypeParams().getMeasures().stream().map(m->m.getName()).collect(
Collectors.toSet());
if(datasource.getMeasures().stream().map(m->m.getName()).collect(Collectors.toSet()).containsAll(measures)){
metrics.add(oriField);
return;
}
}
}

478
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java Normal file
View File

@@ -0,0 +1,478 @@
package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.utils.DateUtils;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import com.tencent.supersonic.semantic.query.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component("CalculateAggConverter")
@Slf4j
public class CalculateAggConverter implements SemanticConverter {
private final SemanticQueryEngine parserService;
private final QueryStructUtils queryStructUtils;
private final SqlGenerateUtils sqlGenerateUtils;
private final Catalog catalog;
@Value("${metricParser.agg.default:sum}")
private String metricAggDefault;
@Value("${metricParser.agg.mysql.lowVersion:5.7}")
private String mysqlLowVersion;
public CalculateAggConverter(
SemanticQueryEngine parserService,
@Lazy QueryStructUtils queryStructUtils,
SqlGenerateUtils sqlGenerateUtils, Catalog catalog) {
this.parserService = parserService;
this.queryStructUtils = queryStructUtils;
this.sqlGenerateUtils = sqlGenerateUtils;
this.catalog = catalog;
}
public interface EngineSql {
String sql(QueryStructReq queryStructCmd, boolean isOver, String metricSql);
}
public ParseSqlReq generateSqlCommend(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum, String version)
throws Exception {
// 同环比
if (isRatioAccept(queryStructCmd)) {
return generateRatioSqlCommand(queryStructCmd, engineTypeEnum, version);
}
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getDomainFullPath(queryStructCmd.getDomainId()));
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructCmd.getMetrics());
metricTable.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAgg(true);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
sqlGenerateUtils.getLimit(queryStructCmd));
if (engineTypeEnum.equals(engineTypeEnum.MYSQL) && Objects.nonNull(version) && version.startsWith(
mysqlLowVersion)) {
sqlCommand.setSupportWith(false);
sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
sqlGenerateUtils.getLimit(queryStructCmd));
}
sqlCommand.setSql(sql);
return sqlCommand;
}
@Override
public boolean accept(QueryStructReq queryStructCmd) {
if (queryStructCmd.getNativeQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {
return false;
}
//todo ck类型暂不拼with语句
if (queryStructCmd.getDomainId().equals(34L)) {
return false;
}
int nonSumFunction = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) {
return false;
}
if (agg.getFunc().equals(AggOperatorEnum.UNKNOWN)) {
return false;
}
if (agg.getFunc() != null
// && !agg.getFunc().equalsIgnoreCase(MetricAggDefault)
) {
nonSumFunction++;
}
}
return nonSumFunction > 0;
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
DatabaseResp databaseResp = catalog.getDatabaseByDomainId(queryStructCmd.getDomainId());
ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd,
EngineTypeEnum.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
sqlCommend.setSql(parseSqlReq.getSql());
sqlCommend.setTables(parseSqlReq.getTables());
sqlCommend.setRootPath(parseSqlReq.getRootPath());
sqlCommend.setVariables(parseSqlReq.getVariables());
sqlCommend.setSupportWith(parseSqlReq.isSupportWith());
}
/**
* Ratio
*/
public boolean isRatioAccept(QueryStructReq queryStructCmd) {
Long ratioFuncNum = queryStructCmd.getAggregators().stream()
.filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL) || f.getFunc()
.equals(AggOperatorEnum.RATIO_OVER))).count();
if (ratioFuncNum > 0) {
return true;
}
return false;
}
public ParseSqlReq generateRatioSqlCommand(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum,
String version)
throws Exception {
check(queryStructCmd);
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getDomainFullPath(queryStructCmd.getDomainId()));
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructCmd.getMetrics());
metricTable.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAgg(true);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
boolean isOver = isOverRatio(queryStructCmd);
String sql = "";
switch (engineTypeEnum) {
case H2:
sql = new H2EngineSql().sql(queryStructCmd, isOver, metricTableName);
break;
case MYSQL:
if (Objects.nonNull(version) && version.startsWith(mysqlLowVersion)) {
sqlCommand.setSupportWith(false);
sql = new MysqlEngineSql().sql(queryStructCmd, isOver, metricTableName);
break;
}
case DORIS:
case CLICKHOUSE:
sql = new CkEngineSql().sql(queryStructCmd, isOver, metricTableName);
break;
}
sqlCommand.setSql(sql);
return sqlCommand;
}
public class H2EngineSql implements EngineSql {
public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getColumn());
} else {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
}
public String getTimeSpan(QueryStructReq queryStructCmd, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryStructCmd.getDateInfo())) {
String addStr = isAdd ? "" : "-";
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) {
return "day," + (isOver ? addStr + "7" : addStr + "1");
}
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) {
return isOver ? "month," + addStr + "1" : "day," + addStr + "7";
}
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) {
return isOver ? "year," + addStr + "1" : "month," + addStr + "1";
}
}
return "";
}
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = getTimeSpan(queryStructCmd, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format(
"%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ",
aliasRight + timeDim, aliasLeft + timeDim, timeSpan, aliasRight + timeDim);
}
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ",
getTimeSpan(queryStructCmd, isOver, false), aliasLeft + timeDim, aliasRight + timeDim);
}
return String.format("%s = TIMESTAMPADD(%s,%s) ",
aliasLeft + timeDim, timeSpan, aliasRight + timeDim);
} else {
return f.getColumn();
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
groups.add(aliasLeft + group + " = " + aliasRight + group);
}
return CollectionUtils.isEmpty(groups) ? aggStr
: String.join(" and ", groups) + " and " + aggStr + " ";
}
@Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, String metricSql) {
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), metricSql, metricSql,
getJoinOn(queryStructCmd, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd));
return sql;
}
}
public class CkEngineSql extends MysqlEngineSql {
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format("toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s','-01')),%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryStructCmd, isOver, false), aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
} else {
return f.getColumn();
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
groups.add(aliasLeft + group + " = " + aliasRight + group);
}
return CollectionUtils.isEmpty(groups) ? aggStr
: String.join(" and ", groups) + " and " + aggStr + " ";
}
@Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, String metricSql) {
String sql = String.format(
",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s from t0 left join t1 on %s ) metric_tb_src %s %s ",
metricSql, metricSql, getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."),
getAllJoinSelect(queryStructCmd, "t1."),
getJoinOn(queryStructCmd, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd));
return sql;
}
}
public class MysqlEngineSql implements EngineSql {
public String getTimeSpan(QueryStructReq queryStructCmd, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryStructCmd.getDateInfo())) {
String addStr = isAdd ? "" : "-";
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.DAY)) {
return isOver ? addStr + "7 day" : addStr + "1 day";
}
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.WEEK)) {
return isOver ? addStr + "1 month" : addStr + "7 day";
}
if (queryStructCmd.getDateInfo().getPeriod().equalsIgnoreCase(Constants.MONTH)) {
return isOver ? addStr + "1 year" : addStr + "1 month";
}
}
return "";
}
public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format(
"if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getColumn());
} else {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
}
public String getJoinOn(QueryStructReq queryStructCmd, boolean isOver, String aliasLeft, String aliasRight) {
String timeDim = getTimeDim(queryStructCmd);
String timeSpan = "INTERVAL " + getTimeSpan(queryStructCmd, isOver, true);
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.MONTH)) {
return String.format("%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%Y-%m') ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryStructCmd.getDateInfo().getPeriod().equals(Constants.WEEK) && isOver) {
return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryStructCmd, isOver, false), aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
} else {
return f.getColumn();
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
groups.add(aliasLeft + group + " = " + aliasRight + group);
}
return CollectionUtils.isEmpty(groups) ? aggStr
: String.join(" and ", groups) + " and " + aggStr + " ";
}
@Override
public String sql(QueryStructReq queryStructCmd, boolean isOver, String metricSql) {
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryStructCmd, isOver), getAllSelect(queryStructCmd, "t0."),
getAllJoinSelect(queryStructCmd, "t1."), metricSql, metricSql,
getJoinOn(queryStructCmd, isOver, "t0.", "t1."),
getOrderBy(queryStructCmd), getLimit(queryStructCmd));
return sql;
}
}
private static String getAllJoinSelect(QueryStructReq queryStructCmd, String alias) {
String aggStr = queryStructCmd.getAggregators().stream()
.map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "")
+ "_roll")
.collect(Collectors.joining(","));
List<String> groups = new ArrayList<>();
for (String group : queryStructCmd.getGroups()) {
groups.add(alias + group + " as " + group + "_roll");
}
return CollectionUtils.isEmpty(groups) ? aggStr
: String.join(",", groups) + "," + aggStr;
}
private String getGroupDimWithOutTime(QueryStructReq queryStructCmd) {
String timeDim = getTimeDim(queryStructCmd);
return queryStructCmd.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(","));
}
private static String getTimeDim(QueryStructReq queryStructCmd) {
DateUtils dateUtils = ContextUtils.getContext().getBean(DateUtils.class);
return dateUtils.getSysDateCol(queryStructCmd.getDateInfo());
}
private static String getLimit(QueryStructReq queryStructCmd) {
if (queryStructCmd.getLimit() > 0) {
return " limit " + String.valueOf(queryStructCmd.getLimit());
}
return "";
}
private static String getAllSelect(QueryStructReq queryStructCmd, String alias) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> getSelectField(f, alias))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: alias + String.join("," + alias, queryStructCmd.getGroups()) + "," + aggStr;
}
private static String getSelectField(final Aggregator agg, String alias) {
if (agg.getFunc().equals(AggOperatorEnum.RATIO_OVER) || agg.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return alias + agg.getColumn();
}
if (CollectionUtils.isEmpty(agg.getArgs())) {
return agg.getFunc() + "( " + alias + agg.getColumn() + " ) AS " + agg.getColumn() + " ";
}
return agg.getFunc() + "( " + agg.getArgs().stream().map(arg ->
arg.equals(agg.getColumn()) ? arg : (StringUtils.isNumeric(arg) ? arg : ("'" + arg + "'"))
).collect(Collectors.joining(",")) + " ) AS " + agg.getColumn() + " ";
}
private String getGroupBy(QueryStructReq queryStructCmd) {
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
return "";
}
return "group by " + String.join(",", queryStructCmd.getGroups());
}
private static String getOrderBy(QueryStructReq queryStructCmd) {
return "order by " + getTimeDim(queryStructCmd) + " desc";
}
private boolean isOverRatio(QueryStructReq queryStructCmd) {
Long overCt = queryStructCmd.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
return overCt > 0;
}
private void check(QueryStructReq queryStructCmd) throws Exception {
Set<String> aggFunctions = queryStructCmd.getAggregators().stream()
.filter(f -> f.getArgs() != null && f.getArgs().get(0) != null)
.map(agg -> agg.getArgs().get(0).toLowerCase()).collect(Collectors.toSet());
Long ratioOverNum = queryStructCmd.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
Long ratioRollNum = queryStructCmd.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count();
if (ratioOverNum > 0 && ratioRollNum > 0) {
throw new Exception("not support over ratio and roll ratio together ");
}
if (getTimeDim(queryStructCmd).isEmpty()) {
throw new Exception("miss time filter");
}
}
}

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/convert/DefaultDimValueConverter.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java
View File

@@ -1,13 +1,13 @@
package com.tencent.supersonic.semantic.query.application.parser.convert;
package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

18
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/convert/MultiSourceJoin.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java
View File

@@ -1,21 +1,19 @@
package com.tencent.supersonic.semantic.query.application.parser.convert;
package com.tencent.supersonic.semantic.query.parser.convert;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.api.core.pojo.Identify;
import com.tencent.supersonic.semantic.api.core.pojo.Measure;
import com.tencent.supersonic.semantic.api.core.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.model.pojo.Identify;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/convert/ParserDefaultConverter.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java
View File

@@ -1,16 +1,16 @@
package com.tencent.supersonic.semantic.query.application.parser.convert;
package com.tencent.supersonic.semantic.query.parser.convert;
import static com.tencent.supersonic.common.constant.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.query.pojo.Param;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.domain.utils.QueryStructUtils;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
@@ -26,11 +26,11 @@ public class ParserDefaultConverter implements SemanticConverter {
@Value("${internal.metric.cnt.suffix:internal_cnt}")
private String internalMetricNameSuffix;
private final CalculateConverterAgg calculateCoverterAgg;
private final CalculateAggConverter calculateCoverterAgg;
private final QueryStructUtils queryStructUtils;
public ParserDefaultConverter(
CalculateConverterAgg calculateCoverterAgg,
CalculateAggConverter calculateCoverterAgg,
QueryStructUtils queryStructUtils) {
this.calculateCoverterAgg = calculateCoverterAgg;
this.queryStructUtils = queryStructUtils;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/infrastructure/mapper/StatMapper.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/mapper/StatMapper.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.infrastructure.mapper;
package com.tencent.supersonic.semantic.query.persistence.mapper;
import com.tencent.supersonic.semantic.api.core.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/pojo/ParserSvrResponse.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/pojo/ParserSvrResponse.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.pojo;
package com.tencent.supersonic.semantic.query.persistence.pojo;
public class ParserSvrResponse<T> {

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/pojo/QueryStatement.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/pojo/QueryStatement.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.semantic.query.domain.pojo;
package com.tencent.supersonic.semantic.query.persistence.pojo;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import lombok.Data;
@Data

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/repository/StatRepository.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepository.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.domain.repository;
package com.tencent.supersonic.semantic.query.persistence.repository;
import com.tencent.supersonic.semantic.api.core.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import java.util.List;

11
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/infrastructure/repository/StatRepositoryImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/persistence/repository/StatRepositoryImpl.java
View File

@@ -1,15 +1,14 @@
package com.tencent.supersonic.semantic.query.infrastructure.repository;
package com.tencent.supersonic.semantic.query.persistence.repository;
import static com.tencent.supersonic.common.constant.Constants.AT_SYMBOL;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.semantic.api.core.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.common.enums.TypeEnums;
import com.tencent.supersonic.semantic.query.domain.repository.StatRepository;
import com.tencent.supersonic.semantic.query.infrastructure.mapper.StatMapper;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.query.persistence.mapper.StatMapper;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.*;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.QueryService;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.service.QueryService;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -32,11 +32,11 @@ public class QueryController {
@PostMapping("/sql")
public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
public Object queryBySql(@RequestBody QueryDslReq queryDslReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
Object queryBySql = queryService.queryBySql(querySqlReq, user);
Object queryBySql = queryService.queryBySql(queryDslReq, user);
log.info("queryBySql:{},queryBySql");
return queryBySql;
}

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/SchemaController.java
View File

@@ -3,14 +3,14 @@ package com.tencent.supersonic.semantic.query.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.semantic.api.core.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.core.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.core.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.DomainResp;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.query.domain.SchemaService;
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import java.util.List;
import javax.servlet.http.HttpServletRequest;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/annotation/DataPermission.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DataPermission.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.annotation;
package com.tencent.supersonic.semantic.query.service;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/QueryService.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.semantic.query.domain;
package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.core.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import java.util.List;
@@ -14,7 +14,7 @@ import javax.servlet.http.HttpServletRequest;
public interface QueryService {
Object queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception;
Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;

59
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/QueryServiceImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -1,28 +1,24 @@
package com.tencent.supersonic.semantic.query.application;
package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.semantic.api.core.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.core.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.application.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.domain.QueryService;
import com.tencent.supersonic.semantic.query.domain.SchemaService;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.annotation.DataPermission;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.domain.utils.QueryReqConverter;
import com.tencent.supersonic.semantic.query.domain.utils.QueryUtils;
import com.tencent.supersonic.semantic.query.domain.utils.StatUtils;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.QueryReqConverter;
import com.tencent.supersonic.semantic.query.utils.QueryUtils;
import com.tencent.supersonic.semantic.query.utils.StatUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -62,7 +58,7 @@ public class QueryServiceImpl implements QueryService {
}
@Override
public Object queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception {
public Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception {
DomainSchemaFilterReq filter = new DomainSchemaFilterReq();
List<Long> domainIds = new ArrayList<>();
domainIds.add(querySqlCmd.getDomainId());
@@ -121,38 +117,37 @@ public class QueryServiceImpl implements QueryService {
@Override
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user)
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user)
throws Exception {
statUtils.initStatInfo(queryMultiStructCmd.getQueryStructCmds().get(0), user);
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
String cacheKey = cacheUtils.generateCacheKey(
queryMultiStructCmd.getQueryStructCmds().get(0).getDomainId().toString(),
queryMultiStructCmd.generateCommandMd5());
boolean isCache = isCache(queryMultiStructCmd);
queryMultiStructReq.getQueryStructReqs().get(0).getDomainId().toString(),
queryMultiStructReq.generateCommandMd5());
boolean isCache = isCache(queryMultiStructReq);
QueryResultWithSchemaResp queryResultWithColumns;
if (isCache) {
queryResultWithColumns = queryByCache(cacheKey, queryMultiStructCmd);
queryResultWithColumns = queryByCache(cacheKey, queryMultiStructReq);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
}
}
log.info("stat queryByStructWithoutCache, queryMultiStructCmd:{}", queryMultiStructCmd);
log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq);
try {
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructCmd : queryMultiStructCmd.getQueryStructCmds()) {
for (QueryStructReq queryStructCmd : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = semanticQueryEngine.plan(queryStructCmd);
queryUtils.checkSqlParse(queryStatement);
sqlParsers.add(queryStatement);
}
log.info("multi sqlParser:{}", sqlParsers);
QueryStatement sqlParser = queryUtils.sqlParserUnion(queryMultiStructCmd, sqlParsers);
QueryStatement sqlParser = queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
queryResultWithColumns = semanticQueryEngine.execute(sqlParser);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
queryUtils.fillItemNameInfo(queryResultWithColumns, queryMultiStructCmd);
queryUtils.fillItemNameInfo(queryResultWithColumns, queryMultiStructReq);
}
queryUtils.cacheResultLogic(cacheKey, queryResultWithColumns);
return queryResultWithColumns;
} catch (Exception e) {
log.warn("exception in queryByMultiStruct, e: ", e);
@@ -196,9 +191,9 @@ public class QueryServiceImpl implements QueryService {
if (!cacheEnable) {
return false;
}
if (!CollectionUtils.isEmpty(queryStructCmd.getQueryStructCmds())
&& queryStructCmd.getQueryStructCmds().get(0).getCacheInfo() != null) {
return queryStructCmd.getQueryStructCmds().get(0).getCacheInfo().getCache();
if (!CollectionUtils.isEmpty(queryStructCmd.getQueryStructReqs())
&& queryStructCmd.getQueryStructReqs().get(0).getCacheInfo() != null) {
return queryStructCmd.getQueryStructReqs().get(0).getCacheInfo().getCache();
}
return false;
}

26
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaService.java Normal file
View File

@@ -0,0 +1,26 @@
package com.tencent.supersonic.semantic.query.service;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import java.util.List;
public interface SchemaService {
List<DomainSchemaResp> fetchDomainSchema(DomainSchemaFilterReq filter, User user);
List<DomainResp> getDomainListForAdmin(User user);
List<DomainResp> getDomainListForViewer(User user);
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq, User user);
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
}

34
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SchemaServiceImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java
View File

@@ -1,29 +1,27 @@
package com.tencent.supersonic.semantic.query.application;
package com.tencent.supersonic.semantic.query.service;
import static com.tencent.supersonic.common.constant.Constants.AT_SYMBOL;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.core.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.core.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.core.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.DomainResp;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.common.enums.TypeEnums;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.query.domain.SchemaService;
import com.tencent.supersonic.semantic.query.domain.QueryService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -125,4 +123,4 @@ public class SchemaServiceImpl implements SchemaService {
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricCmd, User user) {
return metricService.queryMetric(pageMetricCmd);
}
}
}

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/SemanticQueryEngine.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SemanticQueryEngine.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.semantic.query.domain;
package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.query.application.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
public interface SemanticQueryEngine {

19
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticQueryEngineImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SemanticQueryEngineImpl.java
View File

@@ -1,17 +1,16 @@
package com.tencent.supersonic.semantic.query.application;
package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.application.optimizer.QueryOptimizer;
import com.tencent.supersonic.semantic.query.application.parser.QueryParser;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.domain.utils.ComponentFactory;
import com.tencent.supersonic.semantic.query.domain.utils.QueryUtils;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.optimizer.QueryOptimizer;
import com.tencent.supersonic.semantic.query.parser.QueryParser;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.ComponentFactory;
import com.tencent.supersonic.semantic.query.utils.QueryUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

28
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/ComponentFactory.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/ComponentFactory.java
View File

@@ -1,17 +1,17 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.semantic.query.application.executor.JdbcExecutor;
import com.tencent.supersonic.semantic.query.application.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.application.optimizer.DetailQuery;
import com.tencent.supersonic.semantic.query.application.optimizer.QueryOptimizer;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.application.parser.SqlParser;
import com.tencent.supersonic.semantic.query.application.parser.calcite.CalciteSqlParser;
import com.tencent.supersonic.semantic.query.application.parser.convert.CalculateConverterAgg;
import com.tencent.supersonic.semantic.query.application.parser.convert.DefaultDimValueConverter;
import com.tencent.supersonic.semantic.query.application.parser.convert.MultiSourceJoin;
import com.tencent.supersonic.semantic.query.application.parser.convert.ParserDefaultConverter;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.query.executor.JdbcExecutor;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.optimizer.DetailQuery;
import com.tencent.supersonic.semantic.query.optimizer.QueryOptimizer;
import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.parser.SqlParser;
import com.tencent.supersonic.semantic.query.parser.calcite.CalciteSqlParser;
import com.tencent.supersonic.semantic.query.parser.convert.CalculateAggConverter;
import com.tencent.supersonic.semantic.query.parser.convert.DefaultDimValueConverter;
import com.tencent.supersonic.semantic.query.parser.convert.MultiSourceJoin;
import com.tencent.supersonic.semantic.query.parser.convert.ParserDefaultConverter;
import java.util.ArrayList;
import java.util.List;
@@ -64,7 +64,7 @@ public class ComponentFactory {
}
private static void initSemanticConverter() {
semanticConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
semanticConverters.add(getBean("CalculateConverterAgg", CalculateConverterAgg.class));
semanticConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
semanticConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
semanticConverters.add(getBean("MultiSourceJoin", MultiSourceJoin.class));
}

32
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/DataPermissionAOP.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.constant.Constants.MINUS;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -12,22 +12,18 @@ import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.semantic.api.core.pojo.QueryAuthorization;
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.DomainResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.exception.InvalidArgumentException;
import com.tencent.supersonic.common.exception.InvalidPermissionException;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -39,7 +35,9 @@ import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
@@ -72,7 +70,7 @@ public class DataPermissionAOP {
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.semantic.query.domain.annotation.DataPermission)")
@Pointcut("@annotation(com.tencent.supersonic.semantic.query.service.DataPermission)")
public void dataPermissionAOP() {
}

52
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/DateUtils.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DateUtils.java
View File

@@ -1,16 +1,17 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.constant.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.constant.Constants.COMMA;
import static com.tencent.supersonic.common.constant.Constants.DAY;
import static com.tencent.supersonic.common.constant.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.constant.Constants.MONTH;
import static com.tencent.supersonic.common.constant.Constants.MONTH_FORMAT;
import static com.tencent.supersonic.common.constant.Constants.WEEK;
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.core.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
@@ -19,6 +20,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.StringJoiner;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -44,7 +46,7 @@ public class DateUtils {
return false;
}
public boolean hasDataMode(DateConf dateInfo) {
public boolean hasAvailableDataMode(DateConf dateInfo) {
if (Objects.nonNull(dateInfo) && DateConf.DateMode.AVAILABLE_TIME == dateInfo.getDateMode()) {
return true;
}
@@ -139,18 +141,16 @@ public class DateUtils {
}
public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) {
LocalDate endData = LocalDate.parse(dateDate.getEndDate(),
DateTimeFormatter.ofPattern(dateDate.getDateFormat()));
if (dateDate.getDatePeriod() != null && MONTH.equalsIgnoreCase(dateDate.getDatePeriod())) {
Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateDate.getDateFormat(),
ChronoUnit.MONTHS);
LocalDate endData = LocalDate.parse(dateDate.getEndDate(),
DateTimeFormatter.ofPattern(dateDate.getDateFormat()));
return generateMonthSql(endData, unit, dateDate.getDateFormat());
}
String dateFormatStr = MONTH_FORMAT;
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
LocalDate end = LocalDate.parse(dateDate.getEndDate(), formatter);
Integer unit = dateInfo.getUnit() - 1;
return recentMonthStr(end, Long.valueOf(unit), dateFormatStr);
return recentMonthStr(endData, Long.valueOf(unit), dateFormatStr);
}
public String recentWeekStr(LocalDate endData, Long unit) {
@@ -217,7 +217,14 @@ public class DateUtils {
public String listDateStr(ItemDateResp dateDate, DateConf dateInfo) {
StringJoiner joiner = new StringJoiner(COMMA);
dateInfo.getDateList().stream().forEach(date -> joiner.add(APOSTROPHE + date + APOSTROPHE));
return String.format("(%s in (%s))", sysDateCol, joiner.toString());
String dateCol = sysDateCol;
if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
dateCol = sysDateMonthCol;
}
if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
dateCol = sysDateWeekCol;
}
return String.format("(%s in (%s))", dateCol, joiner.toString());
}
/**
@@ -275,4 +282,17 @@ public class DateUtils {
return dateStr;
}
public String getSysDateCol(DateConf dateInfo) {
if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) {
return sysDateCol;
}
if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
return sysDateWeekCol;
}
if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
return sysDateMonthCol;
}
return "";
}
}

178
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java Normal file
View File

@@ -0,0 +1,178 @@
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@Aspect
@Component
@Slf4j
public class DimValueAspect {
@Value("${dimension.value.map.enable:true}")
private Boolean dimensionValueMapEnable;
@Autowired
private DimensionService dimensionService;
@Around("execution(* com.tencent.supersonic.semantic.query.rest.QueryController.queryByStruct(..))" +
" || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStruct(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
return queryResultWithColumns;
}
Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0];
Long domainId = queryStructReq.getDomainId();
List<DimensionResp> dimensions = dimensionService.getDimensions(domainId);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = new ConcurrentHashMap<>();
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = new ConcurrentHashMap<>();
generateAliasAndTechNamePair(dimensions, dimAndAliasAndTechNamePair, dimAndTechNameAndBizNamePair);
rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
if (Objects.nonNull(queryResultWithColumns)) {
rewriteDimValue(queryResultWithColumns, dimAndTechNameAndBizNamePair);
}
return queryResultWithColumns;
}
private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns, Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (!selectDimValueMap(queryResultWithColumns.getColumns(), dimAndTechNameAndBizNamePair)) {
return;
}
log.debug("start rewriteDimValue for resultList");
for (Map<String, Object> line : queryResultWithColumns.getResultList()) {
for (String bizName : line.keySet()) {
String techName = line.get(bizName).toString();
if (dimAndTechNameAndBizNamePair.containsKey(bizName)) {
Map<String, String> techAndBizPair = dimAndTechNameAndBizNamePair.get(bizName);
if (!CollectionUtils.isEmpty(techAndBizPair) && techAndBizPair.containsKey(techName)) {
String bizValueName = techAndBizPair.get(techName);
if (Strings.isNotEmpty(bizValueName)) {
line.put(bizName, bizValueName);
}
}
}
}
}
}
private boolean selectDimValueMap(List<QueryColumn> columns, Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair) || CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)) {
return false;
}
for (QueryColumn queryColumn : columns) {
if (dimAndTechNameAndBizNamePair.containsKey(queryColumn.getNameEn())) {
return true;
}
}
return false;
}
private void rewriteFilter(List<Filter> dimensionFilters, Map<String, Map<String, String>> aliasAndTechNamePair) {
for (Filter filter : dimensionFilters) {
if (Objects.isNull(filter)) {
continue;
}
if (CollectionUtils.isEmpty(filter.getChildren())) {
Object value = filter.getValue();
String bizName = filter.getBizName();
if (aliasAndTechNamePair.containsKey(bizName)) {
Map<String, String> aliasPair = aliasAndTechNamePair.get(bizName);
if (Objects.nonNull(value)) {
if (value instanceof List) {
List<String> values = (List) value;
List<String> valuesNew = new ArrayList<>();
for (String valueSingle : values) {
boolean f = aliasPair.containsKey(valueSingle) ? valuesNew.add(aliasPair.get(valueSingle)) : valuesNew.add(valueSingle);
}
filter.setValue(valuesNew);
}
if (value instanceof String) {
if (aliasPair.containsKey(value)) {
filter.setValue(aliasPair.get(value));
}
}
}
}
return;
}
rewriteFilter(filter.getChildren(), aliasAndTechNamePair);
}
}
private void generateAliasAndTechNamePair(List<DimensionResp> dimensions
, Map<String, Map<String, String>> dimAndAliasAndTechNamePair
, Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
dimensions.stream().forEach(dimension -> {
if (Objects.nonNull(dimension) && Strings.isNotEmpty(dimension.getBizName())
&& !CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
String bizName = dimension.getBizName();
List<DimValueMap> dimValueMaps = dimension.getDimValueMaps();
Map<String, String> innerPairTech = new HashMap<>();
Map<String, String> innerPairBiz = new HashMap<>();
dimValueMaps.stream().forEach(dimValueMap -> {
if (Objects.nonNull(dimValueMap) && !CollectionUtils.isEmpty(dimValueMap.getAlias())
&& Strings.isNotEmpty(dimValueMap.getTechName())) {
// add bizName and techName pair
if (Strings.isNotEmpty(dimValueMap.getBizName())) {
innerPairTech.put(dimValueMap.getBizName(), dimValueMap.getTechName());
}
dimValueMap.getAlias().stream().forEach(alias -> {
if (Strings.isNotEmpty(alias)) {
innerPairTech.put(alias, dimValueMap.getTechName());
}
});
}
if (Objects.nonNull(dimValueMap) && Strings.isNotEmpty(dimValueMap.getTechName())) {
innerPairBiz.put(dimValueMap.getTechName(), dimValueMap.getBizName());
}
});
if (!CollectionUtils.isEmpty(innerPairTech)) {
dimAndAliasAndTechNamePair.put(bizName, innerPairTech);
}
if (!CollectionUtils.isEmpty(innerPairBiz)) {
dimAndTechNameAndBizNamePair.put(bizName, innerPairBiz);
}
}
});
}
}

20
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/QueryReqConverter.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryReqConverter.java
View File

@@ -1,15 +1,15 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.util.calcite.SqlParseUtils;
import com.tencent.supersonic.common.util.calcite.SqlParserInfo;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -28,7 +28,7 @@ public class QueryReqConverter {
@Autowired
private SemanticQueryEngine parserService;
public QueryStatement convert(QuerySqlReq databaseReq, List<DomainSchemaResp> domainSchemas) throws Exception {
public QueryStatement convert(QueryDslReq databaseReq, List<DomainSchemaResp> domainSchemas) throws Exception {
List<MetricTable> tables = new ArrayList<>();
MetricTable metricTable = new MetricTable();
@@ -67,7 +67,9 @@ public class QueryReqConverter {
result.setRootPath(domainService.getDomainFullPath(databaseReq.getDomainId()));
result.setTables(tables);
return parserService.physicalSql(result);
QueryStatement queryStatement = parserService.physicalSql(result);
queryStatement.setSql(String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql()));
return queryStatement;
}
}

30
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/QueryStructUtils.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -1,15 +1,17 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.core.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -18,6 +20,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
@@ -92,15 +95,22 @@ public class QueryStructUtils {
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
List<Long> metricIds = getMetricIds(queryStructCmd);
ItemDateResp dateDate = catalog.getDateDate(
ItemDateResp dateDate = catalog.getItemDate(
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()),
new ItemDateFilter(metricIds, TypeEnums.METRIC.getName()));
if (Objects.isNull(dateDate)
|| Strings.isEmpty(dateDate.getStartDate())
&& Strings.isEmpty(dateDate.getEndDate())) {
if (dateUtils.hasDataMode(dateInfo)) {
if (dateInfo.getDateMode().equals(DateMode.LIST_DISCRETE)) {
return dateUtils.listDateStr(dateDate, dateInfo);
}
if (dateInfo.getDateMode().equals(DateMode.BETWEEN_CONTINUOUS)) {
return dateUtils.betweenDateStr(dateDate, dateInfo);
}
if (dateUtils.hasAvailableDataMode(dateInfo)) {
return dateUtils.hasDataModeStr(dateDate, dateInfo);
}
return dateUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
}
log.info("dateDate:{}", dateDate);

27
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/QueryUtils.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java
View File

@@ -1,20 +1,19 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.constant.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.constant.Constants.UNIONALL;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.api.core.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -102,7 +101,7 @@ public class QueryUtils {
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns,
QueryMultiStructReq queryMultiStructCmd) {
List<Aggregator> aggregators = queryMultiStructCmd.getQueryStructCmds().stream()
List<Aggregator> aggregators = queryMultiStructCmd.getQueryStructReqs().stream()
.flatMap(queryStructCmd -> queryStructCmd.getAggregators().stream())
.collect(Collectors.toList());
log.info("multi agg merge:{}", aggregators);
@@ -174,7 +173,7 @@ public class QueryUtils {
QueryStatement sqlParser = new QueryStatement();
StringBuilder unionSqlBuilder = new StringBuilder();
for (int i = 0; i < sqlParsers.size(); i++) {
String selectStr = SqlGenerateUtils.getUnionSelect(queryMultiStructCmd.getQueryStructCmds().get(i));
String selectStr = SqlGenerateUtils.getUnionSelect(queryMultiStructCmd.getQueryStructReqs().get(i));
unionSqlBuilder.append(String.format("select %s from ( %s ) sub_sql_%s",
selectStr,
sqlParsers.get(i).getSql(), i));

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/SqlFilterUtils.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java
View File

@@ -1,14 +1,14 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.constant.Constants.PARENTHESES_END;
import static com.tencent.supersonic.common.constant.Constants.PARENTHESES_START;
import static com.tencent.supersonic.common.constant.Constants.SPACE;
import static com.tencent.supersonic.common.constant.Constants.SYS_VAR;
import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_END;
import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Criterion;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.Constants;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/SqlGenerateUtils.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlGenerateUtils.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.semantic.api.core.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Aggregator;
import java.util.stream.Collectors;
@@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
@Component
@Slf4j
@@ -17,6 +18,9 @@ public class SqlGenerateUtils {
StringBuilder sb = new StringBuilder();
int locate = 0;
for (String group : queryStructCmd.getGroups()) {
if (group.contains(JOIN_UNDERLINE)) {
group = group.split(JOIN_UNDERLINE)[1];
}
if (!TimeDimensionEnum.getNameList().contains(group)) {
locate++;
sb.append(group).append(" as ").append("name").append(locate).append(",");
@@ -36,7 +40,7 @@ public class SqlGenerateUtils {
public String getLimit(QueryStructReq queryStructCmd) {
if (queryStructCmd.getLimit() > 0) {
return " limit " + String.valueOf(queryStructCmd.getLimit());
return " limit " + queryStructCmd.getLimit();
}
return "";
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/StatUtils.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java
View File

@@ -1,17 +1,17 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.utils;
import com.alibaba.ttl.TransmittableThreadLocal;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.core.enums.QueryTypeBackEnum;
import com.tencent.supersonic.semantic.api.core.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.core.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeBackEnum;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.common.enums.TaskStatusEnum;
import com.tencent.supersonic.semantic.query.domain.repository.StatRepository;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.semantic.query.persistence.repository.StatRepository;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

6
semantic/query/src/main/resources/mapper/StatMapper.xml
View File

@@ -2,10 +2,10 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.tencent.supersonic.semantic.query.infrastructure.mapper.StatMapper">
<mapper namespace="com.tencent.supersonic.semantic.query.persistence.mapper.StatMapper">
<resultMap id="QueryStatDO"
type="com.tencent.supersonic.semantic.api.core.pojo.QueryStat">
type="com.tencent.supersonic.semantic.api.model.pojo.QueryStat">
<id column="id" property="id"/>
<result column="trace_id" property="traceId"/>
<result column="domain_id" property="domainId"/>
@@ -54,7 +54,7 @@
</insert>
<select id="getStatInfo"
resultType="com.tencent.supersonic.semantic.api.core.pojo.QueryStat">
resultType="com.tencent.supersonic.semantic.api.model.pojo.QueryStat">
select *
from s2_query_stat_info
<where>

22
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java
View File

@@ -1,18 +1,18 @@
package com.tencent.supersonic.semantic.query.domain.calcite;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.application.parser.calcite.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.application.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;