topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Update server and core, server calls core one-way (#592)

Browse Source 
* (improvement)(headless) Update server and core, server calls core one-way
* (improvement)(Auth) When obtaining the user information, determine whether the user is a system admin.
---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-01-04 12:08:12 +08:00
committed by GitHub
parent 7acb48da0e
commit 0858c13365
100 changed files with 811 additions and 1516 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatasourceQueryEnum.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/DatasourceQuery.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.server.pojo;
package com.tencent.supersonic.headless.api.enums;
public enum DatasourceQueryEnum {
public enum DatasourceQuery {
SQL_QUERY("sql_query"),
TABLE_QUERY("table_query");
@@ -9,7 +9,7 @@ public enum DatasourceQueryEnum {
private String name;
DatasourceQueryEnum(String name) {
DatasourceQuery(String name) {
this.name = name;
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/EngineTypeEnum.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/EngineType.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.server.pojo;
package com.tencent.supersonic.headless.api.enums;
public enum EngineTypeEnum {
public enum EngineType {
TDW(0, "tdw"),
MYSQL(1, "mysql"),
@@ -15,7 +15,7 @@ public enum EngineTypeEnum {
private String name;
EngineTypeEnum(Integer code, String name) {
EngineType(Integer code, String name) {
this.code = code;
this.name = name;
}

5
headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java
View File

@@ -7,6 +7,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Data
@ToString
@@ -27,4 +28,8 @@ public class QueryS2SQLReq {
return Lists.newArrayList(modelIds);
}
public String getModelIdStr() {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
}

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java
View File

@@ -82,6 +82,10 @@ public class QueryStructReq {
return Lists.newArrayList(modelIds);
}
public String getModelIdStr() {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
public Set<Long> getModelIdSet() {
return modelIds;
}

32
headless/core/pom.xml
View File

@@ -30,12 +30,6 @@
<artifactId>spring-context</artifactId>
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>headless-api</artifactId>
@@ -60,12 +54,6 @@
<version>${jetty.util.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>headless-server</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
@@ -87,26 +75,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>${easyexcel.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml-schemas</artifactId>
<version>${poi.version}</version>
</dependency>
</dependencies>

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/ClickHouseAdaptor.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/ClickHouseAdaptor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.adaptor.db;
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptor.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.adaptor.db;
package com.tencent.supersonic.headless.core.adaptor.db;
public abstract class DbAdaptor {

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.headless.api.enums.EngineType;
import java.util.HashMap;
import java.util.Map;
public class DbAdaptorFactory {
private static Map<String, DbAdaptor> dbAdaptorMap;
static {
dbAdaptorMap = new HashMap<>();
dbAdaptorMap.put(EngineType.CLICKHOUSE.getName(), new ClickHouseAdaptor());
dbAdaptorMap.put(EngineType.MYSQL.getName(), new MysqlAdaptor());
dbAdaptorMap.put(EngineType.H2.getName(), new H2Adaptor());
}
public static DbAdaptor getEngineAdaptor(String engineType) {
return dbAdaptorMap.get(engineType);
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/H2Adaptor.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.adaptor.db;
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/MysqlAdaptor.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/MysqlAdaptor.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.adaptor.db;
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.Constants;

9
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java
View File

@@ -2,9 +2,8 @@ package com.tencent.supersonic.headless.core.executor;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.SqlUtils;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
@@ -25,13 +24,13 @@ public class JdbcExecutor implements QueryExecutor {
}
@Override
public QueryResultWithSchemaResp execute(Catalog catalog, QueryStatement queryStatement) {
public QueryResultWithSchemaResp execute(QueryStatement queryStatement) {
if (Strings.isEmpty(queryStatement.getSourceId())) {
log.warn("data base id is empty");
return null;
}
log.info("query SQL: {}", queryStatement.getSql());
DatabaseResp databaseResp = catalog.getDatabase(Long.parseLong(queryStatement.getSourceId()));
DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp();
log.info("database info:{}", databaseResp);
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp);

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java
View File

@@ -1,12 +1,11 @@
package com.tencent.supersonic.headless.core.executor;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
public interface QueryExecutor {
boolean accept(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(Catalog catalog, QueryStatement queryStatement);
QueryResultWithSchemaResp execute(QueryStatement queryStatement);
}

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/DimensionYamlManager.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.headless.server.manager;
package com.tencent.supersonic.headless.core.manager;
import com.tencent.supersonic.headless.api.enums.IdentifyType;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -24,7 +24,15 @@ public class DimensionYamlManager {
}
return dimensions.stream()
.filter(dimension -> !dimension.getType().equalsIgnoreCase(IdentifyType.primary.name()))
.map(DimensionConverter::convert2DimensionYamlTpl).collect(Collectors.toList());
.map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList());
}
public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) {
DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl();
BeanUtils.copyProperties(dimension, dimensionYamlTpl);
dimensionYamlTpl.setName(dimension.getBizName());
dimensionYamlTpl.setOwners(dimension.getCreatedBy());
return dimensionYamlTpl;
}
}

54
headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java Normal file
View File

@@ -0,0 +1,54 @@
package com.tencent.supersonic.headless.core.manager;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
@Service
public class MetricYamlManager {
public static List<MetricYamlTpl> convert2YamlObj(List<MetricResp> metrics) {
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
for (MetricResp metric : metrics) {
MetricYamlTpl metricYamlTpl = convert2MetricYamlTpl(metric);
metricYamlTpls.add(metricYamlTpl);
}
return metricYamlTpls;
}
public static MetricYamlTpl convert2MetricYamlTpl(MetricResp metric) {
MetricYamlTpl metricYamlTpl = new MetricYamlTpl();
BeanUtils.copyProperties(metric, metricYamlTpl);
metricYamlTpl.setName(metric.getBizName());
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
MetricTypeParams exprMetricTypeParams = metric.getTypeParams();
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
metricTypeParamsYamlTpl.setExpr(exprMetricTypeParams.getExpr());
List<Measure> measures = exprMetricTypeParams.getMeasures();
metricTypeParamsYamlTpl.setMeasures(
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl);
return metricYamlTpl;
}
public static MeasureYamlTpl convert(Measure measure) {
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
measureYamlTpl.setName(measure.getBizName());
return measureYamlTpl;
}
}

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.server.manager;
package com.tencent.supersonic.headless.core.manager;
import com.tencent.supersonic.headless.api.enums.DatasourceQuery;
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Identify;
@@ -7,15 +8,14 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.server.pojo.DatasourceQueryEnum;
import com.tencent.supersonic.headless.server.utils.SysTimeDimensionBuilder;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@@ -45,7 +45,7 @@ public class ModelYamlManager {
dataModelYamlTpl.setName(modelResp.getBizName());
dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
dataModelYamlTpl.setModelSourceTypeEnum(ModelSourceType.of(modelResp.getSourceType()));
if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQueryEnum.SQL_QUERY.getName())) {
if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) {
dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery());
} else {
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java
View File

@@ -2,14 +2,14 @@ package com.tencent.supersonic.headless.core.optimizer;
import com.google.common.base.Strings;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import java.util.Objects;
import java.util.stream.Collectors;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.stream.Collectors;
@Slf4j
@Component("DetailQuery")
public class DetailQuery implements QueryOptimizer {

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.optimizer;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
public interface QueryOptimizer {
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/HeadlessConverter.java
View File

@@ -1,12 +1,11 @@
package com.tencent.supersonic.headless.core.parser;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
public interface HeadlessConverter {
boolean accept(QueryStatement queryStatement);
void converter(Catalog catalog, QueryStatement queryStatement) throws Exception;
void convert(QueryStatement queryStatement) throws Exception;
}

44
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java
View File

@@ -6,9 +6,8 @@ import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
@@ -25,13 +24,6 @@ import java.util.stream.Collectors;
@Primary
public class QueryParser {
private final Catalog catalog;
public QueryParser(Catalog catalog) {
this.catalog = catalog;
}
public QueryStatement logicSql(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
if (Objects.isNull(queryStatement.getParseSqlReq())) {
@@ -44,7 +36,7 @@ public class QueryParser {
for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) {
if (headlessConverter.accept(queryStatement)) {
log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.converter(catalog, queryStatement);
headlessConverter.convert(queryStatement);
}
}
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(),
@@ -58,24 +50,26 @@ public class QueryParser {
}
public QueryStatement parser(ParseSqlReq sqlCommend, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", sqlCommend);
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", parseSqlReq);
try {
if (!CollectionUtils.isEmpty(sqlCommend.getTables())) {
if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) {
List<String[]> tables = new ArrayList<>();
String sourceId = "";
for (MetricTable metricTable : sqlCommend.getTables()) {
for (MetricTable metricTable : parseSqlReq.getTables()) {
MetricQueryReq metricReq = new MetricQueryReq();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
metricReq.setRootPath(sqlCommend.getRootPath());
metricReq.setRootPath(parseSqlReq.getRootPath());
QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false);
tableSql.setMetricReq(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setModelIds(queryStatement.getModelIds());
tableSql.setHeadlessModel(queryStatement.getHeadlessModel());
tableSql = parser(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {
queryStatement.setErrMsg(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
@@ -88,20 +82,20 @@ public class QueryParser {
if (!tables.isEmpty()) {
String sql = "";
if (sqlCommend.isSupportWith()) {
if (parseSqlReq.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + sqlCommend.getSql();
Collectors.toList())) + "\n" + parseSqlReq.getSql();
} else {
sql = sqlCommend.getSql();
sql = parseSqlReq.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (sqlCommend.isWithAlias() ? "" : tb[0]), -1);
"(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setSourceId(sourceId);
queryStatement.setParseSqlReq(sqlCommend);
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
}
@@ -117,18 +111,18 @@ public class QueryParser {
}
public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryReq metricCommand = queryStatement.getMetricReq();
log.info("parser MetricReq [{}] isAgg [{}]", metricCommand, isAgg);
if (metricCommand.getRootPath().isEmpty()) {
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg);
if (metricQueryReq.getRootPath().isEmpty()) {
queryStatement.setErrMsg("rootPath empty");
return queryStatement;
}
try {
queryStatement = ComponentFactory.getSqlParser().explain(queryStatement, isAgg, catalog);
queryStatement = ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
return queryStatement;
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error MetricCommand[{}] error [{}]", metricCommand, e);
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
}
return queryStatement;
}

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/SqlParser.java
View File

@@ -1,10 +1,9 @@
package com.tencent.supersonic.headless.core.parser;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
public interface SqlParser {
QueryStatement explain(QueryStatement queryStatement, AggOption aggOption, Catalog catalog) throws Exception;
QueryStatement explain(QueryStatement queryStatement, AggOption aggOption) throws Exception;
}

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java
View File

@@ -7,24 +7,16 @@ import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import org.springframework.stereotype.Component;
@Component("CalciteSqlParser")
public class CalciteSqlParser implements SqlParser {
private final HeadlessSchemaManager headlessSchemaManager;
public CalciteSqlParser(
HeadlessSchemaManager headlessSchemaManager) {
this.headlessSchemaManager = headlessSchemaManager;
}
@Override
public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg, Catalog catalog) throws Exception {
public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception {
MetricQueryReq metricReq = queryStatement.getMetricReq();
HeadlessModel headlessModel = headlessSchemaManager.get(metricReq.getRootPath());
HeadlessModel headlessModel = queryStatement.getHeadlessModel();
if (headlessModel == null) {
queryStatement.setErrMsg("semanticSchema not found");
return queryStatement;

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java
View File

@@ -4,27 +4,20 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer;
import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer.FilterToGroupScanRule;
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.FilterRender;
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.OutputRender;
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.SourceRender;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Objects;
import java.util.Stack;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.hep.HepPlanner;
@@ -36,6 +29,13 @@ import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Objects;
import java.util.Stack;
@Slf4j
public class AggPlanner implements Planner {

338
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/MaterializationPlanner.java
View File

@@ -1,338 +0,0 @@
package com.tencent.supersonic.headless.core.parser.calcite.planner;
import com.tencent.supersonic.common.util.calcite.SqlParseUtils;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MaterializationElement;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.TimeRange;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.adapter.enumerable.EnumerableRules;
import org.apache.calcite.config.CalciteConnectionConfigImpl;
import org.apache.calcite.jdbc.CalciteSchema;
import org.apache.calcite.plan.ConventionTraitDef;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptMaterialization;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.prepare.CalciteCatalogReader;
import org.apache.calcite.rel.RelDistributionTraitDef;
import org.apache.calcite.rel.RelHomogeneousShuttle;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelShuttle;
import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.rules.materialize.MaterializedViewRules;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.tools.RelBuilder;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.springframework.util.CollectionUtils;
@Slf4j
public class MaterializationPlanner implements Planner {
protected HeadlessSchema schema;
protected CalciteSchema viewSchema;
protected HepProgramBuilder hepProgramBuilder;
protected RelOptPlanner relOptPlanner;
protected RelBuilder relBuilder;
protected CalciteCatalogReader calciteCatalogReader;
protected Comparator materializationSort = new Comparator<Entry<Long, Set<String>>>() {
@Override
public int compare(Entry<Long, Set<String>> o1, Entry<Long, Set<String>> o2) {
if (o1.getValue().size() == o2.getValue().size()) {
Optional<Materialization> o1Lever = schema.getMaterializationList().stream()
.filter(m -> m.getMaterializationId().equals(o1.getKey())).findFirst();
Optional<Materialization> o2Lever = schema.getMaterializationList().stream()
.filter(m -> m.getMaterializationId().equals(o2.getKey())).findFirst();
if (o1Lever.isPresent() && o2Lever.isPresent()) {
return o2Lever.get().getLevel() - o1Lever.get().getLevel();
}
return 0;
}
return o2.getValue().size() - o1.getValue().size();
}
};
public MaterializationPlanner(HeadlessSchema schema) {
this.schema = schema;
init();
}
@Override
public void explain(QueryStatement queryStatement, AggOption isAgg) throws Exception {
// findMatchMaterialization
// checkValid field + time
if (CollectionUtils.isEmpty(queryStatement.getTimeRanges())) {
//has no matchMaterialization time info
return;
}
Set<String> fields = new HashSet<>();
MetricQueryReq metricCommand = queryStatement.getMetricReq();
if (!Objects.isNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
fields.addAll(SqlParseUtils.getFilterField(metricCommand.getWhere()));
}
if (!CollectionUtils.isEmpty(metricCommand.getMetrics())) {
fields.addAll(metricCommand.getMetrics());
}
if (!CollectionUtils.isEmpty(metricCommand.getDimensions())) {
fields.addAll(metricCommand.getDimensions());
}
Map<Long, Set<String>> matchMaterialization = new HashMap<>();
Map<Long, Long> materializationDataBase = schema.getMaterializationList().stream()
.collect(Collectors.toMap(Materialization::getMaterializationId, Materialization::getDataBase));
for (String elem : fields) {
boolean checkOk = false;
for (Materialization materialization : schema.getMaterializationList()) {
if (check(metricCommand, materialization, elem, queryStatement.getTimeRanges())) {
if (!matchMaterialization.containsKey(materialization.getMaterializationId())) {
matchMaterialization.put(materialization.getMaterializationId(), new HashSet<>());
}
matchMaterialization.get(materialization.getMaterializationId()).add(elem);
checkOk = true;
}
}
if (!checkOk) {
log.info("check fail [{}]", elem);
}
}
if (!CollectionUtils.isEmpty(matchMaterialization)) {
List<Entry<Long, Set<String>>> sortedMaterialization = new ArrayList<>(matchMaterialization.entrySet());
sortedMaterialization.stream().collect(Collectors.toList()).sort(materializationSort);
for (Entry<Long, Set<String>> m : sortedMaterialization) {
Optional<Materialization> materialization = schema.getMaterializationList().stream()
.filter(mz -> mz.getMaterializationId().equals(m.getKey())).findFirst();
if (!materialization.isPresent()) {
continue;
}
Set<String> viewField = new HashSet<>(m.getValue());
viewField.add(materialization.get().getEntities());
viewField.add(materialization.get().getDateInfo());
if (materialization.get().getTimePartType().equals(TimePartType.ZIPPER)) {
viewField.add(Constants.MATERIALIZATION_ZIPPER_START + materialization.get().getDateInfo());
viewField.add(Constants.MATERIALIZATION_ZIPPER_END + materialization.get().getDateInfo());
}
if (viewField.containsAll(fields)) {
addDataSource(materialization.get());
break;
}
List<Entry<Long, Set<String>>> linkMaterialization = new ArrayList<>();
for (Entry<Long, Set<String>> mm : sortedMaterialization) {
if (mm.getKey().equals(m.getKey())) {
continue;
}
if (materializationDataBase.get(mm.getKey()).equals(materializationDataBase.get(m.getKey()))) {
linkMaterialization.add(mm);
}
}
if (!CollectionUtils.isEmpty(linkMaterialization)) {
linkMaterialization.sort(materializationSort);
for (Entry<Long, Set<String>> mm : linkMaterialization) {
Set<String> linkField = new HashSet<>(mm.getValue());
linkField.addAll(viewField);
if (linkField.containsAll(fields)) {
Optional<Materialization> linkMaterial = schema.getMaterializationList().stream()
.filter(mz -> mz.getMaterializationId().equals(mm.getKey())).findFirst();
if (linkMaterial.isPresent()) {
addDataSource(materialization.get());
addDataSource(linkMaterial.get());
break;
}
}
}
}
}
}
}
private void addDataSource(Materialization materialization) {
Identify identify = new Identify();
identify.setName(materialization.getEntities());
List<Measure> metrics = materialization.getMetrics().stream()
.map(m -> Measure.builder().name(m.getName()).expr(m.getName()).build()).collect(
Collectors.toList());
List<Dimension> dimensions = materialization.getDimensions().stream()
.map(d -> Dimension.builder().name(d.getName()).expr(d.getName()).build()).collect(
Collectors.toList());
if (materialization.getTimePartType().equals(TimePartType.ZIPPER)) {
dimensions.add(
Dimension.builder().name(Constants.MATERIALIZATION_ZIPPER_START + materialization.getDateInfo())
.type(Constants.DIMENSION_TYPE_TIME)
.expr(Constants.MATERIALIZATION_ZIPPER_START + materialization.getDateInfo()).build());
dimensions.add(
Dimension.builder().name(Constants.MATERIALIZATION_ZIPPER_END + materialization.getDateInfo())
.type(Constants.DIMENSION_TYPE_TIME)
.expr(Constants.MATERIALIZATION_ZIPPER_END + materialization.getDateInfo()).build());
} else {
dimensions.add(Dimension.builder().name(materialization.getDateInfo()).expr(materialization.getDateInfo())
.type(Constants.DIMENSION_TYPE_TIME)
.build());
}
DataSource dataSource = DataSource.builder().sourceId(materialization.getDataBase())
.tableQuery(materialization.getDestinationTable())
.timePartType(materialization.getTimePartType())
.name("v_" + String.valueOf(materialization.getMaterializationId()))
.identifiers(Arrays.asList(identify))
.measures(metrics)
.dimensions(dimensions)
.build();
schema.getDatasource().put(dataSource.getName(), dataSource);
}
@Override
public String getSql() {
return null;
}
@Override
public String getSourceId() {
return null;
}
@Override
public HeadlessSchema findBest() {
return schema;
}
private void init() {
viewSchema = SchemaBuilder.getMaterializationSchema();
hepProgramBuilder = new HepProgramBuilder();
hepProgramBuilder.addRuleInstance(MaterializedViewRules.PROJECT_FILTER);
relOptPlanner = new HepPlanner(hepProgramBuilder.build());
calciteCatalogReader = new CalciteCatalogReader(
CalciteSchema.from(viewSchema.plus()),
CalciteSchema.from(viewSchema.plus()).path(null),
Configuration.typeFactory,
new CalciteConnectionConfigImpl(new Properties()));
relOptPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE);
relOptPlanner.addRelTraitDef(RelDistributionTraitDef.INSTANCE);
EnumerableRules.rules().forEach(relOptPlanner::addRule);
RexBuilder rexBuilder = new RexBuilder(Configuration.typeFactory);
RelOptCluster relOptCluster = RelOptCluster.create(relOptPlanner, rexBuilder);
relBuilder = RelFactories.LOGICAL_BUILDER.create(relOptCluster, calciteCatalogReader);
}
private RexNode getRexNode(List<ImmutablePair<String, String>> timeRanges, String viewField) {
RexNode rexNode = null;
for (ImmutablePair<String, String> timeRange : timeRanges) {
if (rexNode == null) {
rexNode = getRexNodeByTimeRange(TimeRange.builder().start(timeRange.left).end(timeRange.right).build(),
viewField);
continue;
}
rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode,
getRexNodeByTimeRange(TimeRange.builder().start(timeRange.left).end(timeRange.right).build(),
viewField));
}
return rexNode;
}
private RexNode getRexNode(Materialization materialization, String elem, String viewField) {
Optional<MaterializationElement> dim = materialization.getDimensions()
.stream().filter(d -> d.getName().equalsIgnoreCase(elem)).findFirst();
if (!dim.isPresent()) {
dim = materialization.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(elem)).findFirst();
}
RexNode rexNode = null;
if (dim.isPresent()) {
for (TimeRange timeRange : dim.get().getTimeRangeList()) {
if (rexNode == null) {
rexNode = getRexNodeByTimeRange(timeRange, viewField);
continue;
}
rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode, getRexNodeByTimeRange(timeRange, viewField));
}
}
return rexNode;
}
private RexNode getRexNodeByTimeRange(TimeRange timeRange, String field) {
return relBuilder.call(SqlStdOperatorTable.AND,
relBuilder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, relBuilder.field(field),
relBuilder.literal(timeRange.getStart())),
relBuilder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(field),
relBuilder.literal(timeRange.getEnd())));
}
public boolean check(MetricQueryReq metricCommand, Materialization materialization, String elem,
List<ImmutablePair<String, String>> timeRanges)
throws SqlParseException {
boolean isMatch = false;
try {
relBuilder.clear();
if (!CollectionUtils.isEmpty(relOptPlanner.getMaterializations())) {
relOptPlanner.clear();
}
String db = SchemaBuilder.MATERIALIZATION_SYS_DB;
RelBuilder viewBuilder = relBuilder.scan(Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_SOURCE));
RexNode viewFilter = getRexNode(materialization, elem, SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE);
if (viewFilter == null) {
return false;
}
RelNode viewRel = viewBuilder.filter(viewFilter).project(relBuilder.fields()).build();
log.debug("view {}", viewRel.explain());
List<String> view = Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_VIEW);
RelNode replacement = relBuilder.scan(view).build();
RelOptMaterialization relOptMaterialization = new RelOptMaterialization(replacement, viewRel, null, view);
relOptPlanner.addMaterialization(relOptMaterialization);
RelNode checkRel = relBuilder.scan(Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_SOURCE))
.filter(getRexNode(timeRanges, SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE))
.project(relBuilder.field(SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE)).build();
relOptPlanner.setRoot(checkRel);
RelNode optRel = relOptPlanner.findBestExp();
log.debug("findBestExp {}", optRel.explain());
isMatch = !extractTableNames(optRel).contains(SchemaBuilder.MATERIALIZATION_SYS_SOURCE);
} catch (Exception e) {
log.error("check error {}", e);
}
return isMatch;
}
public static Set<String> extractTableNames(RelNode relNode) {
Set<String> tableNames = new HashSet<>();
RelShuttle shuttle = new RelHomogeneousShuttle() {
public RelNode visit(TableScan scan) {
RelOptTable table = scan.getTable();
tableNames.addAll(table.getQualifiedName());
return scan;
}
};
relNode.accept(shuttle);
return tableNames;
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/Planner.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
public interface Planner {

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/DataSource.java
View File

@@ -1,14 +1,17 @@
package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
import java.util.List;
import lombok.Builder;
import lombok.Data;
import java.util.List;
@Data
@Builder
public class DataSource {
private Long id;
private String name;
private Long sourceId;

18
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/HeadlessModel.java
View File

@@ -1,10 +1,15 @@
package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import lombok.Data;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.Data;
import java.util.stream.Collectors;
@Data
public class HeadlessModel {
@@ -15,4 +20,15 @@ public class HeadlessModel {
private Map<String, List<Dimension>> dimensionMap = new HashMap<>();
private List<Materialization> materializationList = new ArrayList<>();
private List<JoinRelation> joinRelations;
private ItemDateResp dataDate;
private DatabaseResp databaseResp;
public List<Dimension> getDimensions() {
return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
}
public Map<Long, DataSource> getModelMap() {
return datasourceMap.values().stream().collect(Collectors.toMap(DataSource::getId, dataSource -> dataSource));
}
}

59
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/DefaultDimValueConverter.java
View File

@@ -1,59 +0,0 @@
package com.tencent.supersonic.headless.core.parser.convert;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Slf4j
@Component("DefaultDimValueConverter")
public class DefaultDimValueConverter implements HeadlessConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
return false;
}
return true;
}
@Override
public void converter(Catalog catalog, QueryStatement queryStatement) {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds());
List<DimensionResp> dimensionResps = catalog.getDimensions(metaFilter);
//dimension which has default values
dimensionResps = dimensionResps.stream()
.filter(dimensionResp -> !CollectionUtils.isEmpty(dimensionResp.getDefaultValues()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(dimensionResps)) {
return;
}
log.info("dimension with default values:{}, queryStruct:{}", dimensionResps, queryStructCmd);
//add dimension default value to filter
List<String> dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream()
.map(Filter::getBizName).collect(Collectors.toList());
for (DimensionResp dimensionResp : dimensionResps) {
if (!dimensionFilterBizName.contains(dimensionResp.getBizName())) {
Filter filter = new Filter();
filter.setBizName(dimensionResp.getBizName());
filter.setValue(dimensionResp.getDefaultValues());
filter.setOperator(FilterOperatorEnum.IN);
filter.setName(dimensionResp.getName());
queryStructCmd.getDimensionFilters().add(filter);
}
}
}
}

86
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/MetricCheckConverter.java
View File

@@ -1,86 +0,0 @@
package com.tencent.supersonic.headless.core.parser.convert;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Component("MetricCheckConverter")
@Slf4j
public class MetricCheckConverter implements HeadlessConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
return false;
}
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
return false;
}
return !CollectionUtils.isEmpty(queryStructCmd.getAggregators());
}
@Override
public void converter(Catalog catalog, QueryStatement queryStatement) {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
MetaFilter metaFilter = new MetaFilter(queryStructReq.getModelIds());
List<MetricResp> metricResps = catalog.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = catalog.getDimensions(metaFilter);
Map<Long, DimensionResp> dimensionMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getId, d -> d));
List<String> metricBizNames = queryStructReq.getMetrics();
List<String> dimensionFilterBizNames = queryStructReq.getDimensionFilters().stream()
.map(Filter::getBizName).collect(Collectors.toList());
List<MetricResp> metricToQuery = metricResps.stream().filter(metricResp ->
metricBizNames.contains(metricResp.getBizName())).collect(Collectors.toList());
List<Long> dimensionToFilter = dimensionResps.stream().filter(dimensionResp ->
dimensionFilterBizNames.contains(dimensionResp.getBizName()))
.map(DimensionResp::getId).collect(Collectors.toList());
for (MetricResp metricResp : metricToQuery) {
Set<Long> necessaryDimensionIds = metricResp.getNecessaryDimensionIds();
if (CollectionUtils.isEmpty(necessaryDimensionIds)) {
continue;
}
DimensionResp dimensionResp = null;
for (Long dimensionId : necessaryDimensionIds) {
dimensionResp = dimensionMap.get(dimensionId);
if (dimensionResp != null) {
break;
}
}
if (dimensionResp == null) {
continue;
}
String message = String.format("该指标必须配合维度[%s]来进行过滤查询", dimensionResp.getName());
if (CollectionUtils.isEmpty(dimensionToFilter)) {
throw new InvalidArgumentException(message);
}
boolean flag = false;
for (Long dimensionId : dimensionToFilter) {
if (necessaryDimensionIds.contains(dimensionId)) {
flag = true;
break;
}
}
if (!flag) {
throw new InvalidArgumentException(message);
}
}
}
}

80
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ParserDefaultConverter.java
View File

@@ -1,80 +0,0 @@
package com.tencent.supersonic.headless.core.parser.convert;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.Param;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.stream.Collectors;
@Component("ParserDefaultConverter")
@Slf4j
public class ParserDefaultConverter implements HeadlessConverter {
private final CalculateAggConverter calculateCoverterAgg;
private final QueryStructUtils queryStructUtils;
public ParserDefaultConverter(
CalculateAggConverter calculateCoverterAgg,
QueryStructUtils queryStructUtils) {
this.calculateCoverterAgg = calculateCoverterAgg;
this.queryStructUtils = queryStructUtils;
}
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
return false;
}
return !calculateCoverterAgg.accept(queryStatement);
}
@Override
public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
MetricQueryReq metricCommand = queryStatement.getMetricReq();
MetricQueryReq metricReq = generateSqlCommand(catalog, queryStructCmd);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructCmd));
BeanUtils.copyProperties(metricReq, metricCommand);
}
public MetricQueryReq generateSqlCommand(Catalog catalog, QueryStructReq queryStructCmd) {
MetricQueryReq sqlCommend = new MetricQueryReq();
sqlCommend.setMetrics(queryStructCmd.getMetrics());
sqlCommend.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
log.info("in generateSqlCommend, complete where:{}", where);
sqlCommend.setWhere(where);
sqlCommend.setOrder(queryStructCmd.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
sqlCommend.setVariables(queryStructCmd.getParams().stream()
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
sqlCommend.setLimit(queryStructCmd.getLimit());
String rootPath = catalog.getModelFullPath(queryStructCmd.getModelIds());
sqlCommend.setRootPath(rootPath);
// todo tmp delete
// support detail query
if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
for (Long modelId : queryStructCmd.getModelIds()) {
String internalMetricName = queryStructUtils.generateInternalMetricName(
modelId, queryStructCmd.getGroups());
sqlCommend.getMetrics().add(internalMetricName);
}
}
return sqlCommend;
}
}

75
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/ZipperModelConverter.java
View File

@@ -1,75 +0,0 @@
package com.tencent.supersonic.headless.core.parser.convert;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Objects;
/**
* correct the Query parameters when the model source type is zipper
*/
@Component("ZipperModelConverter")
@Slf4j
public class ZipperModelConverter implements HeadlessConverter {
private final QueryStructUtils queryStructUtils;
private final Catalog catalog;
public ZipperModelConverter(QueryStructUtils queryStructUtils,
Catalog catalog) {
this.queryStructUtils = queryStructUtils;
this.catalog = catalog;
}
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq())) {
return false;
}
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
List<ModelResp> modelRespList = catalog.getModelList(queryStructCmd.getModelIds());
if (!CollectionUtils.isEmpty(modelRespList)) {
// all data sources are zipper tables
long zipperCnt = modelRespList.stream().filter(m -> ModelSourceType.isZipper(m.getSourceType()))
.count();
return modelRespList.size() == zipperCnt;
}
return false;
}
@Override
public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq();
MetricQueryReq metricCommand = queryStatement.getMetricReq();
doSingleZipperSource(queryStructCmd, sqlCommend, metricCommand);
}
protected void doSingleZipperSource(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricQueryReq metricCommand) {
// all data sources are zipper tables
// request time field rewrite to start_ end_
if (!sqlCommend.getSql().isEmpty()) {
String sqlNew = queryStructUtils.generateZipperWhere(queryStructCmd, sqlCommend);
log.info("doSingleZipperSource before[{}] after[{}]", sqlCommend.getSql(), sqlNew);
sqlCommend.setSql(sqlNew);
} else {
String where = queryStructUtils.generateZipperWhere(queryStructCmd);
if (!where.isEmpty() && Objects.nonNull(metricCommand)) {
log.info("doSingleZipperSource before[{}] after[{}]", metricCommand.getWhere(), where);
metricCommand.setWhere(where);
}
}
}
}

89
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/CalculateAggConverter.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
View File

@@ -1,5 +1,4 @@
package com.tencent.supersonic.headless.core.parser.convert;
package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
@@ -7,20 +6,15 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -35,24 +29,11 @@ import java.util.stream.Collectors;
@Slf4j
public class CalculateAggConverter implements HeadlessConverter {
private final HeadlessQueryEngine parserService;
private final QueryStructUtils queryStructUtils;
private final SqlGenerateUtils sqlGenerateUtils;
private final Catalog catalog;
@Value("${metricParser.agg.default:sum}")
private String metricAggDefault;
public CalculateAggConverter(
HeadlessQueryEngine parserService,
@Lazy QueryStructUtils queryStructUtils,
SqlGenerateUtils sqlGenerateUtils, Catalog catalog) {
this.parserService = parserService;
this.queryStructUtils = queryStructUtils;
public CalculateAggConverter(SqlGenerateUtils sqlGenerateUtils) {
this.sqlGenerateUtils = sqlGenerateUtils;
this.catalog = catalog;
}
public interface EngineSql {
@@ -60,34 +41,36 @@ public class CalculateAggConverter implements HeadlessConverter {
String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql);
}
public ParseSqlReq generateSqlCommend(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum, String version)
public ParseSqlReq generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version)
throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
// 同环比
if (isRatioAccept(queryStructCmd)) {
return generateRatioSqlCommand(queryStructCmd, engineTypeEnum, version);
if (isRatioAccept(queryStructReq)) {
return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
}
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds()));
sqlCommand.setRootPath(queryStructReq.getModelIdStr());
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructCmd.getMetrics());
metricTable.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq,
queryStatement.getHeadlessModel().getDataDate());
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructReq),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
sqlGenerateUtils.getLimit(queryStructCmd));
if (!queryStructUtils.isSupportWith(engineTypeEnum, version)) {
sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq),
sqlGenerateUtils.getLimit(queryStructReq));
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false);
sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructReq),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
sqlGenerateUtils.getLimit(queryStructCmd));
sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq),
sqlGenerateUtils.getLimit(queryStructReq));
}
sqlCommand.setSql(sql);
return sqlCommand;
@@ -122,12 +105,11 @@ public class CalculateAggConverter implements HeadlessConverter {
}
@Override
public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
public void convert(QueryStatement queryStatement) throws Exception {
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq();
DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelIds().get(0));
ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd,
EngineTypeEnum.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp();
ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement,
EngineType.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
sqlCommend.setSql(parseSqlReq.getSql());
sqlCommend.setTables(parseSqlReq.getTables());
sqlCommend.setRootPath(parseSqlReq.getRootPath());
@@ -149,38 +131,39 @@ public class CalculateAggConverter implements HeadlessConverter {
return false;
}
public ParseSqlReq generateRatioSqlCommand(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum,
public ParseSqlReq generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
String version)
throws Exception {
check(queryStructCmd);
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
check(queryStructReq);
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds()));
sqlCommand.setRootPath(queryStructReq.getModelIdStr());
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructCmd.getMetrics());
metricTable.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
metricTable.setMetrics(queryStructReq.getMetrics());
metricTable.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getHeadlessModel().getDataDate());
log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
boolean isOver = isOverRatio(queryStructCmd);
boolean isOver = isOverRatio(queryStructReq);
String sql = "";
switch (engineTypeEnum) {
case H2:
sql = new H2EngineSql().sql(queryStructCmd, isOver, true, metricTableName);
sql = new H2EngineSql().sql(queryStructReq, isOver, true, metricTableName);
break;
case MYSQL:
case DORIS:
case CLICKHOUSE:
if (!queryStructUtils.isSupportWith(engineTypeEnum, version)) {
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false);
}
if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) {
sql = new MysqlEngineSql().sql(queryStructCmd, isOver, sqlCommand.isSupportWith(), metricTableName);
sql = new MysqlEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName);
} else {
sql = new CkEngineSql().sql(queryStructCmd, isOver, sqlCommand.isSupportWith(), metricTableName);
sql = new CkEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName);
}
break;
default:

82
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java Normal file
View File

@@ -0,0 +1,82 @@
package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.Param;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@Component("ParserDefaultConverter")
@Slf4j
public class ParserDefaultConverter implements HeadlessConverter {
private final SqlGenerateUtils sqlGenerateUtils;
private final CalculateAggConverter calculateConverterAgg;
public ParserDefaultConverter(CalculateAggConverter calculateConverterAgg,
SqlGenerateUtils sqlGenerateUtils) {
this.calculateConverterAgg = calculateConverterAgg;
this.sqlGenerateUtils = sqlGenerateUtils;
}
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
return false;
}
return !calculateConverterAgg.accept(queryStatement);
}
@Override
public void convert(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq,
queryStatement.getHeadlessModel().getDataDate()));
BeanUtils.copyProperties(metricReq, metricQueryReq);
}
public MetricQueryReq generateSqlCommand(QueryStructReq queryStructReq, QueryStatement queryStatement) {
MetricQueryReq metricQueryReq = new MetricQueryReq();
metricQueryReq.setMetrics(queryStructReq.getMetrics());
metricQueryReq.setDimensions(queryStructReq.getGroups());
String where = sqlGenerateUtils.generateWhere(queryStructReq,
queryStatement.getHeadlessModel().getDataDate());
log.info("in generateSqlCommend, complete where:{}", where);
metricQueryReq.setWhere(where);
metricQueryReq.setOrder(queryStructReq.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
metricQueryReq.setVariables(queryStructReq.getParams().stream()
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
metricQueryReq.setLimit(queryStructReq.getLimit());
String rootPath = queryStructReq.getModelIdStr();
metricQueryReq.setRootPath(rootPath);
// support detail query
if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) {
Map<Long, DataSource> dataSourceMap = queryStatement.getHeadlessModel().getModelMap();
for (Long modelId : queryStructReq.getModelIds()) {
String modelBizName = dataSourceMap.get(modelId).getName();
String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName);
metricQueryReq.getMetrics().add(internalMetricName);
}
}
return metricQueryReq;
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ConnectInfo.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ConnectInfo.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo;
package com.tencent.supersonic.headless.core.pojo;
import lombok.Data;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/Database.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo;
package com.tencent.supersonic.headless.core.pojo;
import com.google.common.collect.Lists;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/JdbcDataSource.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo;
package com.tencent.supersonic.headless.core.pojo;
import com.alibaba.druid.filter.Filter;
import com.alibaba.druid.pool.DruidDataSource;
@@ -6,7 +6,7 @@ import com.alibaba.druid.wall.WallConfig;
import com.alibaba.druid.wall.WallFilter;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.server.utils.JdbcDataSourceUtils;
import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/ParserSvrResponse.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ParserSvrResponse.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.persistence.pojo;
package com.tencent.supersonic.headless.core.pojo;
public class ParserSvrResponse<T> {

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/QueryStatement.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -1,13 +1,14 @@
package com.tencent.supersonic.headless.core.persistence.pojo;
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
import lombok.Data;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import java.util.List;
import org.apache.commons.lang3.tuple.Triple;
@Data
public class QueryStatement {
@@ -26,6 +27,8 @@ public class QueryStatement {
private Boolean enableOptimize = true;
private Triple<String, String, String> minMaxTime;
private HeadlessModel headlessModel;
public boolean isOk() {
this.ok = "".equals(errMsg) && !"".equals(sql);
return ok;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
package com.tencent.supersonic.headless.core.pojo.yaml;
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
import lombok.Data;
@@ -9,6 +9,8 @@ import java.util.List;
@Data
public class DataModelYamlTpl {
private Long id;
private String name;
private Long sourceId;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionTimeTypeParamsTpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.Data;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
package com.tencent.supersonic.headless.core.pojo.yaml;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/IdentifyYamlTpl.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/IdentifyYamlTpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.AllArgsConstructor;
import lombok.Data;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MeasureYamlTpl.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MeasureYamlTpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.AllArgsConstructor;
import lombok.Data;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricTypeParamsYamlTpl.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.Data;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/MetricYamlTpl.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricYamlTpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.Data;

42
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java
View File

@@ -1,18 +1,15 @@
package com.tencent.supersonic.headless.core.utils;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.SqlParser;
import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser;
import com.tencent.supersonic.headless.core.parser.convert.DefaultDimValueConverter;
import com.tencent.supersonic.headless.core.parser.convert.ZipperModelConverter;
import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.optimizer.DetailQuery;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.convert.CalculateAggConverter;
import com.tencent.supersonic.headless.core.parser.convert.MetricCheckConverter;
import com.tencent.supersonic.headless.core.parser.convert.ParserDefaultConverter;
import com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter;
import com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -22,14 +19,14 @@ import java.util.stream.Collectors;
public class ComponentFactory {
private static List<HeadlessConverter> headlessConverters = new ArrayList<>();
private static List<QueryExecutor> queryExecutors = new ArrayList<>();
private static Map<String, QueryOptimizer> queryOptimizers = new HashMap<>();
private static List<QueryExecutor> queryExecutors = new ArrayList<>();
private static SqlParser sqlParser;
static {
initSemanticConverter();
initQueryExecutors();
initQueryOptimizer();
initQueryExecutors();
}
public static List<HeadlessConverter> getSemanticConverters() {
@@ -39,13 +36,6 @@ public class ComponentFactory {
return headlessConverters;
}
public static List<QueryExecutor> getQueryExecutors() {
if (queryExecutors.isEmpty()) {
initQueryExecutors();
}
return queryExecutors;
}
public static List<QueryOptimizer> getQueryOptimizers() {
if (queryOptimizers.isEmpty()) {
initQueryOptimizer();
@@ -53,6 +43,13 @@ public class ComponentFactory {
return queryOptimizers.values().stream().collect(Collectors.toList());
}
public static List<QueryExecutor> getQueryExecutors() {
if (queryExecutors.isEmpty()) {
initQueryExecutors();
}
return queryExecutors;
}
public static SqlParser getSqlParser() {
if (sqlParser == null) {
sqlParser = ContextUtils.getContext().getBean("CalciteSqlParser", CalciteSqlParser.class);
@@ -76,16 +73,13 @@ public class ComponentFactory {
queryOptimizers.put("DetailQuery", getBean("DetailQuery", DetailQuery.class));
}
private static void initSemanticConverter() {
headlessConverters.add(getBean("MetricCheckConverter", MetricCheckConverter.class));
headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
headlessConverters.add(getBean("ZipperModelConverter", ZipperModelConverter.class));
}
private static void initQueryExecutors() {
queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", JdbcExecutor.class));
}
private static void initSemanticConverter() {
headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
}
}

37
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/JdbcDataSourceUtils.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java
View File

@@ -1,4 +1,22 @@
package com.tencent.supersonic.headless.server.utils;
package com.tencent.supersonic.headless.core.utils;
import com.alibaba.druid.util.StringUtils;
import com.tencent.supersonic.common.util.MD5Util;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import static com.tencent.supersonic.common.pojo.Constants.COLON;
@@ -9,23 +27,6 @@ import static com.tencent.supersonic.common.pojo.Constants.NEW_LINE_CHAR;
import static com.tencent.supersonic.common.pojo.Constants.PATTERN_JDBC_TYPE;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import com.alibaba.druid.util.StringUtils;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.common.util.MD5Util;
import com.tencent.supersonic.headless.server.pojo.Database;
import com.tencent.supersonic.headless.server.pojo.JdbcDataSource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import javax.sql.DataSource;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class JdbcDataSourceUtils {

154
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -1,21 +1,61 @@
package com.tencent.supersonic.headless.core.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
@Component
@Slf4j
public class SqlGenerateUtils {
private final SqlFilterUtils sqlFilterUtils;
private final DateModeUtils dateModeUtils;
@Value("${metricParser.agg.mysql.lowVersion:5.7}")
private String mysqlLowVersion;
@Value("${metricParser.agg.ck.lowVersion:20.4}")
private String ckLowVersion;
@Value("${internal.metric.cnt.suffix:internal_cnt}")
private String internalMetricNameSuffix;
public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils,
DateModeUtils dateModeUtils) {
this.sqlFilterUtils = sqlFilterUtils;
this.dateModeUtils = dateModeUtils;
}
public static String getUnionSelect(QueryStructReq queryStructCmd) {
StringBuilder sb = new StringBuilder();
int locate = 0;
@@ -82,4 +122,112 @@ public class SqlGenerateUtils {
.collect(Collectors.joining(","));
}
public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) {
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter());
String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp);
return mergeDateWhereClause(queryStructReq, whereClauseFromFilter, whereFromDate);
}
private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter,
String whereFromDate) {
if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
} else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
return whereClauseFromFilter;
} else if (Strings.isNotEmpty(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
return whereFromDate;
} else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
log.info("the current date information is empty, enter the date initialization logic");
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
}
return whereClauseFromFilter;
}
private String getDateWhereClause(DateConf dateInfo, ItemDateResp dateDate) {
if (Objects.isNull(dateDate)
|| Strings.isEmpty(dateDate.getStartDate())
&& Strings.isEmpty(dateDate.getEndDate())) {
if (dateInfo.getDateMode().equals(DateConf.DateMode.LIST)) {
return dateModeUtils.listDateStr(dateDate, dateInfo);
}
if (dateInfo.getDateMode().equals(DateConf.DateMode.BETWEEN)) {
return dateModeUtils.betweenDateStr(dateDate, dateInfo);
}
if (dateModeUtils.hasAvailableDataMode(dateInfo)) {
return dateModeUtils.hasDataModeStr(dateDate, dateInfo);
}
return dateModeUtils.defaultRecentDateInfo(dateInfo);
}
log.info("dateDate:{}", dateDate);
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
}
public Triple<String, String, String> getBeginEndTime(QueryStructReq queryStructCmd, ItemDateResp dataDate) {
if (Objects.isNull(queryStructCmd.getDateInfo())) {
return Triple.of("", "", "");
}
DateConf dateConf = queryStructCmd.getDateInfo();
String dateInfo = dateModeUtils.getSysDateCol(dateConf);
if (dateInfo.isEmpty()) {
return Triple.of("", "", "");
}
switch (dateConf.getDateMode()) {
case AVAILABLE:
case BETWEEN:
return Triple.of(dateInfo, dateConf.getStartDate(), dateConf.getEndDate());
case LIST:
return Triple.of(dateInfo, Collections.min(dateConf.getDateList()),
Collections.max(dateConf.getDateList()));
case RECENT:
LocalDate dateMax = LocalDate.now().minusDays(1);
LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1);
if (Objects.isNull(dataDate)) {
return Triple.of(dateInfo, dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)),
dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT)));
}
switch (dateConf.getPeriod()) {
case DAY:
ImmutablePair<String, String> dayInfo = dateModeUtils.recentDay(dataDate, dateConf);
return Triple.of(dateInfo, dayInfo.left, dayInfo.right);
case WEEK:
ImmutablePair<String, String> weekInfo = dateModeUtils.recentWeek(dataDate, dateConf);
return Triple.of(dateInfo, weekInfo.left, weekInfo.right);
case MONTH:
List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dataDate, dateConf);
Optional<String> minBegins = rets.stream().map(i -> i.left).sorted().findFirst();
Optional<String> maxBegins = rets.stream().map(i -> i.right).sorted(Comparator.reverseOrder())
.findFirst();
if (minBegins.isPresent() && maxBegins.isPresent()) {
return Triple.of(dateInfo, minBegins.get(), maxBegins.get());
}
break;
default:
break;
}
break;
default:
break;
}
return Triple.of("", "", "");
}
public boolean isSupportWith(EngineType engineTypeEnum, String version) {
if (engineTypeEnum.equals(EngineType.MYSQL) && Objects.nonNull(version) && version.startsWith(
mysqlLowVersion)) {
return false;
}
if (engineTypeEnum.equals(EngineType.CLICKHOUSE) && Objects.nonNull(version)
&& StringUtil.compareVersion(version,
ckLowVersion) < 0) {
return false;
}
return true;
}
public String generateInternalMetricName(String modelBizName) {
return modelBizName + UNDERLINE + internalMetricNameSuffix;
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/SqlUtils.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.utils;
package com.tencent.supersonic.headless.core.utils;
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
@@ -7,7 +7,7 @@ import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.server.pojo.JdbcDataSource;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import java.rmi.ServerException;
import java.sql.Connection;
import java.sql.ResultSet;

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/SysTimeDimensionBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java
View File

@@ -1,14 +1,14 @@
package com.tencent.supersonic.headless.server.utils;
package com.tencent.supersonic.headless.core.utils;
import com.tencent.supersonic.headless.api.enums.DimensionType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.enums.DimensionType;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor;
import java.util.List;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@Slf4j
public class SysTimeDimensionBuilder {

61
headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/MaterializationQueryTest.java
View File

@@ -1,61 +0,0 @@
package com.tencent.supersonic.headless.query.domain.calcite;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.core.utils.QueryUtils;
import lombok.extern.slf4j.Slf4j;
import java.util.Arrays;
@Slf4j
public class MaterializationQueryTest {
private final QueryParser queryParser;
private final QueryUtils queryUtils;
public MaterializationQueryTest(QueryParser queryParser,
QueryUtils queryUtils) {
this.queryParser = queryParser;
this.queryUtils = queryUtils;
}
public void test() {
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setModelId(1L);
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.UNKNOWN);
aggregator.setColumn("pv");
queryStructReq.setAggregators(Arrays.asList(aggregator));
queryStructReq.setGroups(Arrays.asList("department"));
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateMode.LIST);
dateConf.setDateList(Arrays.asList("2023-08-01"));
queryStructReq.setDateInfo(dateConf);
try {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setIsS2SQL(false);
queryStatement = queryParser.logicSql(queryStatement);
queryUtils.checkSqlParse(queryStatement);
queryStatement.setModelIds(queryStructReq.getModelIds());
log.info("queryStatement:{}", queryStatement);
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStructReq, queryStatement);
}
//queryParser.test(queryStructReq,metricReq);
log.info("queryStatement:{}", queryStatement);
} catch (Exception e) {
e.printStackTrace();
}
}
}

28
headless/server/pom.xml
View File

@@ -44,13 +44,7 @@
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>headless-api</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>common</artifactId>
<artifactId>headless-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -96,6 +90,26 @@
<artifactId>h2</artifactId>
<version>${h2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml-schemas</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>${easyexcel.version}</version>
</dependency>
</dependencies>
</project>

24
headless/server/src/main/java/com/tencent/supersonic/headless/server/adaptor/db/DbAdaptorFactory.java
View File

@@ -1,24 +0,0 @@
package com.tencent.supersonic.headless.server.adaptor.db;
import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum;
import java.util.HashMap;
import java.util.Map;
public class DbAdaptorFactory {
private static Map<String, DbAdaptor> dbAdaptorMap;
static {
dbAdaptorMap = new HashMap<>();
dbAdaptorMap.put(EngineTypeEnum.CLICKHOUSE.getName(), new ClickHouseAdaptor());
dbAdaptorMap.put(EngineTypeEnum.MYSQL.getName(), new MysqlAdaptor());
dbAdaptorMap.put(EngineTypeEnum.H2.getName(), new H2Adaptor());
}
public static DbAdaptor getEngineAdaptor(String engineType) {
return dbAdaptorMap.get(engineType);
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/ApiHeaderCheck.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.annotation;
package com.tencent.supersonic.headless.server.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/S2SQLDataPermission.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2SQLDataPermission.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.annotation;
package com.tencent.supersonic.headless.server.annotation;
import java.lang.annotation.Target;
import java.lang.annotation.RetentionPolicy;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/annotation/StructDataPermission.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/StructDataPermission.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.annotation;
package com.tencent.supersonic.headless.server.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/ApiHeaderCheckAspect.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.aspect;
package com.tencent.supersonic.headless.server.aspect;
import com.tencent.supersonic.common.pojo.Pair;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
@@ -33,7 +33,7 @@ public class ApiHeaderCheckAspect {
@Autowired
private AppService appService;
@Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck)")
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck)")
private void apiPermissionCheck() {
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/AuthCheckBaseAspect.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.aspect;
package com.tencent.supersonic.headless.server.aspect;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/DimValueAspect.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.aspect;
package com.tencent.supersonic.headless.server.aspect;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Filter;
@@ -47,7 +47,7 @@ public class DimValueAspect {
@Autowired
private DimensionService dimensionService;
@Around("execution(* com.tencent.supersonic.headless.core.service.impl.QueryServiceImpl.queryBySql(..))")
@Around("execution(* com.tencent.supersonic.headless.server.service.impl.QueryServiceImpl.queryBySql(..))")
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapSqlEnable) {
log.debug("sql dimensionValueMapEnable is false, skip dimensionValueMap");
@@ -133,9 +133,9 @@ public class DimValueAspect {
filedNameToValueMap.put(fieldName, map);
}
@Around("execution(* com.tencent.supersonic.headless.core.rest.QueryController.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.headless.core.service.QueryService.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.headless.core.service.QueryService.queryByStructWithAuth(..))")
@Around("execution(* com.tencent.supersonic.headless.server.rest.QueryController.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByStruct(..))"
+ " || execution(* com.tencent.supersonic.headless.server.service.QueryService.queryByStructWithAuth(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/S2SQLDataAspect.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.aspect;
package com.tencent.supersonic.headless.server.aspect;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -56,7 +56,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.S2SQLDataPermission)")
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission)")
private void s2SQLPermissionCheck() {
}

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/aspect/StructDataAspect.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.aspect;
package com.tencent.supersonic.headless.server.aspect;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelService;
@@ -50,7 +50,7 @@ public class StructDataAspect extends AuthCheckBaseAspect {
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.StructDataPermission)")
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.StructDataPermission)")
public void dataPermissionAOP() {
}

28
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/HeadlessSchemaManager.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java
View File

@@ -1,17 +1,11 @@
package com.tencent.supersonic.headless.core.parser.calcite;
package com.tencent.supersonic.headless.server.manager;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
@@ -25,6 +19,13 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
@@ -75,12 +76,14 @@ public class HeadlessSchemaManager {
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
Map<Long, String> modelIdName = new HashMap<>();
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next());
headlessModel.setDatabaseResp(databaseResp);
List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
if (!CollectionUtils.isEmpty(modelRelas)) {
headlessModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(d -> getDatasource(d))
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource)
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
headlessModel.setDatasourceMap(dataSourceMap);
}
@@ -116,9 +119,10 @@ public class HeadlessSchemaManager {
}
public static DataSource getDatasource(final DataModelYamlTpl d) {
DataSource datasource = DataSource.builder().sourceId(d.getSourceId()).sqlQuery(d.getSqlQuery())
.name(d.getName()).tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasures(d.getMeasures())).dimensions(getDimensions(d.getDimensions())).build();
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId())
.sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery())
.identifiers(getIdentify(d.getIdentifiers())).measures(getMeasures(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions())).build();
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));

27
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java
View File

@@ -1,27 +0,0 @@
package com.tencent.supersonic.headless.server.manager;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class MetricYamlManager {
public static List<MetricYamlTpl> convert2YamlObj(List<MetricResp> metrics) {
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
for (MetricResp metric : metrics) {
MetricYamlTpl metricYamlTpl = MetricConverter.convert2MetricYamlTpl(metric);
metricYamlTpls.add(metricYamlTpl);
}
return metricYamlTpls;
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/mapper/StatMapper.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.persistence.mapper;
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.api.pojo.QueryStat;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository/StatRepository.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.persistence.repository;
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.request.ItemUseReq;

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/repository/StatRepositoryImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java
View File

@@ -1,12 +1,13 @@
package com.tencent.supersonic.headless.core.persistence.repository;
package com.tencent.supersonic.headless.server.persistence.repository.impl;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.core.persistence.mapper.StatMapper;
import com.tencent.supersonic.headless.server.persistence.mapper.StatMapper;
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/persistence/pojo/DataDownload.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.persistence.pojo;
package com.tencent.supersonic.headless.server.pojo;
import lombok.Builder;
import lombok.Data;

16
headless/core/src/main/java/com/tencent/supersonic/headless/core/rest/QueryController.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java
View File

@@ -1,28 +1,28 @@
package com.tencent.supersonic.headless.core.rest;
package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.request.DownloadStructReq;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SqlParserResp;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.service.DownloadService;
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.core.service.QueryService;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.DownloadService;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.service.QueryService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/rest/SchemaController.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.rest;
package com.tencent.supersonic.headless.server.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -12,7 +12,7 @@ import com.tencent.supersonic.headless.api.response.DomainResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.service.SchemaService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/Catalog.java
View File

@@ -7,9 +7,9 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;
@@ -22,8 +22,6 @@ public interface Catalog {
DatabaseResp getDatabaseByModelId(Long modelId);
String getModelFullPath(List<Long> modelIds);
DimensionResp getDimension(String bizName, Long modelId);
DimensionResp getDimension(Long id);

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/service/DownloadService.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DownloadService.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.service;
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/service/HeadlessQueryEngine.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.core.service;
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
public interface HeadlessQueryEngine {

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java
View File

@@ -11,9 +11,9 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.MeasureResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import java.util.List;

16
headless/core/src/main/java/com/tencent/supersonic/headless/core/service/QueryService.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
View File

@@ -1,20 +1,20 @@
package com.tencent.supersonic.headless.core.service;
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck;
import javax.servlet.http.HttpServletRequest;
import java.util.List;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/service/SchemaService.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.service;
package com.tencent.supersonic.headless.server.service;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java
View File

@@ -7,9 +7,9 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.DatabaseService;
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
@Component
@@ -58,11 +57,6 @@ public class CatalogImpl implements Catalog {
return modelService.getDatabaseByModelId(modelId);
}
@Override
public String getModelFullPath(List<Long> modelIds) {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
@Override
public DimensionResp getDimension(String bizName, Long modelId) {
return dimensionService.getDimension(bizName, modelId);

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -5,17 +5,17 @@ import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import com.tencent.supersonic.headless.server.persistence.repository.DatabaseRepository;
import com.tencent.supersonic.headless.server.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import com.tencent.supersonic.headless.server.utils.JdbcDataSourceUtils;
import com.tencent.supersonic.headless.server.utils.SqlUtils;
import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
import com.tencent.supersonic.headless.core.utils.SqlUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/DownloadServiceImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.service.impl;
package com.tencent.supersonic.headless.server.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
@@ -12,7 +12,6 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.core.persistence.pojo.DataDownload;
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.request.DownloadStructReq;
import com.tencent.supersonic.headless.api.enums.SemanticType;
@@ -23,10 +22,11 @@ import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.service.QueryService;
import com.tencent.supersonic.headless.core.utils.DataTransformUtils;
import com.tencent.supersonic.headless.core.service.DownloadService;
import com.tencent.supersonic.headless.server.pojo.DataDownload;
import com.tencent.supersonic.headless.server.service.DownloadService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.QueryService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;

33
headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/HeadlessQueryEngineImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java
View File

@@ -1,17 +1,21 @@
package com.tencent.supersonic.headless.core.service.impl;
package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.core.utils.QueryUtils;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -23,19 +27,24 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
private final QueryParser queryParser;
private final Catalog catalog;
private final QueryUtils queryUtils;
private final QueryStructUtils queryStructUtils;
private final HeadlessSchemaManager headlessSchemaManager;
public HeadlessQueryEngineImpl(QueryParser queryParser, Catalog catalog,
QueryUtils queryUtils) {
QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager,
QueryStructUtils queryStructUtils) {
this.queryParser = queryParser;
this.catalog = catalog;
this.queryUtils = queryUtils;
this.headlessSchemaManager = headlessSchemaManager;
this.queryStructUtils = queryStructUtils;
}
public QueryResultWithSchemaResp execute(QueryStatement queryStatement) {
QueryResultWithSchemaResp queryResultWithColumns = null;
QueryExecutor queryExecutor = route(queryStatement);
if (queryExecutor != null) {
queryResultWithColumns = queryExecutor.execute(catalog, queryStatement);
queryResultWithColumns = queryExecutor.execute(queryStatement);
queryResultWithColumns.setSql(queryStatement.getSql());
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelIds());
@@ -46,6 +55,10 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
public QueryStatement plan(QueryStatement queryStatement) throws Exception {
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
HeadlessModel headlessModel = headlessSchemaManager.get(queryStatement.getQueryStructReq().getModelIdStr());
ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStatement.getQueryStructReq());
headlessModel.setDataDate(itemDateResp);
queryStatement.setHeadlessModel(headlessModel);
queryStatement = queryParser.logicSql(queryStatement);
queryUtils.checkSqlParse(queryStatement);
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
@@ -70,7 +83,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
}
@Override
public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception {
public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructCmd);
queryStatement.setParseSqlReq(sqlCommend);
@@ -78,7 +91,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement));
}
public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception {
public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructCmd);
queryStatement.setMetricReq(metricCommand);

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -30,12 +30,12 @@ import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.server.manager.MetricYamlManager;
import com.tencent.supersonic.headless.server.manager.ModelYamlManager;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.core.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.core.manager.MetricYamlManager;
import com.tencent.supersonic.headless.core.manager.ModelYamlManager;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;

48
headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/QueryServiceImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.service.impl;
package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
@@ -14,44 +14,44 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Item;
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Item;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.response.AppDetailResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck;
import com.tencent.supersonic.headless.core.annotation.S2SQLDataPermission;
import com.tencent.supersonic.headless.core.annotation.StructDataPermission;
import com.tencent.supersonic.headless.core.aspect.ApiHeaderCheckAspect;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck;
import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission;
import com.tencent.supersonic.headless.server.annotation.StructDataPermission;
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.parser.convert.QueryReqConverter;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.core.service.QueryService;
import com.tencent.supersonic.headless.core.service.SchemaService;
import com.tencent.supersonic.headless.core.utils.QueryUtils;
import com.tencent.supersonic.headless.core.utils.StatUtils;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.service.AppService;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -125,13 +125,13 @@ public class QueryServiceImpl implements QueryService {
return headlessQueryEngine.execute(queryStatement);
}
private QueryStatement convertToQueryStatement(QueryS2SQLReq querySqlCmd, User user) throws Exception {
private QueryStatement convertToQueryStatement(QueryS2SQLReq queryS2SQLReq, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
filter.setModelIds(querySqlCmd.getModelIds());
filter.setModelIds(queryS2SQLReq.getModelIds());
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, modelSchemaResps);
queryStatement.setModelIds(querySqlCmd.getModelIds());
QueryStatement queryStatement = queryReqConverter.convert(queryS2SQLReq, modelSchemaResps);
queryStatement.setModelIds(queryS2SQLReq.getModelIds());
return queryStatement;
}

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/service/impl/SchemaServiceImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.service.impl;
package com.tencent.supersonic.headless.server.service.impl;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -16,12 +16,12 @@ import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.service.QueryService;
import com.tencent.supersonic.headless.core.service.SchemaService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DatabaseConverter.java
View File

@@ -4,8 +4,8 @@ import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import com.tencent.supersonic.headless.server.pojo.ConnectInfo;
import com.tencent.supersonic.headless.server.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.ConnectInfo;
import com.tencent.supersonic.headless.core.pojo.Database;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import java.util.Arrays;

9
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java
View File

@@ -9,7 +9,6 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.request.DimensionReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
@@ -76,12 +75,4 @@ public class DimensionConverter {
return dimensionResp;
}
public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) {
DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl();
BeanUtils.copyProperties(dimension, dimensionYamlTpl);
dimensionYamlTpl.setName(dimension.getBizName());
dimensionYamlTpl.setOwners(dimension.getCreatedBy());
return dimensionYamlTpl;
}
}

27
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
View File

@@ -1,25 +1,19 @@
package com.tencent.supersonic.headless.server.utils;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import org.springframework.beans.BeanUtils;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class MetricConverter {
@@ -50,12 +44,6 @@ public class MetricConverter {
return metricDO;
}
public static MeasureYamlTpl convert(Measure measure) {
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
measureYamlTpl.setName(measure.getBizName());
return measureYamlTpl;
}
public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap, List<Long> collect) {
MetricResp metricResp = new MetricResp();
BeanUtils.copyProperties(metricDO, metricResp);
@@ -80,19 +68,4 @@ public class MetricConverter {
return metricResp;
}
public static MetricYamlTpl convert2MetricYamlTpl(MetricResp metric) {
MetricYamlTpl metricYamlTpl = new MetricYamlTpl();
BeanUtils.copyProperties(metric, metricYamlTpl);
metricYamlTpl.setName(metric.getBizName());
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
MetricTypeParams exprMetricTypeParams = metric.getTypeParams();
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
metricTypeParamsYamlTpl.setExpr(exprMetricTypeParams.getExpr());
List<Measure> measures = exprMetricTypeParams.getMeasures();
metricTypeParamsYamlTpl.setMeasures(
measures.stream().map(MetricConverter::convert).collect(Collectors.toList()));
metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl);
return metricYamlTpl;
}
}

75
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/convert/QueryReqConverter.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.parser.convert;
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.Aggregator;
@@ -10,22 +10,21 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -46,37 +45,40 @@ import java.util.stream.Stream;
@Component
@Slf4j
public class QueryReqConverter {
@Autowired
private ModelService domainService;
@Autowired
private HeadlessQueryEngine parserService;
private HeadlessQueryEngine headlessQueryEngine;
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private SqlGenerateUtils sqlGenerateUtils;
@Autowired
private Catalog catalog;
public QueryStatement convert(QueryS2SQLReq databaseReq, List<ModelSchemaResp> modelSchemaResps) throws Exception {
public QueryStatement convert(QueryS2SQLReq queryS2SQLReq,
List<ModelSchemaResp> modelSchemaResps) throws Exception {
if (CollectionUtils.isEmpty(modelSchemaResps)) {
return new QueryStatement();
}
Map<Long, ModelSchemaResp> modelSchemaRespMap = modelSchemaResps.stream()
.collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp));
//1.convert name to bizName
convertNameToBizName(databaseReq, modelSchemaResps);
convertNameToBizName(queryS2SQLReq, modelSchemaResps);
//2.functionName corrector
functionNameCorrector(databaseReq);
functionNameCorrector(queryS2SQLReq);
//3.correct tableName
correctTableName(databaseReq);
correctTableName(queryS2SQLReq);
String tableName = SqlParserSelectHelper.getTableName(databaseReq.getSql());
String tableName = SqlParserSelectHelper.getTableName(queryS2SQLReq.getSql());
if (StringUtils.isEmpty(tableName)) {
return new QueryStatement();
}
//4.build MetricTables
List<String> allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql());
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
List<String> metrics = getMetrics(modelSchemaResps, allFields);
QueryStructReq queryStructCmd = new QueryStructReq();
QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable();
metricTable.setMetrics(metrics);
@@ -88,42 +90,43 @@ public class QueryReqConverter {
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>());
for (Long modelId : databaseReq.getModelIds()) {
metricTable.getMetrics().add(queryStructUtils.generateInternalMetricName(modelId,
metricTable.getDimensions()));
for (Long modelId : queryS2SQLReq.getModelIds()) {
ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId);
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName()));
}
} else {
queryStructCmd.setAggregators(
queryStructReq.setAggregators(
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
Collectors.toList()));
}
AggOption aggOption = getAggOption(databaseReq);
AggOption aggOption = getAggOption(queryS2SQLReq);
metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq();
BeanUtils.copyProperties(databaseReq, result);
BeanUtils.copyProperties(queryS2SQLReq, result);
result.setRootPath(catalog.getModelFullPath(databaseReq.getModelIds()));
result.setRootPath(queryS2SQLReq.getModelIdStr());
result.setTables(tables);
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0));
if (!queryStructUtils.isSupportWith(EngineTypeEnum.valueOf(database.getType().toUpperCase()),
DatabaseResp database = catalog.getDatabaseByModelId(queryS2SQLReq.getModelIds().get(0));
if (!sqlGenerateUtils.isSupportWith(EngineType.valueOf(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
result.setWithAlias(false);
}
//5.physicalSql by ParseSqlReq
queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql()));
queryStructCmd.setModelIds(databaseReq.getModelIds().stream().collect(Collectors.toSet()));
queryStructCmd.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd);
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryS2SQLReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(queryS2SQLReq.getModelIds()));
queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructCmd);
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setParseSqlReq(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructCmd));
queryStatement = parserService.plan(queryStatement);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setModelIds(queryS2SQLReq.getModelIds());
queryStatement = headlessQueryEngine.plan(queryStatement);
queryStatement.setSql(String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql()));
return queryStatement;
}

212
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryStructUtils.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.utils;
package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -9,32 +9,25 @@ import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.service.SchemaService;
import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -55,7 +48,6 @@ import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
@@ -75,12 +67,6 @@ public class QueryStructUtils {
private final DateModeUtils dateModeUtils;
private final SqlFilterUtils sqlFilterUtils;
private final Catalog catalog;
@Value("${internal.metric.cnt.suffix:internal_cnt}")
private String internalMetricNameSuffix;
@Value("${metricParser.agg.mysql.lowVersion:5.7}")
private String mysqlLowVersion;
@Value("${metricParser.agg.ck.lowVersion:20.4}")
private String ckLowVersion;
@Autowired
private SchemaService schemaService;
@@ -113,7 +99,6 @@ public class QueryStructUtils {
dimensionIds.add(pair.get(col).get(0).getId());
}
}
return dimensionIds;
}
@@ -136,63 +121,6 @@ public class QueryStructUtils {
return metricIds;
}
public String getDateWhereClause(QueryStructReq queryStructCmd) {
DateConf dateInfo = queryStructCmd.getDateInfo();
if (Objects.isNull(dateInfo) || Objects.isNull(dateInfo.getDateMode())) {
return "";
}
if (dateInfo.getDateMode().equals(DateMode.RECENT)) {
if (dateInfo.getUnit() <= 0) {
return "";
}
}
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
List<Long> metricIds = getMetricIds(queryStructCmd);
ItemDateResp dateDate = catalog.getItemDate(
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()),
new ItemDateFilter(metricIds, TypeEnums.METRIC.getName()));
if (Objects.isNull(dateDate)
|| Strings.isEmpty(dateDate.getStartDate())
&& Strings.isEmpty(dateDate.getEndDate())) {
if (dateInfo.getDateMode().equals(DateMode.LIST)) {
return dateModeUtils.listDateStr(dateDate, dateInfo);
}
if (dateInfo.getDateMode().equals(DateMode.BETWEEN)) {
return dateModeUtils.betweenDateStr(dateDate, dateInfo);
}
if (dateModeUtils.hasAvailableDataMode(dateInfo)) {
return dateModeUtils.hasDataModeStr(dateDate, dateInfo);
}
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
}
log.info("dateDate:{}", dateDate);
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
}
public String generateWhere(QueryStructReq queryStructCmd) {
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructCmd.getOriginalFilter());
String whereFromDate = getDateWhereClause(queryStructCmd);
return mergeDateWhereClause(queryStructCmd, whereClauseFromFilter, whereFromDate);
}
public String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter,
String whereFromDate) {
if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
} else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
return whereClauseFromFilter;
} else if (Strings.isNotEmpty(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
return whereFromDate;
} else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
log.info("the current date information is empty, enter the date initialization logic");
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
}
return whereClauseFromFilter;
}
public Set<String> getResNameEn(QueryStructReq queryStructCmd) {
Set<String> resNameEnSet = new HashSet<>();
queryStructCmd.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn()));
@@ -254,76 +182,13 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public String generateInternalMetricName(Long modelId, List<String> groups) {
String internalMetricNamePrefix = "";
MetaFilter modelFilter = new MetaFilter(Collections.singletonList(modelId));
List<DimensionResp> dimensions = catalog.getDimensions(modelFilter);
if (!CollectionUtils.isEmpty(dimensions)) {
internalMetricNamePrefix = dimensions.get(0).getModelBizName();
}
return internalMetricNamePrefix + UNDERLINE + internalMetricNameSuffix;
}
public boolean isSupportWith(EngineTypeEnum engineTypeEnum, String version) {
if (engineTypeEnum.equals(EngineTypeEnum.MYSQL) && Objects.nonNull(version) && version.startsWith(
mysqlLowVersion)) {
return false;
}
if (engineTypeEnum.equals(EngineTypeEnum.CLICKHOUSE) && Objects.nonNull(version)
&& StringUtil.compareVersion(version,
ckLowVersion) < 0) {
return false;
}
return true;
}
public String generateZipperWhere(QueryStructReq queryStructCmd) {
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructCmd.getOriginalFilter());
String whereFromDate = getZipperDateWhereClause(queryStructCmd);
return mergeDateWhereClause(queryStructCmd, whereClauseFromFilter, whereFromDate);
}
public String generateZipperWhere(QueryStructReq queryStructCmd, ParseSqlReq parseSqlReq) {
if (Objects.nonNull(parseSqlReq.getSql()) && !CollectionUtils.isEmpty(parseSqlReq.getTables())
&& Objects.nonNull(queryStructCmd.getDateInfo())) {
String sql = SqlParserRemoveHelper.removeWhere(parseSqlReq.getSql(),
dateModeUtils.getDateCol());
parseSqlReq.getTables().stream().forEach(t -> {
if (Objects.nonNull(t)) {
List<String> dimensions = new ArrayList<>();
if (!CollectionUtils.isEmpty(t.getDimensions())) {
dimensions.addAll(t.getDimensions().stream()
.filter(d -> !dateModeUtils.getDateCol().contains(d.toLowerCase())).collect(
Collectors.toList()));
}
dimensions.add(dateModeUtils.getDateColBegin(queryStructCmd.getDateInfo()));
dimensions.add(dateModeUtils.getDateColEnd(queryStructCmd.getDateInfo()));
t.setDimensions(dimensions);
}
});
return SqlParserAddHelper.addWhere(sql,
SqlParserSelectHelper.getTimeFilter(getTimeRanges(queryStructCmd),
dateModeUtils.getDateColBegin(queryStructCmd.getDateInfo()),
dateModeUtils.getDateColEnd(queryStructCmd.getDateInfo())));
}
return parseSqlReq.getSql();
}
public String getZipperDateWhereClause(QueryStructReq queryStructCmd) {
List<ImmutablePair<String, String>> timeRanges = getTimeRanges(queryStructCmd);
List<String> wheres = new ArrayList<>();
if (!CollectionUtils.isEmpty(timeRanges)) {
for (ImmutablePair<String, String> range : timeRanges) {
String strWhere = dateModeUtils.getDateWhereStr(queryStructCmd.getDateInfo(), range);
if (!strWhere.isEmpty()) {
wheres.add(strWhere);
}
}
if (!wheres.isEmpty()) {
return wheres.stream().collect(Collectors.joining(" or ", "(", ")"));
}
}
return "";
public ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) {
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
List<Long> metricIds = getMetricIds(queryStructCmd);
ItemDateResp dateDate = catalog.getItemDate(
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()),
new ItemDateFilter(metricIds, TypeEnums.METRIC.getName()));
return dateDate;
}
public Triple<String, String, String> getBeginEndTime(QueryStructReq queryStructCmd) {
@@ -377,61 +242,6 @@ public class QueryStructUtils {
return Triple.of("", "", "");
}
public List<ImmutablePair<String, String>> getTimeRanges(QueryStructReq queryStructCmd) {
List<ImmutablePair<String, String>> ret = new ArrayList<>();
if (Objects.isNull(queryStructCmd) || Objects.isNull(queryStructCmd.getDateInfo())) {
return ret;
}
DateConf dateConf = queryStructCmd.getDateInfo();
switch (dateConf.getDateMode()) {
case AVAILABLE:
case BETWEEN:
ret.add(ImmutablePair.of(dateConf.getStartDate(), dateConf.getEndDate()));
break;
case LIST:
for (String date : dateConf.getDateList()) {
ret.add(ImmutablePair.of(date, date));
}
break;
case RECENT:
ItemDateResp dateDate = getItemDateResp(queryStructCmd);
LocalDate dateMax = LocalDate.now().minusDays(1);
LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1);
if (Objects.isNull(dateDate)) {
ret.add(ImmutablePair.of(dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)),
dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT))));
break;
}
switch (dateConf.getPeriod()) {
case DAY:
ret.add(dateModeUtils.recentDay(dateDate, dateConf));
break;
case WEEK:
ret.add(dateModeUtils.recentWeek(dateDate, dateConf));
break;
case MONTH:
List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dateDate, dateConf);
ret.addAll(rets);
break;
default:
break;
}
break;
default:
break;
}
return ret;
}
private ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) {
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
List<Long> metricIds = getMetricIds(queryStructCmd);
ItemDateResp dateDate = catalog.getItemDate(
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()),
new ItemDateFilter(metricIds, TypeEnums.METRIC.getName()));
return dateDate;
}
public DateConf getDateConfBySql(String sql) {
List<FieldExpression> fieldExpressions = SqlParserSelectHelper.getFilterExpression(sql);
if (!CollectionUtils.isEmpty(fieldExpressions)) {

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/QueryUtils.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.utils;
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
@@ -10,7 +10,8 @@ import com.tencent.supersonic.headless.api.enums.SemanticType;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/StatUtils.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.utils;
package com.tencent.supersonic.headless.server.utils;
import com.alibaba.ttl.TransmittableThreadLocal;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -7,17 +7,17 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.enums.QueryOptMode;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.enums.QueryTypeBack;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.enums.QueryOptMode;
import com.tencent.supersonic.headless.api.enums.QueryTypeBack;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.persistence.repository.StatRepository;
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;

2
headless/core/src/main/resources/mapper/StatMapper.xml → headless/server/src/main/resources/mapper/StatMapper.xml
View File

@@ -2,7 +2,7 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.tencent.supersonic.headless.core.persistence.mapper.StatMapper">
<mapper namespace="com.tencent.supersonic.headless.server.persistence.mapper.StatMapper">
<resultMap id="QueryStatDO"
type="com.tencent.supersonic.headless.api.pojo.QueryStat">

28
headless/core/src/test/java/com/tencent/supersonic/headless/query/domain/calcite/HeadlessParserServiceTest.java → headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -1,26 +1,27 @@
package com.tencent.supersonic.headless.query.domain.calcite;
package com.tencent.supersonic.headless.server.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.response.SqlParserResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.core.parser.calcite.HeadlessSchemaManager;
import com.tencent.supersonic.headless.api.response.SqlParserResp;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@Slf4j
class HeadlessParserServiceTest {
@@ -162,7 +163,7 @@ class HeadlessParserServiceTest {
metric2.setTypeParams(metricTypeParams1);
metric.add(metric2);
HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
//HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
MetricQueryReq metricCommand = new MetricQueryReq();
metricCommand.setRootPath("s2");
@@ -250,6 +251,5 @@ class HeadlessParserServiceTest {
headlessSchema.getDimension()
.put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
}
}

8
headless/core/src/test/java/com/tencent/supersonic/headless/query/service/DownloadServiceImplTest.java → headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java
View File

@@ -1,20 +1,18 @@
package com.tencent.supersonic.headless.query.service;
package com.tencent.supersonic.headless.server.service;
import com.alibaba.excel.util.FileUtils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.core.service.QueryService;
import com.tencent.supersonic.headless.core.service.impl.DownloadServiceImpl;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.impl.DownloadServiceImpl;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;

8
headless/server/src/test/java/com/tencent/supersonic/headless/model/application/ModelServiceImplTest.java → headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.model.application;
package com.tencent.supersonic.headless.server.service;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -17,12 +17,6 @@ import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;
import com.tencent.supersonic.headless.server.persistence.repository.ModelRepository;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.impl.ModelServiceImpl;
import com.tencent.supersonic.headless.server.utils.ModelConverter;
import org.junit.jupiter.api.Assertions;

3
headless/core/src/test/java/com/tencent/supersonic/headless/query/utils/DataTransformUtilsTest.java → headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/DataTransformUtilsTest.java
View File

@@ -1,10 +1,11 @@
package com.tencent.supersonic.headless.query.utils;
package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.headless.core.utils.DataTransformUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;