topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 21:17:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project] supersonic 0.6.0 version update (#16)

Browse Source 
Co-authored-by: lexluo <lexluo@tencent.com>
This commit is contained in:
lexluo09
2023-07-16 21:32:33 +08:00
committed by GitHub
parent a0869dc7bd
commit 041daad1e4
261 changed files with 12031 additions and 3266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/ParserServiceImpl.java
View File

@@ -1,115 +0,0 @@
package com.tencent.supersonic.semantic.query.application;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.query.domain.ParserService;
import com.tencent.supersonic.semantic.query.domain.parser.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.domain.parser.SemanticSqlService;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.SemanticModel;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
@Primary
@Slf4j
public class ParserServiceImpl implements ParserService {
private final SemanticSchemaManager semanticSchemaManager;
private final SemanticSqlService semanticSqlService;
public ParserServiceImpl(SemanticSchemaManager schemaManager,
SemanticSqlService semanticSqlService) {
this.semanticSchemaManager = schemaManager;
this.semanticSqlService = semanticSqlService;
}
@Override
public SqlParserResp physicalSql(ParseSqlReq sqlCommend) throws Exception {
return parser(sqlCommend);
}
@Override
public SqlParserResp physicalSql(MetricReq metricCommand) throws Exception {
return parser(metricCommand);
}
public SqlParserResp parser(ParseSqlReq sqlCommend) {
log.info("parser MetricReq [{}] ", sqlCommend);
SqlParserResp sqlParserInfo = new SqlParserResp();
try {
if (!CollectionUtils.isEmpty(sqlCommend.getTables())) {
List<String> tables = new ArrayList<>();
String sourceId = "";
for (MetricTable metricTable : sqlCommend.getTables()) {
MetricReq metricReq = new MetricReq();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(formatWhere(metricTable.getWhere()));
metricReq.setRootPath(sqlCommend.getRootPath());
SqlParserResp tableSql = parser(metricReq, metricTable.isAgg());
if (!tableSql.isOk()) {
sqlParserInfo.setErrMsg(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
tableSql.getErrMsg()));
return sqlParserInfo;
}
tables.add(String.format("%s as (%s)", metricTable.getAlias(), tableSql.getSql()));
sourceId = tableSql.getSourceId();
}
if (!tables.isEmpty()) {
String sql = "with " + String.join(",", tables) + "\n" + sqlCommend.getSql();
sqlParserInfo.setSql(sql);
sqlParserInfo.setSourceId(sourceId);
return sqlParserInfo;
}
}
} catch (Exception e) {
log.error("physicalSql error {}", e);
sqlParserInfo.setErrMsg(e.getMessage());
}
return sqlParserInfo;
}
public SqlParserResp parser(MetricReq metricCommand) {
return parser(metricCommand, true);
}
public SqlParserResp parser(MetricReq metricCommand, boolean isAgg) {
log.info("parser MetricReq [{}] isAgg [{}]", metricCommand, isAgg);
SqlParserResp sqlParser = new SqlParserResp();
if (metricCommand.getRootPath().isEmpty()) {
sqlParser.setErrMsg("rootPath empty");
return sqlParser;
}
try {
SemanticModel semanticModel = semanticSchemaManager.get(metricCommand.getRootPath());
if (semanticModel == null) {
sqlParser.setErrMsg("semanticSchema not found");
return sqlParser;
}
return semanticSqlService.explain(metricCommand, isAgg, semanticModel);
} catch (Exception e) {
sqlParser.setErrMsg(e.getMessage());
log.error("parser error MetricCommand[{}] error [{}]", metricCommand, e);
}
return sqlParser;
}
private String formatWhere(String where) {
if (StringUtils.isEmpty(where)) {
return where;
}
return where.replace("\"", "\\\\\"");
}
}

129
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/QueryServiceImpl.java
View File

@@ -8,25 +8,27 @@ import com.tencent.supersonic.semantic.api.core.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.core.request.DomainSchemaFilterReq;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.core.domain.DatabaseService;
import com.tencent.supersonic.semantic.query.domain.ParserService;
import com.tencent.supersonic.semantic.query.application.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.domain.QueryService;
import com.tencent.supersonic.semantic.query.domain.SchemaService;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.annotation.DataPermission;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.domain.utils.QueryReqConverter;
import com.tencent.supersonic.semantic.query.domain.utils.QueryStructUtils;
import com.tencent.supersonic.semantic.query.domain.utils.QueryUtils;
import com.tencent.supersonic.semantic.query.domain.utils.StatUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import javax.servlet.http.HttpServletRequest;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -35,28 +37,28 @@ import org.springframework.stereotype.Service;
@Slf4j
public class QueryServiceImpl implements QueryService {
private final ParserService parserService;
private final DatabaseService databaseService;
private final QueryStructUtils queryStructUtils;
private final StatUtils statUtils;
private final CacheUtils cacheUtils;
private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter;
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
public QueryServiceImpl(ParserService parserService,
DatabaseService databaseService,
QueryStructUtils queryStructUtils,
private final SemanticQueryEngine semanticQueryEngine;
public QueryServiceImpl(
StatUtils statUtils,
CacheUtils cacheUtils,
QueryReqConverter queryReqConverter) {
this.parserService = parserService;
this.databaseService = databaseService;
this.queryStructUtils = queryStructUtils;
QueryUtils queryUtils,
QueryReqConverter queryReqConverter,
SemanticQueryEngine semanticQueryEngine) {
this.statUtils = statUtils;
this.cacheUtils = cacheUtils;
this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter;
this.semanticQueryEngine = semanticQueryEngine;
}
@Override
@@ -69,35 +71,45 @@ public class QueryServiceImpl implements QueryService {
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<DomainSchemaResp> domainSchemas = schemaService.fetchDomainSchema(filter, user);
SqlParserResp sqlParser = queryReqConverter.convert(querySqlCmd, domainSchemas);
return databaseService.executeSql(sqlParser.getSql(), querySqlCmd.getDomainId());
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchemas);
queryStatement.setDomainId(querySqlCmd.getDomainId());
return semanticQueryEngine.execute(queryStatement);
}
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception {
QueryResultWithSchemaResp queryResultWithColumns;
QueryResultWithSchemaResp queryResultWithColumns = null;
log.info("[queryStructCmd:{}]", queryStructCmd);
try {
statUtils.initStatInfo(queryStructCmd, user);
String cacheKey = cacheUtils.generateCacheKey(queryStructCmd.getDomainId().toString(),
queryStructCmd.generateCommandMd5());
handleGlobalCacheDisable(queryStructCmd);
if (queryStructUtils.queryCache(queryStructCmd.getCacheInfo())) {
queryResultWithColumns = queryStructUtils.queryByStructByCache(queryStructCmd, cacheKey);
} else {
queryResultWithColumns = queryStructUtils.queryByStructWithoutCache(queryStructCmd, cacheKey);
boolean isCache = isCache(queryStructCmd);
if (isCache) {
queryResultWithColumns = queryByCache(cacheKey, queryStructCmd);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
}
}
StatUtils.get().setUseResultCache(false);
QueryStatement queryStatement = semanticQueryEngine.plan(queryStructCmd);
QueryExecutor queryExecutor = semanticQueryEngine.route(queryStatement);
if (queryExecutor != null) {
queryResultWithColumns = semanticQueryEngine.execute(queryStatement);
if (isCache) {
// if queryResultWithColumns is not null, update cache data
queryUtils.cacheResultLogic(cacheKey, queryResultWithColumns);
}
}
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
} catch (Exception e) {
log.warn("exception in queryByStruct, e: ", e);
statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR);
throw e;
}
return queryResultWithColumns;
}
@Override
@@ -115,7 +127,38 @@ public class QueryServiceImpl implements QueryService {
String cacheKey = cacheUtils.generateCacheKey(
queryMultiStructCmd.getQueryStructCmds().get(0).getDomainId().toString(),
queryMultiStructCmd.generateCommandMd5());
return queryStructUtils.queryByMultiStructWithoutCache(queryMultiStructCmd, cacheKey);
boolean isCache = isCache(queryMultiStructCmd);
QueryResultWithSchemaResp queryResultWithColumns;
if (isCache) {
queryResultWithColumns = queryByCache(cacheKey, queryMultiStructCmd);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
}
}
log.info("stat queryByStructWithoutCache, queryMultiStructCmd:{}", queryMultiStructCmd);
try {
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructCmd : queryMultiStructCmd.getQueryStructCmds()) {
QueryStatement queryStatement = semanticQueryEngine.plan(queryStructCmd);
queryUtils.checkSqlParse(queryStatement);
sqlParsers.add(queryStatement);
}
log.info("multi sqlParser:{}", sqlParsers);
QueryStatement sqlParser = queryUtils.sqlParserUnion(queryMultiStructCmd, sqlParsers);
queryResultWithColumns = semanticQueryEngine.execute(sqlParser);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
queryUtils.fillItemNameInfo(queryResultWithColumns, queryMultiStructCmd);
}
queryUtils.cacheResultLogic(cacheKey, queryResultWithColumns);
return queryResultWithColumns;
} catch (Exception e) {
log.warn("exception in queryByMultiStruct, e: ", e);
statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR);
throw e;
}
}
@@ -139,5 +182,37 @@ public class QueryServiceImpl implements QueryService {
return statUtils.getQueryStatInfoWithoutCache(itemUseCommend);
}
private boolean isCache(QueryStructReq queryStructCmd) {
if (!cacheEnable) {
return false;
}
if (queryStructCmd.getCacheInfo() != null) {
return queryStructCmd.getCacheInfo().getCache();
}
return false;
}
private boolean isCache(QueryMultiStructReq queryStructCmd) {
if (!cacheEnable) {
return false;
}
if (!CollectionUtils.isEmpty(queryStructCmd.getQueryStructCmds())
&& queryStructCmd.getQueryStructCmds().get(0).getCacheInfo() != null) {
return queryStructCmd.getQueryStructCmds().get(0).getCacheInfo().getCache();
}
return false;
}
private QueryResultWithSchemaResp queryByCache(String key, Object queryCmd) {
Object resultObject = cacheUtils.get(key);
if (Objects.nonNull(resultObject)) {
log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString());
statUtils.updateResultCacheKey(key);
return (QueryResultWithSchemaResp) resultObject;
}
return null;
}
}

5
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SchemaServiceImpl.java
View File

@@ -111,6 +111,11 @@ public class SchemaServiceImpl implements SchemaService {
return domainService.getDomainListForAdmin(user.getName());
}
@Override
public List<DomainResp> getDomainListForViewer(User user) {
return domainService.getDomainListForViewer(user.getName());
}
@Override
public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionCmd, User user) {
return dimensionService.queryDimension(pageDimensionCmd);

77
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/SemanticQueryEngineImpl.java Normal file
View File

@@ -0,0 +1,77 @@
package com.tencent.supersonic.semantic.query.application;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.application.optimizer.QueryOptimizer;
import com.tencent.supersonic.semantic.query.application.parser.QueryParser;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.domain.utils.ComponentFactory;
import com.tencent.supersonic.semantic.query.domain.utils.QueryUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class SemanticQueryEngineImpl implements SemanticQueryEngine {
private final QueryParser queryParser;
private final Catalog catalog;
private final QueryUtils queryUtils;
public SemanticQueryEngineImpl(QueryParser queryParser, Catalog catalog,
QueryUtils queryUtils) {
this.queryParser = queryParser;
this.catalog = catalog;
this.queryUtils = queryUtils;
}
public QueryResultWithSchemaResp execute(QueryStatement queryStatement) {
QueryResultWithSchemaResp queryResultWithColumns = null;
QueryExecutor queryExecutor = route(queryStatement);
if (queryExecutor != null) {
queryResultWithColumns = queryExecutor.execute(catalog, queryStatement);
queryResultWithColumns.setSql(queryStatement.getSql());
if (queryStatement.getDomainId() > 0) {
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getDomainId());
}
}
return queryResultWithColumns;
}
public QueryStatement plan(QueryStructReq queryStructCmd) throws Exception {
QueryStatement queryStatement = queryParser.logicSql(queryStructCmd);
queryUtils.checkSqlParse(queryStatement);
queryStatement.setDomainId(queryStructCmd.getDomainId());
log.info("queryStatement:{}", queryStatement);
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStructCmd, queryStatement);
}
return queryStatement;
}
public QueryExecutor route(QueryStatement queryStatement) {
for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
if (queryExecutor.accept(queryStatement)) {
return queryExecutor;
}
}
return null;
}
@Override
public QueryStatement physicalSql(ParseSqlReq sqlCommend) throws Exception {
return queryParser.parser(sqlCommend);
}
public QueryStatement physicalSql(MetricReq metricCommand) throws Exception {
return queryParser.parser(metricCommand);
}
}

42
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/executor/JdbcExecutor.java Normal file
View File

@@ -0,0 +1,42 @@
package com.tencent.supersonic.semantic.query.application.executor;
import com.tencent.supersonic.semantic.api.core.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.core.domain.utils.SqlUtils;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
@Component("JdbcExecutor")
@Slf4j
public class JdbcExecutor implements QueryExecutor {
private final SqlUtils sqlUtils;
public JdbcExecutor(SqlUtils sqlUtils) {
this.sqlUtils = sqlUtils;
}
@Override
public boolean accept(QueryStatement queryStatement) {
return true;
}
@Override
public QueryResultWithSchemaResp execute(Catalog catalog, QueryStatement queryStatement) {
if (Strings.isEmpty(queryStatement.getSourceId())) {
log.warn("data base id is empty");
return null;
}
log.info("query SQL: {}", queryStatement.getSql());
DatabaseResp databaseResp = catalog.getDatabase(Long.parseLong(queryStatement.getSourceId()));
log.info("database info:{}", databaseResp);
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp);
sqlUtils.queryInternal(queryStatement.getSql(), queryResultWithColumns);
return queryResultWithColumns;
}
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/executor/QueryExecutor.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.semantic.query.application.executor;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
public interface QueryExecutor {
boolean accept(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(Catalog catalog,QueryStatement queryStatement);
}

38
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/optimizer/DetailQuery.java Normal file
View File

@@ -0,0 +1,38 @@
package com.tencent.supersonic.semantic.query.application.optimizer;
import com.google.common.base.Strings;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@Component("DetailQuery")
public class DetailQuery implements QueryOptimizer {
@Override
public void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement) {
String sqlRaw = queryStatement.getSql().trim();
if (Strings.isNullOrEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");
}
log.info("before handleNoMetric, sql:{}", sqlRaw);
if (isDetailQuery(queryStructCmd)) {
if (queryStructCmd.getMetrics().size() == 0) {
String sqlForm = "select %s from ( %s ) src_no_metric";
String sql = String.format(sqlForm, queryStructCmd.getGroups().stream().collect(
Collectors.joining(",")), sqlRaw);
queryStatement.setSql(sql);
}
}
log.info("after handleNoMetric, sql:{}", queryStatement.getSql());
}
public boolean isDetailQuery(QueryStructReq queryStructCmd) {
return Objects.nonNull(queryStructCmd) && queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(
queryStructCmd.getMetrics());
}
}

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/optimizer/QueryOptimizer.java Normal file
View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.semantic.query.application.optimizer;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
public interface QueryOptimizer {
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
}

113
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/QueryParser.java Normal file
View File

@@ -0,0 +1,113 @@
package com.tencent.supersonic.semantic.query.application.parser;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.domain.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
@Primary
public class QueryParser {
private final Catalog catalog;
public QueryParser(Catalog catalog) {
this.catalog = catalog;
}
public QueryStatement logicSql(QueryStructReq queryStructReq) throws Exception {
ParseSqlReq parseSqlReq = new ParseSqlReq();
MetricReq metricReq = new MetricReq();
log.info("SemanticConverter before [{}]", queryStructReq);
for (SemanticConverter semanticConverter : ComponentFactory.getSemanticConverters()) {
if (semanticConverter.accept(queryStructReq)) {
log.info("SemanticConverter accept [{}]", semanticConverter.getClass().getName());
semanticConverter.converter(catalog, queryStructReq, parseSqlReq, metricReq);
}
}
log.info("SemanticConverter after {} {} {}", queryStructReq, metricReq, parseSqlReq);
if (!parseSqlReq.getSql().isEmpty()) {
return parser(parseSqlReq);
}
return parser(metricReq);
}
public QueryStatement parser(ParseSqlReq sqlCommend) {
log.info("parser MetricReq [{}] ", sqlCommend);
QueryStatement queryStatement = new QueryStatement();
try {
if (!CollectionUtils.isEmpty(sqlCommend.getTables())) {
List<String> tables = new ArrayList<>();
String sourceId = "";
for (MetricTable metricTable : sqlCommend.getTables()) {
MetricReq metricReq = new MetricReq();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(formatWhere(metricTable.getWhere()));
metricReq.setRootPath(sqlCommend.getRootPath());
QueryStatement tableSql = parser(metricReq, metricTable.isAgg());
if (!tableSql.isOk()) {
queryStatement.setErrMsg(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
tableSql.getErrMsg()));
return queryStatement;
}
tables.add(String.format("%s as (%s)", metricTable.getAlias(), tableSql.getSql()));
sourceId = tableSql.getSourceId();
}
if (!tables.isEmpty()) {
String sql = "with " + String.join(",", tables) + "\n" + sqlCommend.getSql();
queryStatement.setSql(sql);
queryStatement.setSourceId(sourceId);
return queryStatement;
}
}
} catch (Exception e) {
log.error("physicalSql error {}", e);
queryStatement.setErrMsg(e.getMessage());
}
return queryStatement;
}
public QueryStatement parser(MetricReq metricCommand) {
return parser(metricCommand, true);
}
public QueryStatement parser(MetricReq metricCommand, boolean isAgg) {
log.info("parser MetricReq [{}] isAgg [{}]", metricCommand, isAgg);
QueryStatement queryStatement = new QueryStatement();
if (metricCommand.getRootPath().isEmpty()) {
queryStatement.setErrMsg("rootPath empty");
return queryStatement;
}
try {
queryStatement = ComponentFactory.getSqlParser().explain(metricCommand, isAgg, catalog);
return queryStatement;
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error MetricCommand[{}] error [{}]", metricCommand, e);
}
return queryStatement;
}
private String formatWhere(String where) {
if (StringUtils.isEmpty(where)) {
return where;
}
return where.replace("\"", "\\\\\"");
}
}

15
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticConverter.java Normal file
View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.semantic.query.application.parser;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
public interface SemanticConverter {
boolean accept(QueryStructReq queryStructCmd);
void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend, MetricReq metricCommand)
throws Exception;
}

32
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSqlServiceImpl.java
View File

@@ -1,32 +0,0 @@
package com.tencent.supersonic.semantic.query.application.parser;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.domain.parser.SemanticSqlService;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import org.springframework.stereotype.Service;
@Service("SemanticSqlService")
public class SemanticSqlServiceImpl implements SemanticSqlService {
@Override
public SqlParserResp explain(MetricReq metricReq, boolean isAgg, SemanticModel semanticModel) throws Exception {
SqlParserResp sqlParserResp = new SqlParserResp();
SemanticSchema semanticSchema = getSemanticSchema(semanticModel);
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
aggBuilder.explain(metricReq, isAgg);
sqlParserResp.setSql(aggBuilder.getSql());
sqlParserResp.setSourceId(aggBuilder.getSourceId());
return sqlParserResp;
}
private SemanticSchema getSemanticSchema(SemanticModel semanticModel) {
SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getRootPath()).build();
semanticSchema.setDatasource(semanticModel.getDatasourceMap());
semanticSchema.setDimension(semanticModel.getDimensionMap());
semanticSchema.setMetric(semanticModel.getMetrics());
return semanticSchema;
}
}

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SqlParser.java Normal file
View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.semantic.query.application.parser;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
public interface SqlParser {
QueryStatement explain(MetricReq metricReq, boolean isAgg, Catalog catalog) throws Exception;
}

45
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/CalciteSqlParser.java Normal file
View File

@@ -0,0 +1,45 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SqlParser;
import com.tencent.supersonic.semantic.query.application.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
@Component("CalciteSqlParser")
public class CalciteSqlParser implements SqlParser {
private final SemanticSchemaManager semanticSchemaManager;
public CalciteSqlParser(
SemanticSchemaManager semanticSchemaManager) {
this.semanticSchemaManager = semanticSchemaManager;
}
@Override
public QueryStatement explain(MetricReq metricReq, boolean isAgg, Catalog catalog) throws Exception {
QueryStatement queryStatement = new QueryStatement();
SemanticModel semanticModel = semanticSchemaManager.get(metricReq.getRootPath());
if (semanticModel == null) {
queryStatement.setErrMsg("semanticSchema not found");
return queryStatement;
}
SemanticSchema semanticSchema = getSemanticSchema(semanticModel);
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
aggBuilder.explain(metricReq, isAgg);
queryStatement.setSql(aggBuilder.getSql());
queryStatement.setSourceId(aggBuilder.getSourceId());
return queryStatement;
}
private SemanticSchema getSemanticSchema(SemanticModel semanticModel) {
SemanticSchema semanticSchema = SemanticSchema.newBuilder(semanticModel.getRootPath()).build();
semanticSchema.setDatasource(semanticModel.getDatasourceMap());
semanticSchema.setDimension(semanticModel.getDimensionMap());
semanticSchema.setMetric(semanticModel.getMetrics());
return semanticSchema;
}
}

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/Configuration.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/Configuration.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor;
package com.tencent.supersonic.semantic.query.application.parser.calcite;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.DSLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSqlDialect;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.DSLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSqlDialect;
import java.util.Properties;
import org.apache.calcite.avatica.util.Casing;
import org.apache.calcite.avatica.util.Quoting;

136
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/SemanticSchemaManagerImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/SemanticSchemaManager.java
View File

@@ -1,5 +1,4 @@
package com.tencent.supersonic.semantic.query.application.parser;
package com.tencent.supersonic.semantic.query.application.parser.calcite;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -11,23 +10,22 @@ import com.tencent.supersonic.semantic.api.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.core.domain.DatasourceService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.query.domain.parser.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Dimension;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Identify;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Measure;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Metric;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.MetricTypeParams;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.MetricTypeParams;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -38,35 +36,74 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j
@Primary
@Service("SemanticSchemaManager")
public class SemanticSchemaManagerImpl implements SemanticSchemaManager {
@Service
public class SemanticSchemaManager {
private final DatasourceService datasourceService;
private final DomainService domainService;
@Autowired
private LoadingCache<String, SemanticModel> loadingCache;
private final Catalog catalog;
public SemanticSchemaManager(Catalog catalog) {
this.catalog = catalog;
}
public SemanticSchemaManagerImpl(DatasourceService datasourceService,
DomainService domainService) {
this.datasourceService = datasourceService;
this.domainService = domainService;
public SemanticModel reload(String rootPath) {
SemanticModel semanticModel = new SemanticModel();
semanticModel.setRootPath(rootPath);
Map<Long, String> domainFullPathMap = catalog.getDomainFullPath();
log.info("domainFullPathMap {}", domainFullPathMap);
Set<Long> domainIds = domainFullPathMap.entrySet().stream().filter(e -> e.getValue().startsWith(rootPath))
.map(e -> e.getKey()).collect(Collectors.toSet());
if (domainIds.isEmpty()) {
log.error("get domainId empty {}", rootPath);
return semanticModel;
}
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DatasourceYamlTpl> datasourceYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
catalog.getModelYamlTplByDomainIds(domainIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls);
if (!datasourceYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = datasourceYamlTpls.stream().map(d -> getDatasource(d))
.collect(Collectors.toMap(DataSource::getName, item -> item));
semanticModel.setDatasourceMap(dataSourceMap);
}
if (!dimensionYamlTpls.isEmpty()) {
Map<String, List<Dimension>> dimensionMap = new HashMap<>();
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
}
semanticModel.setDimensionMap(dimensionMap);
}
if (!metricYamlTpls.isEmpty()) {
semanticModel.setMetrics(getMetrics(metricYamlTpls));
}
return semanticModel;
}
//private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>();
public SemanticModel get(String rootPath) throws Exception {
rootPath = formatKey(rootPath);
SemanticModel schema = loadingCache.get(rootPath);
if (schema == null) {
return null;
}
return schema;
}
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
return getMetricsByMetricYamlTpl(t);
}
public static List<Dimension> getDimensions(final List<DimensionYamlTpl> t) {
return getDimension(t);
}
public static DataSource getDatasource(final DatasourceYamlTpl d) {
DataSource datasource = new DataSource();
datasource.setSqlQuery(d.getSqlQuery());
@@ -150,6 +187,7 @@ public class SemanticSchemaManagerImpl implements SemanticSchemaManager {
return identifies;
}
public static void update(SemanticSchema schema, List<Metric> metric) throws Exception {
if (schema != null) {
updateMetric(metric, schema.getMetrics());
@@ -159,7 +197,7 @@ public class SemanticSchemaManagerImpl implements SemanticSchemaManager {
public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) throws Exception {
if (schema != null) {
String dataSourceName = datasourceYamlTpl.getName();
Optional<Map.Entry<String, DataSource>> datasourceYamlTplMap = schema.getDatasource().entrySet().stream()
Optional<Entry<String, DataSource>> datasourceYamlTplMap = schema.getDatasource().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
if (datasourceYamlTplMap.isPresent()) {
datasourceYamlTplMap.get().setValue(datasourceYamlTpl);
@@ -225,51 +263,6 @@ public class SemanticSchemaManagerImpl implements SemanticSchemaManager {
return key;
}
@Override
public SemanticModel reload(String rootPath) {
SemanticModel semanticModel = new SemanticModel();
semanticModel.setRootPath(rootPath);
Map<Long, String> domainFullPathMap = domainService.getDomainFullPath();
log.info("domainFullPathMap {}", domainFullPathMap);
Set<Long> domainIds = domainFullPathMap.entrySet().stream().filter(e -> e.getValue().startsWith(rootPath))
.map(e -> e.getKey()).collect(Collectors.toSet());
if (domainIds.isEmpty()) {
log.error("get domainId empty {}", rootPath);
return semanticModel;
}
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DatasourceYamlTpl> datasourceYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
datasourceService.getModelYamlTplByDomainIds(domainIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls);
if (!datasourceYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = datasourceYamlTpls.stream().map(d -> getDatasource(d))
.collect(Collectors.toMap(DataSource::getName, item -> item));
semanticModel.setDatasourceMap(dataSourceMap);
}
if (!dimensionYamlTpls.isEmpty()) {
Map<String, List<Dimension>> dimensionMap = new HashMap<>();
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
}
semanticModel.setDimensionMap(dimensionMap);
}
if (!metricYamlTpls.isEmpty()) {
semanticModel.setMetrics(getMetrics(metricYamlTpls));
}
return semanticModel;
}
//private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>();
@Override
public SemanticModel get(String rootPath) throws Exception {
rootPath = formatKey(rootPath);
SemanticModel schema = loadingCache.get(rootPath);
if (schema == null) {
return null;
}
return schema;
}
@Configuration
@EnableCaching
public class GuavaCacheConfig {
@@ -290,7 +283,7 @@ public class SemanticSchemaManagerImpl implements SemanticSchemaManager {
@Override
public SemanticModel load(String key) {
log.info("load SemanticSchema [{}]", key);
return SemanticSchemaManagerImpl.this.reload(key);
return SemanticSchemaManager.this.reload(key);
}
}
);
@@ -298,5 +291,4 @@ public class SemanticSchemaManagerImpl implements SemanticSchemaManager {
}
}
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/Constants.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Constants.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
public class Constants {

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/DataSource.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/DataSource.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
import java.util.List;
import lombok.Data;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/Dimension.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Dimension.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticItem;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticItem;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/DimensionTimeTypeParams.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/DimensionTimeTypeParams.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/Identify.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Identify.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
import lombok.AllArgsConstructor;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/Measure.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Measure.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
import lombok.AllArgsConstructor;
import lombok.Data;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/Metric.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/Metric.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticItem;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticItem;
import java.util.List;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/MetricTypeParams.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/MetricTypeParams.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
import java.util.List;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/dsl/SemanticModel.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/dsl/SemanticModel.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.dsl;
package com.tencent.supersonic.semantic.query.application.parser.calcite.dsl;
import java.util.ArrayList;
import java.util.HashMap;

22
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/planner/AggPlanner.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/planner/AggPlanner.java
View File

@@ -1,17 +1,17 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.planner;
package com.tencent.supersonic.semantic.query.application.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.Renderer;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.TableView;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.render.FilterRender;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.render.OutputRender;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.render.SourceRender;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render.FilterRender;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render.OutputRender;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render.SourceRender;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/planner/Planner.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/planner/Planner.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.planner;
package com.tencent.supersonic.semantic.query.application.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/schema/DataSourceTable.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/DataSourceTable.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.schema;
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
import java.util.ArrayList;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/schema/SchemaBuilder.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SchemaBuilder.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.domain.parser.schema;
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.Configuration;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.DSLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.application.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.DSLSqlValidatorImpl;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SemanticItem.java Normal file
View File

@@ -0,0 +1,6 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
public interface SemanticItem {
public String getName();
}

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/schema/SemanticSchema.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SemanticSchema.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.semantic.query.domain.parser.schema;
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Dimension;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Metric;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.SemanticModel;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/schema/SemanticSqlConformance.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SemanticSqlConformance.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.schema;
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
import org.apache.calcite.sql.fun.SqlLibrary;
import org.apache.calcite.sql.validate.SqlConformance;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/schema/SemanticSqlDialect.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/schema/SemanticSqlDialect.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.schema;
package com.tencent.supersonic.semantic.query.application.parser.calcite.schema;
import com.google.common.base.Preconditions;
import org.apache.calcite.avatica.util.Casing;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/DSLSqlValidatorImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/DSLSqlValidatorImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.sql.SqlOperatorTable;

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/Optimization.java Normal file
View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
public interface Optimization {
public void visit(SemanticNode semanticNode);
}

20
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/Renderer.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/Renderer.java
View File

@@ -1,16 +1,16 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Dimension;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Identify;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Measure;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Metric;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/TableView.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/TableView.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql;
import java.util.ArrayList;
import java.util.List;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/AggFunctionNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/AggFunctionNode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/DataSourceNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.Configuration;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Constants;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Dimension;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@@ -34,7 +34,7 @@ public class DataSourceNode extends SemanticNode {
if (sqlTable.isEmpty()) {
throw new Exception("DatasourceNode build error [tableSqlNode not found]");
}
return SemanticNode.buildAs(datasource.getName(), getTable(sqlTable, scope));
return buildAs(datasource.getName(), getTable(sqlTable, scope));
}
private static SqlNode getTable(String sqlQuery, SqlValidatorScope scope) throws Exception {
@@ -143,7 +143,7 @@ public class DataSourceNode extends SemanticNode {
if (metricCommand.getWhere() != null && !metricCommand.getWhere().isEmpty()) {
Set<String> whereFields = new HashSet<>();
SqlNode sqlNode = SemanticNode.parse(metricCommand.getWhere(), scope);
SqlNode sqlNode = parse(metricCommand.getWhere(), scope);
FilterNode.getFilterField(sqlNode, whereFields);
}
return isAllMatch;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/DimensionNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/DimensionNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import java.util.List;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/FilterNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/FilterNode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import java.util.Set;
import org.apache.calcite.sql.SqlBasicCall;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/IdentifyNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/IdentifyNode.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/JoinNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/JoinNode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import lombok.Data;
import org.apache.calcite.sql.SqlNode;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/MeasureNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/MeasureNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Measure;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Measure;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/MetricNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/MetricNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/node/SemanticNode.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/node/SemanticNode.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.Configuration;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.Optimization;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSqlDialect;
import com.tencent.supersonic.semantic.query.application.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Optimization;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSqlDialect;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;

21
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/render/FilterRender.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/render/FilterRender.java
View File

@@ -1,18 +1,17 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.render;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.Renderer;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.TableView;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Constants;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Metric;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;

28
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/render/JoinRender.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/render/JoinRender.java
View File

@@ -1,19 +1,19 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.render;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.Renderer;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.TableView;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.AggFunctionNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Constants;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Dimension;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Identify;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Metric;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.AggFunctionNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/render/OutputRender.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/render/OutputRender.java
View File

@@ -1,13 +1,13 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.render;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.Renderer;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.TableView;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.List;
import org.apache.calcite.sql.SqlNode;

105
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/render/SourceRender.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/calcite/sql/render/SourceRender.java
View File

@@ -1,23 +1,24 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.render;
package com.tencent.supersonic.semantic.query.application.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.Renderer;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.TableView;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.DimensionNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.IdentifyNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Constants;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.DataSource;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Dimension;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Identify;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Measure;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.Metric;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.DimensionNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.application.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.IdentifyNode;
import com.tencent.supersonic.semantic.query.application.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -69,6 +70,9 @@ public class SourceRender extends Renderer {
}
}
for (String dimension : queryDimensions) {
if(dimension.contains(Constants.DIMENSION_IDENTIFY) && queryDimensions.contains(dimension.split(Constants.DIMENSION_IDENTIFY)[1])){
continue;
}
buildDimension(dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension : "",
dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension.split(Constants.DIMENSION_IDENTIFY)[1]
: dimension, datasource, schema, nonAgg, dataSet, output, scope);
@@ -209,39 +213,48 @@ public class SourceRender extends Renderer {
if (queryDimensions.contains(field) || queryMetrics.contains(field)) {
continue;
}
Optional<Dimension> dimension = datasource.getDimensions().stream()
String filterField = field;
if(field.contains(Constants.DIMENSION_IDENTIFY)) {
filterField = field.split(Constants.DIMENSION_IDENTIFY)[1];
}
addField(filterField,field,datasource,schema,dimensions,metrics);
}
}
private static void addField(String field,String oriField,DataSource datasource, SemanticSchema schema, Set<String> dimensions,
Set<String> metrics) {
Optional<Dimension> dimension = datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
if (dimension.isPresent()) {
dimensions.add(oriField);
return;
}
Optional<Identify> identify = datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(field)).findFirst();
if (identify.isPresent()) {
dimensions.add(oriField);
return;
}
if (schema.getDimension().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim = schema.getDimension().get(datasource.getName()).stream()
.filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
if (dimension.isPresent()) {
dimensions.add(field);
continue;
}
Optional<Identify> identify = datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(field)).findFirst();
if (identify.isPresent()) {
dimensions.add(field);
continue;
}
if (schema.getDimension().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim = schema.getDimension().get(datasource.getName()).stream()
.filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
if (dataSourceDim.isPresent()) {
dimensions.add(field);
continue;
}
}
Optional<Measure> metric = datasource.getMeasures()
.stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst();
if (metric.isPresent()) {
metrics.add(field);
continue;
}
Optional<Metric> datasourceMetric = schema.getMetrics()
.stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst();
if (datasourceMetric.isPresent()) {
metrics.add(field);
continue;
if (dataSourceDim.isPresent()) {
dimensions.add(oriField);
return;
}
}
Optional<Measure> metric = datasource.getMeasures()
.stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst();
if (metric.isPresent()) {
metrics.add(oriField);
return;
}
Optional<Metric> datasourceMetric = schema.getMetrics()
.stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst();
if (datasourceMetric.isPresent()) {
metrics.add(oriField);
return;
}
}
public static boolean isDimension(String name, DataSource datasource, SemanticSchema schema) {

123
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/calculate/CalculateConverterRatio.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/convert/CalculateConverterAgg.java
View File

@@ -1,14 +1,19 @@
package com.tencent.supersonic.semantic.query.domain.utils.calculate;
package com.tencent.supersonic.semantic.query.application.parser.convert;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.common.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.utils.QueryStructUtils;
import com.tencent.supersonic.semantic.query.domain.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -23,15 +28,18 @@ import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/**
*
*/
@Component("CalculateConverterAgg")
@Slf4j
@Component
public class CalculateConverterRatio implements CalculateConverter {
public class CalculateConverterAgg implements SemanticConverter {
private final DomainService domainService;
private final SemanticQueryEngine parserService;
private final QueryStructUtils queryStructUtils;
private final SqlGenerateUtils sqlGenerateUtils;
private final Catalog catalog;
@Value("${metricParser.agg.default:sum}")
private String metricAggDefault;
@Value("${metricParser.agg.ratio_roll.name:ratio_roll}")
private String metricAggRatioRollName;
@@ -49,14 +57,89 @@ public class CalculateConverterRatio implements CalculateConverter {
Arrays.asList(metricAggRatioOverWeek, metricAggRatioOverMonth, metricAggRatioOverQuarter,
metricAggRatioOverYear));
public CalculateConverterRatio(DomainService domainService,
@Lazy QueryStructUtils queryStructUtils) {
this.domainService = domainService;
public CalculateConverterAgg(
SemanticQueryEngine parserService,
@Lazy QueryStructUtils queryStructUtils,
SqlGenerateUtils sqlGenerateUtils, Catalog catalog) {
this.parserService = parserService;
this.queryStructUtils = queryStructUtils;
this.sqlGenerateUtils = sqlGenerateUtils;
this.catalog = catalog;
}
public ParseSqlReq generateSqlCommend(QueryStructReq queryStructCmd) throws Exception {
// 同环比
if (isRatioAccept(queryStructCmd)) {
return generateRatioSqlCommand(queryStructCmd);
}
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(catalog.getDomainFullPath(queryStructCmd.getDomainId()));
String metricTableName = "metric_tb";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructCmd.getMetrics());
metricTable.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAgg(true);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
sqlGenerateUtils.getLimit(queryStructCmd));
sqlCommand.setSql(sql);
return sqlCommand;
}
@Override
public boolean accept(QueryStructReq queryStructCmd) {
if (queryStructCmd.getNativeQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {
return false;
}
//todo ck类型暂不拼with语句
if (queryStructCmd.getDomainId().equals(34L)) {
return false;
}
int nonSumFunction = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) {
return false;
}
if (agg.getFunc().equals(AggOperatorEnum.UNKNOWN)) {
return false;
}
if (agg.getFunc() != null
// && !agg.getFunc().equalsIgnoreCase(MetricAggDefault)
) {
nonSumFunction++;
}
}
return nonSumFunction > 0;
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd);
sqlCommend.setSql(parseSqlReq.getSql());
sqlCommend.setTables(parseSqlReq.getTables());
sqlCommend.setRootPath(parseSqlReq.getRootPath());
sqlCommend.setVariables(parseSqlReq.getVariables());
}
/**
* Ratio
*/
public boolean isRatioAccept(QueryStructReq queryStructCmd) {
Long ratioFuncNum = queryStructCmd.getAggregators().stream()
.filter(f -> f.getArgs() != null && f.getArgs().get(0) != null
&& metricAggRatioRollName.equalsIgnoreCase(f.getArgs().get(0))).count();
@@ -66,20 +149,10 @@ public class CalculateConverterRatio implements CalculateConverter {
return false;
}
public SqlParserResp getSqlParser(QueryStructReq queryStructCmd) throws Exception {
return null;
}
/**
* @param queryStructCmd
* @return
*/
public ParseSqlReq generateSqlCommand(QueryStructReq queryStructCmd) throws Exception {
public ParseSqlReq generateRatioSqlCommand(QueryStructReq queryStructCmd) throws Exception {
check(queryStructCmd);
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(domainService.getDomainFullPath(queryStructCmd.getDomainId()));
sqlCommand.setRootPath(catalog.getDomainFullPath(queryStructCmd.getDomainId()));
String metricTableName = "metric_tb";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);

52
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/convert/DefaultDimValueConverter.java Normal file
View File

@@ -0,0 +1,52 @@
package com.tencent.supersonic.semantic.query.application.parser.convert;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
@Component("DefaultDimValueConverter")
public class DefaultDimValueConverter implements SemanticConverter {
@Override
public boolean accept(QueryStructReq queryStructCmd) {
return true;
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend, MetricReq metricCommand) throws Exception {
List<DimensionResp> dimensionResps = catalog.getDimensions(queryStructCmd.getDomainId());
//dimension which has default values
dimensionResps = dimensionResps.stream()
.filter(dimensionResp -> !CollectionUtils.isEmpty(dimensionResp.getDefaultValues()))
.collect(Collectors.toList());
if(CollectionUtils.isEmpty(dimensionResps)){
return;
}
log.info("dimension with default values:{}, queryStruct:{}", dimensionResps, queryStructCmd);
//add dimension default value to filter
List<String> dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream()
.map(Filter::getBizName).collect(Collectors.toList());
for (DimensionResp dimensionResp : dimensionResps) {
if (!dimensionFilterBizName.contains(dimensionResp.getBizName())) {
Filter filter = new Filter();
filter.setBizName(dimensionResp.getBizName());
filter.setValue(dimensionResp.getDefaultValues());
filter.setOperator(FilterOperatorEnum.IN);
filter.setName(dimensionResp.getName());
queryStructCmd.getDimensionFilters().add(filter);
}
}
}
}

51
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/MultiSourceJoinUtils.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/convert/MultiSourceJoin.java
View File

@@ -1,19 +1,21 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.application.parser.convert;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.api.core.pojo.Identify;
import com.tencent.supersonic.semantic.api.core.pojo.Measure;
import com.tencent.supersonic.semantic.api.core.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.core.domain.DatasourceService;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -26,22 +28,14 @@ import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@Component
public class MultiSourceJoinUtils {
@Component("MultiSourceJoin")
public class MultiSourceJoin implements SemanticConverter {
private final Catalog catalog;
private DatasourceService datasourceService;
private DimensionService dimensionService;
private MetricService metricService;
public MultiSourceJoinUtils(DatasourceService datasourceService, DimensionService dimensionService,
MetricService metricService) {
this.datasourceService = datasourceService;
this.dimensionService = dimensionService;
this.metricService = metricService;
public MultiSourceJoin(Catalog catalog) {
this.catalog = catalog;
}
@@ -58,13 +52,13 @@ public class MultiSourceJoinUtils {
}
Long domainId = queryStructCmd.getDomainId();
List<String> aggs = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList());
Map<String, DimensionResp> dimensionMap = dimensionService.getDimensions(domainId).stream()
Map<String, DimensionResp> dimensionMap = catalog.getDimensions(domainId).stream()
.filter(dimensionDesc -> fields.contains(dimensionDesc.getBizName()))
.collect(Collectors.toMap(DimensionResp::getBizName, dimensionDesc -> dimensionDesc));
List<MetricResp> metricDescList = metricService.getMetrics(domainId).stream()
List<MetricResp> metricDescList = catalog.getMetrics(domainId).stream()
.filter(metricDesc -> aggs.contains(metricDesc.getBizName()))
.collect(Collectors.toList());
Map<Long, DatasourceResp> datasourceMap = datasourceService.getDatasourceList(domainId)
Map<Long, DatasourceResp> datasourceMap = catalog.getDatasourceList(domainId)
.stream().collect(Collectors.toMap(DatasourceResp::getId, datasource -> datasource));
//check groups filters and aggs is in same datasource
if (!isInSameDatasource(new ArrayList<>(dimensionMap.values()), metricDescList)) {
@@ -130,5 +124,16 @@ public class MultiSourceJoinUtils {
}
@Override
public boolean accept(QueryStructReq queryStructCmd) {
return true;
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand) throws Exception {
buildJoinPrefix(queryStructCmd);
}
}

72
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/ParserCommandConverter.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/application/parser/convert/ParserDefaultConverter.java
View File

@@ -1,68 +1,56 @@
package com.tencent.supersonic.semantic.query.domain.utils;
package com.tencent.supersonic.semantic.query.application.parser.convert;
import static com.tencent.supersonic.common.constant.Constants.UNDERLINE;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.pojo.Param;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.query.domain.ParserService;
import com.tencent.supersonic.semantic.query.domain.utils.calculate.CalculateConverter;
import com.tencent.supersonic.semantic.query.domain.utils.calculate.CalculateConverterAgg;
import java.util.LinkedList;
import java.util.List;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.domain.utils.QueryStructUtils;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component("ParserDefaultConverter")
@Slf4j
@Component
public class ParserCommandConverter {
public class ParserDefaultConverter implements SemanticConverter {
private final ParserService parserService;
private final DomainService domainService;
private final CalculateConverterAgg calculateCoverterAgg;
private final DimensionService dimensionService;
private final QueryStructUtils queryStructUtils;
@Value("${internal.metric.cnt.suffix:internal_cnt}")
private String internalMetricNameSuffix;
private List<CalculateConverter> calculateCoverters = new LinkedList<>();
public ParserCommandConverter(ParserService parserService,
DomainService domainService,
private final CalculateConverterAgg calculateCoverterAgg;
private final QueryStructUtils queryStructUtils;
public ParserDefaultConverter(
CalculateConverterAgg calculateCoverterAgg,
DimensionService dimensionService,
@Lazy QueryStructUtils queryStructUtils) {
this.parserService = parserService;
this.domainService = domainService;
QueryStructUtils queryStructUtils) {
this.calculateCoverterAgg = calculateCoverterAgg;
this.dimensionService = dimensionService;
this.queryStructUtils = queryStructUtils;
calculateCoverters.add(calculateCoverterAgg);
}
public SqlParserResp getSqlParser(QueryStructReq queryStructCmd) throws Exception {
StatUtils.get().setUseSqlCache(false);
for (CalculateConverter calculateConverter : calculateCoverters) {
if (calculateConverter.accept(queryStructCmd)) {
log.info("getSqlParser {}", calculateConverter.getClass());
return calculateConverter.getSqlParser(queryStructCmd);
}
}
return parserService.physicalSql(generateSqlCommand(queryStructCmd));
@Override
public boolean accept(QueryStructReq queryStructCmd) {
return !calculateCoverterAgg.accept(queryStructCmd);
}
@Override
public void converter(Catalog catalog, QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
MetricReq metricCommand)
throws Exception {
MetricReq metricReq = generateSqlCommand(catalog, queryStructCmd);
BeanUtils.copyProperties(metricReq, metricCommand);
}
public MetricReq generateSqlCommand(QueryStructReq queryStructCmd) {
public MetricReq generateSqlCommand(Catalog catalog, QueryStructReq queryStructCmd) {
MetricReq sqlCommend = new MetricReq();
sqlCommend.setMetrics(queryStructCmd.getMetrics());
sqlCommend.setDimensions(queryStructCmd.getGroups());
@@ -75,13 +63,13 @@ public class ParserCommandConverter {
sqlCommend.setVariables(queryStructCmd.getParams().stream()
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
sqlCommend.setLimit(queryStructCmd.getLimit());
String rootPath = domainService.getDomainFullPath(queryStructCmd.getDomainId());
String rootPath = catalog.getDomainFullPath(queryStructCmd.getDomainId());
sqlCommend.setRootPath(rootPath);
// todo tmp delete
// support detail query
if (queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
String internalMetricName = generateInternalMetricName(queryStructCmd);
String internalMetricName = generateInternalMetricName(catalog, queryStructCmd);
sqlCommend.getMetrics().add(internalMetricName);
}
@@ -89,14 +77,14 @@ public class ParserCommandConverter {
}
public String generateInternalMetricName(QueryStructReq queryStructCmd) {
public String generateInternalMetricName(Catalog catalog, QueryStructReq queryStructCmd) {
String internalMetricNamePrefix = "";
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
log.warn("group is empty!");
} else {
String group = queryStructCmd.getGroups().get(0).equalsIgnoreCase("sys_imp_date")
? queryStructCmd.getGroups().get(1) : queryStructCmd.getGroups().get(0);
DimensionResp dimension = dimensionService.getDimension(group, queryStructCmd.getDomainId());
DimensionResp dimension = catalog.getDimension(group, queryStructCmd.getDomainId());
String datasourceBizName = dimension.getDatasourceBizName();
if (Strings.isNotEmpty(datasourceBizName)) {
internalMetricNamePrefix = datasourceBizName + UNDERLINE;

13
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/ParserService.java
View File

@@ -1,13 +0,0 @@
package com.tencent.supersonic.semantic.query.domain;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
public interface ParserService {
SqlParserResp physicalSql(ParseSqlReq sqlCommend) throws Exception;
SqlParserResp physicalSql(MetricReq metricCommand) throws Exception;
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/SchemaService.java
View File

@@ -18,6 +18,8 @@ public interface SchemaService {
List<DomainResp> getDomainListForAdmin(User user);
List<DomainResp> getDomainListForViewer(User user);
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq, User user);
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);

18
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/SemanticQueryEngine.java Normal file
View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.semantic.query.domain;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.query.application.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
public interface SemanticQueryEngine {
QueryStatement plan(QueryStructReq queryStructCmd) throws Exception;
QueryExecutor route(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(QueryStatement queryStatement);
QueryStatement physicalSql(ParseSqlReq sqlCommend) throws Exception;
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/SemanticSchemaManager.java
View File

@@ -1,12 +0,0 @@
package com.tencent.supersonic.semantic.query.domain.parser;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.SemanticModel;
public interface SemanticSchemaManager {
// get the schema from cache , if not exit , will refresh by the reload function
SemanticModel get(String rootPath) throws Exception;
// refresh cache ,will return the data get from db or other storage
SemanticModel reload(String rootPath);
}

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/SemanticSqlService.java
View File

@@ -1,10 +0,0 @@
package com.tencent.supersonic.semantic.query.domain.parser;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.domain.parser.dsl.SemanticModel;
public interface SemanticSqlService {
SqlParserResp explain(MetricReq metricReq, boolean isAgg, SemanticModel semanticModel) throws Exception;
}

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/convertor/sql/Optimization.java
View File

@@ -1,9 +0,0 @@
package com.tencent.supersonic.semantic.query.domain.parser.convertor.sql;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.sql.node.SemanticNode;
public interface Optimization {
public void visit(SemanticNode semanticNode);
}

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/parser/schema/SemanticItem.java
View File

@@ -1,6 +0,0 @@
package com.tencent.supersonic.semantic.query.domain.parser.schema;
public interface SemanticItem {
public String getName();
}

23
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/pojo/QueryStatement.java Normal file
View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.semantic.query.domain.pojo;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import lombok.Data;
@Data
public class QueryStatement {
private Long domainId = 0L;
private String sql = "";
private String sourceId = "";
private String errMsg = "";
private Boolean ok;
public boolean isOk() {
this.ok = "".equals(errMsg) && !"".equals(sql);
return ok;
}
public QueryStatement error(String msg) {
this.setErrMsg(msg);
return this;
}
}

79
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/ComponentFactory.java Normal file
View File

@@ -0,0 +1,79 @@
package com.tencent.supersonic.semantic.query.domain.utils;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.semantic.query.application.executor.JdbcExecutor;
import com.tencent.supersonic.semantic.query.application.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.application.optimizer.DetailQuery;
import com.tencent.supersonic.semantic.query.application.optimizer.QueryOptimizer;
import com.tencent.supersonic.semantic.query.application.parser.SemanticConverter;
import com.tencent.supersonic.semantic.query.application.parser.SqlParser;
import com.tencent.supersonic.semantic.query.application.parser.calcite.CalciteSqlParser;
import com.tencent.supersonic.semantic.query.application.parser.convert.CalculateConverterAgg;
import com.tencent.supersonic.semantic.query.application.parser.convert.DefaultDimValueConverter;
import com.tencent.supersonic.semantic.query.application.parser.convert.MultiSourceJoin;
import com.tencent.supersonic.semantic.query.application.parser.convert.ParserDefaultConverter;
import java.util.ArrayList;
import java.util.List;
public class ComponentFactory {
private static List<SemanticConverter> semanticConverters = new ArrayList<>();
private static List<QueryExecutor> queryExecutors = new ArrayList<>();
private static List<QueryOptimizer> queryOptimizers = new ArrayList<>();
private static SqlParser sqlParser;
static {
initSemanticConverter();
initQueryExecutors();
initQueryOptimizer();
}
public static List<SemanticConverter> getSemanticConverters() {
if (semanticConverters.isEmpty()) {
initSemanticConverter();
}
return semanticConverters;
}
public static List<QueryExecutor> getQueryExecutors() {
if (queryExecutors.isEmpty()) {
initQueryExecutors();
}
return queryExecutors;
}
public static List<QueryOptimizer> getQueryOptimizers() {
if(queryOptimizers.isEmpty()){
initQueryOptimizer();
}
return queryOptimizers;
}
public static SqlParser getSqlParser() {
if (sqlParser == null) {
sqlParser = ContextUtils.getContext().getBean("CalciteSqlParser", CalciteSqlParser.class);
}
return sqlParser;
}
public static void setSqlParser(SqlParser parser) {
sqlParser = parser;
}
private static void initQueryOptimizer() {
queryOptimizers.add(getBean("DetailQuery", DetailQuery.class));
}
private static void initSemanticConverter() {
semanticConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
semanticConverters.add(getBean("CalculateConverterAgg", CalculateConverterAgg.class));
semanticConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
semanticConverters.add(getBean("MultiSourceJoin", MultiSourceJoin.class));
}
private static void initQueryExecutors() {
queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", JdbcExecutor.class));
}
public static <T> T getBean(String name, Class<T> tClass) {
return ContextUtils.getContext().getBean(name, tClass);
}
}

36
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/DateUtils.java
View File

@@ -131,14 +131,33 @@ public class DateUtils {
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, start, sysDateCol, dateDate.getEndDate());
}
public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
String endStr = endData.format(formatter);
String start = endData.minusMonths(unit).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, start, sysDateMonthCol, endStr);
}
public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) {
if (dateDate.getDatePeriod() != null && MONTH.equalsIgnoreCase(dateDate.getDatePeriod())) {
Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateDate.getDateFormat(),
ChronoUnit.MONTHS);
LocalDate endData = LocalDate.parse(dateDate.getEndDate(),
DateTimeFormatter.ofPattern(dateDate.getDateFormat()));
return generateMonthSql(endData, unit, dateDate.getDateFormat());
}
String dateFormatStr = MONTH_FORMAT;
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
LocalDate end = LocalDate.parse(dateDate.getEndDate(), formatter);
String endStr = end.format(formatter);
Integer unit = dateInfo.getUnit() - 1;
String start = end.minusMonths(unit).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, start, sysDateMonthCol, endStr);
return recentMonthStr(end, Long.valueOf(unit), dateFormatStr);
}
public String recentWeekStr(LocalDate endData, Long unit) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DAY_FORMAT);
String start = endData.minusDays(unit * 7).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, start, sysDateWeekCol,
endData.format(formatter));
}
public String recentWeekStr(ItemDateResp dateDate, DateConf dateInfo) {
@@ -149,9 +168,7 @@ public class DateUtils {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
LocalDate end = LocalDate.parse(dateDate.getEndDate(), formatter);
Integer unit = dateInfo.getUnit() - 1;
String start = end.minusDays(unit * 7).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, start, sysDateWeekCol,
dateDate.getEndDate());
return recentWeekStr(end, Long.valueOf(unit));
}
private Long getInterval(String startDate, String endDate, String dateFormat, ChronoUnit chronoUnit) {
@@ -224,7 +241,12 @@ public class DateUtils {
if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
return generateMonthSql(dateMax, unit.longValue(), DAY_FORMAT);
//return generateMonthSql(dateMax, unit.longValue(), DAY_FORMAT);
return recentMonthStr(dateMax, unit.longValue(), MONTH_FORMAT);
}
if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
return recentWeekStr(dateMax, unit.longValue());
}
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, LocalDate.now().minusDays(2), sysDateCol,

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/QueryReqConverter.java
View File

@@ -8,7 +8,8 @@ import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.query.domain.ParserService;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -25,9 +26,9 @@ public class QueryReqConverter {
private DomainService domainService;
@Autowired
private ParserService parserService;
private SemanticQueryEngine parserService;
public SqlParserResp convert(QuerySqlReq databaseReq, List<DomainSchemaResp> domainSchemas) throws Exception {
public QueryStatement convert(QuerySqlReq databaseReq, List<DomainSchemaResp> domainSchemas) throws Exception {
List<MetricTable> tables = new ArrayList<>();
MetricTable metricTable = new MetricTable();
@@ -37,7 +38,7 @@ public class QueryReqConverter {
List<String> allFields = sqlParseInfo.getAllFields();
if (CollectionUtils.isEmpty(domainSchemas)) {
return new SqlParserResp();
return new QueryStatement();
}
Set<String> dimensions = domainSchemas.get(0).getDimensions().stream()

183
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/QueryStructUtils.java
View File

@@ -1,27 +1,15 @@
package com.tencent.supersonic.semantic.query.domain.utils;
import static com.tencent.supersonic.common.constant.Constants.UNIONALL;
import com.tencent.supersonic.semantic.api.core.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.core.domain.DatabaseService;
import com.tencent.supersonic.semantic.core.domain.DatasourceService;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.semantic.query.domain.ParserService;
import com.tencent.supersonic.semantic.api.core.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -29,172 +17,37 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@Component
public class QueryStructUtils {
public static Set<String> internalCols = new HashSet<>(
Arrays.asList("dayno", "plat_sys_var", "sys_imp_date", "sys_imp_week", "sys_imp_month"));
private final DatabaseService databaseService;
private final QueryUtils queryUtils;
private final ParserService parserService;
private final SqlParserUtils sqlParserUtils;
private final StatUtils statUtils;
private final DimensionService dimensionService;
private final MetricService metricService;
private final DatasourceService datasourceService;
private final DateUtils dateUtils;
private final SqlFilterUtils sqlFilterUtils;
private final CacheUtils cacheUtils;
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
private final Catalog catalog;
public QueryStructUtils(DatabaseService databaseService,
QueryUtils queryUtils,
ParserService parserService,
SqlParserUtils sqlParserUtils,
StatUtils statUtils,
DimensionService dimensionService,
MetricService metricService,
DatasourceService datasourceService,
public static Set<String> internalCols = new HashSet<>(
Arrays.asList("dayno", "plat_sys_var", "sys_imp_date", "sys_imp_week", "sys_imp_month"));
public QueryStructUtils(
DateUtils dateUtils,
SqlFilterUtils sqlFilterUtils,
CacheUtils cacheUtils) {
this.databaseService = databaseService;
this.queryUtils = queryUtils;
this.parserService = parserService;
this.sqlParserUtils = sqlParserUtils;
this.statUtils = statUtils;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.datasourceService = datasourceService;
SqlFilterUtils sqlFilterUtils, Catalog catalog) {
this.dateUtils = dateUtils;
this.sqlFilterUtils = sqlFilterUtils;
this.cacheUtils = cacheUtils;
this.catalog = catalog;
}
public QueryResultWithSchemaResp queryByStructByCache(QueryStructReq queryStructCmd, String key) throws Exception {
QueryResultWithSchemaResp queryResultWithColumns;
Object resultObject = cacheUtils.get(key);
if (Objects.nonNull(resultObject)) {
log.info("queryByStructWithCache, key:{}, queryStructCmd:{}", key, queryStructCmd.toString());
statUtils.updateResultCacheKey(key);
return (QueryResultWithSchemaResp) resultObject;
}
// if cache data is null, query database
queryResultWithColumns = queryByStructWithoutCache(queryStructCmd, key);
return queryResultWithColumns;
}
public boolean queryCache(Cache cacheInfo) {
if (Objects.isNull(cacheInfo)) {
return true;
}
return Objects.nonNull(cacheInfo)
&& cacheInfo.getCache();
}
public QueryResultWithSchemaResp queryByStructWithoutCache(QueryStructReq queryStructCmd, String key)
throws Exception {
log.info("stat queryByStructWithoutCache, queryStructCmd:{}", queryStructCmd);
StatUtils.get().setUseResultCache(false);
SqlParserResp sqlParser = getSqlParser(queryStructCmd);
queryUtils.checkSqlParse(sqlParser);
log.info("sqlParser:{}", sqlParser);
// todo tmp delete
//queryUtils.handleDetail(queryStructCmd, sqlParser);
queryUtils.handleNoMetric(queryStructCmd, sqlParser);
QueryResultWithSchemaResp queryResultWithColumns = databaseService.queryWithColumns(sqlParser);
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStructCmd.getDomainId());
queryResultWithColumns.setSql(sqlParser.getSql());
// if queryResultWithColumns is not null, update cache data
cacheResultLogic(key, queryResultWithColumns);
return queryResultWithColumns;
}
private void cacheResultLogic(String key, QueryResultWithSchemaResp queryResultWithColumns) {
if (cacheEnable && Objects.nonNull(queryResultWithColumns) && !CollectionUtils.isEmpty(
queryResultWithColumns.getResultList())) {
QueryResultWithSchemaResp finalQueryResultWithColumns = queryResultWithColumns;
CompletableFuture.supplyAsync(() -> cacheUtils.put(key, finalQueryResultWithColumns))
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
statUtils.updateResultCacheKey(key);
log.info("add record to cache, key:{}", key);
}
}
public QueryResultWithSchemaResp queryByMultiStructWithoutCache(QueryMultiStructReq queryMultiStructCmd, String key)
throws Exception {
log.info("stat queryByStructWithoutCache, queryMultiStructCmd:{}", queryMultiStructCmd);
QueryResultWithSchemaResp queryResultWithColumns;
List<SqlParserResp> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructCmd : queryMultiStructCmd.getQueryStructCmds()) {
SqlParserResp sqlParser = getSqlParser(queryStructCmd);
queryUtils.checkSqlParse(sqlParser);
queryUtils.handleDetail(queryStructCmd, sqlParser);
sqlParsers.add(sqlParser);
}
log.info("multi sqlParser:{}", sqlParsers);
SqlParserResp sqlParser = sqlParserUnion(queryMultiStructCmd, sqlParsers);
queryResultWithColumns = databaseService.queryWithColumns(sqlParser);
queryUtils.fillItemNameInfo(queryResultWithColumns, queryMultiStructCmd);
queryResultWithColumns.setSql(sqlParser.getSql());
cacheResultLogic(key, queryResultWithColumns);
return queryResultWithColumns;
}
private SqlParserResp sqlParserUnion(QueryMultiStructReq queryMultiStructCmd, List<SqlParserResp> sqlParsers) {
SqlParserResp sqlParser = new SqlParserResp();
StringBuilder unionSqlBuilder = new StringBuilder();
for (int i = 0; i < sqlParsers.size(); i++) {
String selectStr = SqlGenerateUtils.getUnionSelect(queryMultiStructCmd.getQueryStructCmds().get(i));
unionSqlBuilder.append(String.format("select %s from ( %s ) sub_sql_%s",
selectStr,
sqlParsers.get(i).getSql(), i));
unionSqlBuilder.append(UNIONALL);
}
String unionSql = unionSqlBuilder.substring(0, unionSqlBuilder.length() - Constants.UNIONALL.length());
sqlParser.setSql(unionSql);
sqlParser.setSourceId(sqlParsers.get(0).getSourceId());
log.info("union sql parser:{}", sqlParser);
return sqlParser;
}
private SqlParserResp getSqlParser(QueryStructReq queryStructCmd) throws Exception {
return sqlParserUtils.getSqlParserWithoutCache(queryStructCmd);
}
private List<Long> getDimensionIds(QueryStructReq queryStructCmd) {
List<Long> dimensionIds = new ArrayList<>();
List<DimensionResp> dimensions = dimensionService.getDimensions(queryStructCmd.getDomainId());
List<DimensionResp> dimensions = catalog.getDimensions(queryStructCmd.getDomainId());
Map<String, List<DimensionResp>> pair = dimensions.stream()
.collect(Collectors.groupingBy(DimensionResp::getBizName));
for (String group : queryStructCmd.getGroups()) {
@@ -215,7 +68,7 @@ public class QueryStructUtils {
private List<Long> getMetricIds(QueryStructReq queryStructCmd) {
List<Long> metricIds = new ArrayList<>();
List<MetricResp> metrics = metricService.getMetrics(queryStructCmd.getDomainId());
List<MetricResp> metrics = catalog.getMetrics(queryStructCmd.getDomainId());
Map<String, List<MetricResp>> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName));
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (pair.containsKey(agg.getColumn())) {
@@ -239,7 +92,7 @@ public class QueryStructUtils {
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
List<Long> metricIds = getMetricIds(queryStructCmd);
ItemDateResp dateDate = datasourceService.getDateDate(
ItemDateResp dateDate = catalog.getDateDate(
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()),
new ItemDateFilter(metricIds, TypeEnums.METRIC.getName()));
if (Objects.isNull(dateDate)

199
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/QueryUtils.java
View File

@@ -1,24 +1,20 @@
package com.tencent.supersonic.semantic.query.domain.utils;
import static com.tencent.supersonic.common.constant.Constants.END_SUBQUERY;
import static com.tencent.supersonic.common.constant.Constants.GROUP_UPPER;
import static com.tencent.supersonic.common.constant.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.constant.Constants.LIMIT_UPPER;
import static com.tencent.supersonic.common.constant.Constants.ORDER_UPPER;
import static com.tencent.supersonic.common.constant.Constants.SPACE;
import static com.tencent.supersonic.common.constant.Constants.UNIONALL;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.api.core.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -26,13 +22,13 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.concurrent.CompletableFuture;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -42,23 +38,8 @@ import org.springframework.util.CollectionUtils;
public class QueryUtils {
private final Set<Pattern> patterns = new HashSet<>();
private final MetricService metricService;
private final DimensionService dimensionService;
private final ParserCommandConverter parserCommandConverter;
public QueryUtils(MetricService metricService,
DimensionService dimensionService,
@Lazy ParserCommandConverter parserCommandConverter) {
this.metricService = metricService;
this.dimensionService = dimensionService;
this.parserCommandConverter = parserCommandConverter;
}
private static void addSysTimeDimension(Map<String, String> namePair, Map<String, String> nameTypePair) {
for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) {
namePair.put(timeDimensionEnum.getName(), "date");
nameTypePair.put(timeDimensionEnum.getName(), "DATE");
}
}
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
@PostConstruct
public void fillPattern() {
@@ -69,102 +50,26 @@ public class QueryUtils {
}
}
public void checkSqlParse(SqlParserResp sqlParser) {
if (Strings.isNullOrEmpty(sqlParser.getSql()) || Strings.isNullOrEmpty(sqlParser.getSourceId())) {
throw new RuntimeException("parse Exception: " + sqlParser.getErrMsg());
}
private final CacheUtils cacheUtils;
private final StatUtils statUtils;
private final Catalog catalog;
public QueryUtils(
CacheUtils cacheUtils, StatUtils statUtils, Catalog catalog) {
this.cacheUtils = cacheUtils;
this.statUtils = statUtils;
this.catalog = catalog;
}
public boolean isDetailQuery(QueryStructReq queryStructCmd) {
return Objects.nonNull(queryStructCmd) && queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(
queryStructCmd.getMetrics());
}
public SqlParserResp handleNoMetric(QueryStructReq queryStructCmd, SqlParserResp sqlParser) {
String sqlRaw = sqlParser.getSql().trim();
if (Strings.isNullOrEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");
}
log.info("before handleNoMetric, sql:{}", sqlRaw);
if (isDetailQuery(queryStructCmd)) {
if (queryStructCmd.getMetrics().size() == 0) {
String sql = String.format("select %s from ( %s ) src_no_metric",
queryStructCmd.getGroups().stream().collect(Collectors.joining(",")), sqlRaw);
sqlParser.setSql(sql);
}
}
log.info("after handleNoMetric, sql:{}", sqlParser.getSql());
return sqlParser;
}
public SqlParserResp handleDetail(QueryStructReq queryStructCmd, SqlParserResp sqlParser) {
String sqlRaw = sqlParser.getSql().trim();
if (Strings.isNullOrEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");
}
log.info("before handleDetail, sql:{}", sqlRaw);
String sql = sqlRaw;
if (isDetailQuery(queryStructCmd)) {
String internalMetricName = parserCommandConverter.generateInternalMetricName(queryStructCmd);
// select handle
log.info("size:{}, metric:{}, contain:{}", queryStructCmd.getMetrics().size(), queryStructCmd.getMetrics(),
queryStructCmd.getMetrics().contains(internalMetricName));
if (queryStructCmd.getMetrics().size() == 0) {
Set<String> internalCntSet = new HashSet<>(
Arrays.asList(
String.format(", SUM(%s) AS %s", internalMetricName, internalMetricName),
String.format(", %s AS %s", internalMetricName, internalMetricName))
);
for (String target : internalCntSet) {
sql = sql.replace(target, SPACE);
}
} else {
// dimension + metric
for (Pattern pattern : patterns) {
Matcher matcher = pattern.matcher(sql);
while (matcher.find()) {
String target = matcher.group(1);
String replace = matcher.group(2);
sql = sql.replace(target, replace);
}
}
}
// group handle
String groupTarget = "";
if (sql.contains(GROUP_UPPER)) {
String afterLastGroup = sql.substring(sql.lastIndexOf(GROUP_UPPER));
log.info("afterLastGroup:{}", afterLastGroup);
if (!Strings.isNullOrEmpty(afterLastGroup)) {
int tmp = afterLastGroup.length();
if (afterLastGroup.contains(END_SUBQUERY)) {
tmp = afterLastGroup.indexOf(END_SUBQUERY);
} else if (afterLastGroup.contains(ORDER_UPPER)) {
tmp = afterLastGroup.indexOf(ORDER_UPPER);
} else if (afterLastGroup.contains(LIMIT_UPPER)) {
tmp = afterLastGroup.indexOf(LIMIT_UPPER);
}
groupTarget = afterLastGroup.substring(0, tmp);
}
if (!Strings.isNullOrEmpty(groupTarget)) {
sql = sql.replace(groupTarget, SPACE);
}
}
sqlParser.setSql(sql);
}
log.info("after handleDetail, sql:{}", sqlParser.getSql());
return sqlParser;
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, Long domainId) {
List<MetricResp> metricDescList = metricService.getMetrics(domainId);
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(domainId);
List<MetricResp> metricDescList = catalog.getMetrics(domainId);
List<DimensionResp> dimensionDescList = catalog.getDimensions(domainId);
Map<String,MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a,(k1, k2)->k1));
Map<String, String> namePair = new HashMap<>();
Map<String, String> nameTypePair = new HashMap<>();
addSysTimeDimension(namePair, nameTypePair);
@@ -188,6 +93,10 @@ public class QueryUtils {
if (nameTypePair.containsKey(nameEn)) {
column.setShowType(nameTypePair.get(nameEn));
}
if(metricRespMap.containsKey(nameEn)){
column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType());
column.setDataFormat(metricRespMap.get(nameEn).getDataFormat());
}
});
}
@@ -244,4 +153,52 @@ public class QueryUtils {
}
return map;
}
private static void addSysTimeDimension(Map<String, String> namePair, Map<String, String> nameTypePair) {
for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) {
namePair.put(timeDimensionEnum.getName(), "date");
nameTypePair.put(timeDimensionEnum.getName(), "DATE");
}
}
public void checkSqlParse(QueryStatement sqlParser) {
if (com.google.common.base.Strings.isNullOrEmpty(sqlParser.getSql())
|| com.google.common.base.Strings.isNullOrEmpty(sqlParser.getSourceId())) {
throw new RuntimeException("parse Exception: " + sqlParser.getErrMsg());
}
}
public QueryStatement sqlParserUnion(QueryMultiStructReq queryMultiStructCmd, List<QueryStatement> sqlParsers) {
QueryStatement sqlParser = new QueryStatement();
StringBuilder unionSqlBuilder = new StringBuilder();
for (int i = 0; i < sqlParsers.size(); i++) {
String selectStr = SqlGenerateUtils.getUnionSelect(queryMultiStructCmd.getQueryStructCmds().get(i));
unionSqlBuilder.append(String.format("select %s from ( %s ) sub_sql_%s",
selectStr,
sqlParsers.get(i).getSql(), i));
unionSqlBuilder.append(UNIONALL);
}
String unionSql = unionSqlBuilder.substring(0, unionSqlBuilder.length() - Constants.UNIONALL.length());
sqlParser.setSql(unionSql);
sqlParser.setSourceId(sqlParsers.get(0).getSourceId());
log.info("union sql parser:{}", sqlParser);
return sqlParser;
}
public void cacheResultLogic(String key, QueryResultWithSchemaResp queryResultWithColumns) {
if (cacheEnable && Objects.nonNull(queryResultWithColumns) && !CollectionUtils.isEmpty(
queryResultWithColumns.getResultList())) {
QueryResultWithSchemaResp finalQueryResultWithColumns = queryResultWithColumns;
CompletableFuture.supplyAsync(() -> cacheUtils.put(key, finalQueryResultWithColumns))
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
statUtils.updateResultCacheKey(key);
log.info("add record to cache, key:{}", key);
}
}
}

30
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/SqlParserUtils.java
View File

@@ -1,30 +0,0 @@
package com.tencent.supersonic.semantic.query.domain.utils;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class SqlParserUtils {
private final ParserCommandConverter parserCommandConverter;
private final MultiSourceJoinUtils multiSourceJoinUtils;
public SqlParserUtils(ParserCommandConverter parserCommandConverter, MultiSourceJoinUtils multiSourceJoinUtils) {
this.parserCommandConverter = parserCommandConverter;
this.multiSourceJoinUtils = multiSourceJoinUtils;
}
public SqlParserResp getSqlParserWithoutCache(QueryStructReq queryStructCmd) throws Exception {
log.info("stat getSqlParser without cache");
//multiSourceJoinUtils.buildJoinPrefix(queryStructCmd);
SqlParserResp sqlParser = parserCommandConverter.getSqlParser(queryStructCmd);
return sqlParser;
}
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/calculate/CalculateConverter.java
View File

@@ -1,12 +0,0 @@
package com.tencent.supersonic.semantic.query.domain.utils.calculate;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
public interface CalculateConverter {
boolean accept(QueryStructReq queryStructCmd);
SqlParserResp getSqlParser(QueryStructReq queryStructCmd) throws Exception;
}

108
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/domain/utils/calculate/CalculateConverterAgg.java
View File

@@ -1,108 +0,0 @@
package com.tencent.supersonic.semantic.query.domain.utils.calculate;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.query.domain.ParserService;
import com.tencent.supersonic.semantic.query.domain.utils.QueryStructUtils;
import com.tencent.supersonic.semantic.query.domain.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.Collections;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
public class CalculateConverterAgg implements CalculateConverter {
private final DomainService domainService;
private final ParserService parserService;
private final CalculateConverterRatio calculateCoverterRatio;
private final QueryStructUtils queryStructUtils;
private final SqlGenerateUtils sqlGenerateUtils;
@Value("${metricParser.agg.default:sum}")
private String metricAggDefault;
public CalculateConverterAgg(DomainService domainService,
ParserService parserService,
CalculateConverterRatio calculateCoverterRatio,
@Lazy QueryStructUtils queryStructUtils,
SqlGenerateUtils sqlGenerateUtils) {
this.domainService = domainService;
this.parserService = parserService;
this.calculateCoverterRatio = calculateCoverterRatio;
this.queryStructUtils = queryStructUtils;
this.sqlGenerateUtils = sqlGenerateUtils;
}
public ParseSqlReq generateSqlCommend(QueryStructReq queryStructCmd) throws Exception {
// 同环比
if (calculateCoverterRatio.accept(queryStructCmd)) {
return calculateCoverterRatio.generateSqlCommand(queryStructCmd);
}
ParseSqlReq sqlCommand = new ParseSqlReq();
sqlCommand.setRootPath(domainService.getDomainFullPath(queryStructCmd.getDomainId()));
String metricTableName = "metric_tb";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryStructCmd.getMetrics());
metricTable.setDimensions(queryStructCmd.getGroups());
String where = queryStructUtils.generateWhere(queryStructCmd);
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAgg(true);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
metricTableName,
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
sqlGenerateUtils.getLimit(queryStructCmd));
sqlCommand.setSql(sql);
return sqlCommand;
}
@Override
public boolean accept(QueryStructReq queryStructCmd) {
if (queryStructCmd.getNativeQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {
return false;
}
//todo ck类型暂不拼with语句
if (queryStructCmd.getDomainId().equals(34L)) {
return false;
}
int nonSumFunction = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) {
return false;
}
if (agg.getFunc().equals(AggOperatorEnum.UNKNOWN)) {
return false;
}
if (agg.getFunc() != null
// && !agg.getFunc().equalsIgnoreCase(MetricAggDefault)
) {
nonSumFunction++;
}
}
return nonSumFunction > 0;
}
@Override
public SqlParserResp getSqlParser(QueryStructReq queryStructCmd) throws Exception {
return parserService.physicalSql(generateSqlCommend(queryStructCmd));
}
}

11
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -5,12 +5,14 @@ import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.*;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.domain.ParserService;
import com.tencent.supersonic.semantic.query.domain.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.domain.QueryService;
import com.tencent.supersonic.semantic.query.domain.pojo.QueryStatement;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
@@ -26,7 +28,7 @@ public class QueryController {
@Autowired
private QueryService queryService;
@Autowired
private ParserService parserService;
private SemanticQueryEngine semanticQueryEngine;
@PostMapping("/sql")
@@ -52,7 +54,10 @@ public class QueryController {
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return parserService.physicalSql(parseSqlReq);
QueryStatement queryStatement = semanticQueryEngine.physicalSql(parseSqlReq);
SqlParserResp sqlParserResp = new SqlParserResp();
BeanUtils.copyProperties(queryStatement, sqlParserResp);
return sqlParserResp;
}
/**

7
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/SchemaController.java
View File

@@ -50,6 +50,13 @@ public class SchemaController {
return schemaService.getDomainListForAdmin(user);
}
@GetMapping("/domain/view/list")
public List<DomainResp> getDomainListForViewer(HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.getDomainListForViewer(user);
}
@PostMapping("/dimension/page")
public PageInfo<DimensionResp> queryDimension(@RequestBody PageDimensionReq pageDimensionCmd,
HttpServletRequest request,

137
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/parser/SemanticParserServiceTest.java → semantic/query/src/test/java/com/tencent/supersonic/semantic/query/domain/calcite/SemanticParserServiceTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.domain.parser;
package com.tencent.supersonic.semantic.query.domain.calcite;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DimensionTimeTypeParamsTpl;
@@ -10,9 +10,9 @@ import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.api.core.response.SqlParserResp;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.domain.parser.convertor.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.domain.parser.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.application.parser.SemanticSchemaManagerImpl;
import com.tencent.supersonic.semantic.query.application.parser.calcite.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.application.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.application.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -47,66 +47,6 @@ class SemanticParserServiceTest {
return sqlParser;
}
private static void addDepartment(SemanticSchema semanticSchema) {
DatasourceYamlTpl datasource = new DatasourceYamlTpl();
datasource.setName("user_department");
datasource.setSourceId(1L);
datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");
MeasureYamlTpl measure = new MeasureYamlTpl();
measure.setAgg("count");
measure.setName("user_department_internal_cnt");
measure.setCreateMetric("true");
measure.setExpr("1");
List<MeasureYamlTpl> measures = new ArrayList<>();
measures.add(measure);
datasource.setMeasures(measures);
DimensionYamlTpl dimension = new DimensionYamlTpl();
dimension.setName("sys_imp_date");
dimension.setExpr("imp_date");
dimension.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl();
dimensionTimeTypeParams.setIsPrimary("true");
dimensionTimeTypeParams.setTimeGranularity("day");
dimension.setTypeParams(dimensionTimeTypeParams);
List<DimensionYamlTpl> dimensions = new ArrayList<>();
dimensions.add(dimension);
DimensionYamlTpl dimension3 = new DimensionYamlTpl();
dimension3.setName("sys_imp_week");
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
dimension3.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl();
dimensionTimeTypeParams3.setIsPrimary("true");
dimensionTimeTypeParams3.setTimeGranularity("week");
dimension3.setTypeParams(dimensionTimeTypeParams3);
dimensions.add(dimension3);
datasource.setDimensions(dimensions);
List<IdentifyYamlTpl> identifies = new ArrayList<>();
IdentifyYamlTpl identify = new IdentifyYamlTpl();
identify.setName("user_name");
identify.setType("primary");
identifies.add(identify);
datasource.setIdentifiers(identifies);
semanticSchema.getDatasource().put("user_department", SemanticSchemaManagerImpl.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("department");
dimension1.setName("department");
dimension1.setType("categorical");
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1);
semanticSchema.getDimension()
.put("user_department", SemanticSchemaManagerImpl.getDimensions(dimensionYamlTpls));
}
//@Test
public void test() throws Exception {
@@ -180,7 +120,7 @@ class SemanticParserServiceTest {
SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build();
SemanticSchemaManagerImpl.update(semanticSchema, SemanticSchemaManagerImpl.getDatasource(datasource));
SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("page");
@@ -189,8 +129,8 @@ class SemanticParserServiceTest {
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1);
SemanticSchemaManagerImpl.update(semanticSchema, "s2_pv_uv_statis",
SemanticSchemaManagerImpl.getDimensions(dimensionYamlTpls));
SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
SemanticSchemaManager.getDimensions(dimensionYamlTpls));
MetricYamlTpl metric1 = new MetricYamlTpl();
metric1.setName("pv");
@@ -219,7 +159,7 @@ class SemanticParserServiceTest {
metric2.setTypeParams(metricTypeParams1);
metric.add(metric2);
SemanticSchemaManagerImpl.update(semanticSchema, SemanticSchemaManagerImpl.getMetrics(metric));
SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.getMetrics(metric));
MetricReq metricCommand = new MetricReq();
metricCommand.setRootPath("s2");
@@ -249,4 +189,65 @@ class SemanticParserServiceTest {
}
private static void addDepartment(SemanticSchema semanticSchema) {
DatasourceYamlTpl datasource = new DatasourceYamlTpl();
datasource.setName("user_department");
datasource.setSourceId(1L);
datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");
MeasureYamlTpl measure = new MeasureYamlTpl();
measure.setAgg("count");
measure.setName("user_department_internal_cnt");
measure.setCreateMetric("true");
measure.setExpr("1");
List<MeasureYamlTpl> measures = new ArrayList<>();
measures.add(measure);
datasource.setMeasures(measures);
DimensionYamlTpl dimension = new DimensionYamlTpl();
dimension.setName("sys_imp_date");
dimension.setExpr("imp_date");
dimension.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl();
dimensionTimeTypeParams.setIsPrimary("true");
dimensionTimeTypeParams.setTimeGranularity("day");
dimension.setTypeParams(dimensionTimeTypeParams);
List<DimensionYamlTpl> dimensions = new ArrayList<>();
dimensions.add(dimension);
DimensionYamlTpl dimension3 = new DimensionYamlTpl();
dimension3.setName("sys_imp_week");
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
dimension3.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl();
dimensionTimeTypeParams3.setIsPrimary("true");
dimensionTimeTypeParams3.setTimeGranularity("week");
dimension3.setTypeParams(dimensionTimeTypeParams3);
dimensions.add(dimension3);
datasource.setDimensions(dimensions);
List<IdentifyYamlTpl> identifies = new ArrayList<>();
IdentifyYamlTpl identify = new IdentifyYamlTpl();
identify.setName("user_name");
identify.setType("primary");
identifies.add(identify);
datasource.setIdentifiers(identifies);
semanticSchema.getDatasource().put("user_department", SemanticSchemaManager.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("department");
dimension1.setName("department");
dimension1.setType("categorical");
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1);
semanticSchema.getDimension()
.put("user_department", SemanticSchemaManager.getDimensions(dimensionYamlTpls));
}
}