Make some refactoring in Headless (#672)

* [improvement][headless]Move QueryCache from server to core and move yaml-related classes from core to server

* [improvement][headless]Declare QueryParser and QueryPlanner as interface instead of class.

---------

Co-authored-by: LXW
This commit is contained in:
Jun Zhang
2024-01-21 16:14:39 +08:00
committed by GitHub
parent 7afa42b4bc
commit 97c767a45b
37 changed files with 275 additions and 255 deletions

View File

@@ -1,2 +1,2 @@
com.tencent.supersonic.headless.server.cache.CacheManager=\
com.tencent.supersonic.headless.server.cache.CaffeineCacheManager
com.tencent.supersonic.headless.core.cache.CacheManager=\
com.tencent.supersonic.headless.core.cache.CaffeineCacheManager

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.config;
package com.tencent.supersonic.headless.core.cache;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.cache;
package com.tencent.supersonic.headless.core.cache;
public interface CacheManager {

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.config;
package com.tencent.supersonic.headless.core.cache;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;

View File

@@ -1,9 +1,8 @@
package com.tencent.supersonic.headless.server.cache;
package com.tencent.supersonic.headless.core.cache;
import com.github.benmanes.caffeine.cache.Cache;
import com.google.common.base.Joiner;
import com.tencent.supersonic.headless.server.config.CacheCommonConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.cache;
package com.tencent.supersonic.headless.core.cache;
import com.tencent.supersonic.headless.api.pojo.Cache;

View File

@@ -0,0 +1,153 @@
package com.tencent.supersonic.headless.core.parser;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
@Primary
public class DefaultQueryParser implements QueryParser {
public void parse(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
if (Objects.isNull(queryStatement.getParseSqlReq())) {
queryStatement.setParseSqlReq(new ParseSqlReq());
}
if (Objects.isNull(queryStatement.getMetricReq())) {
queryStatement.setMetricReq(new MetricQueryReq());
}
log.info("SemanticConverter before [{}]", queryStructReq);
for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) {
if (headlessConverter.accept(queryStatement)) {
log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.convert(queryStatement);
}
}
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(),
queryStatement.getMetricReq());
if (!queryStatement.getParseSqlReq().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getParseSqlReq(), queryStatement);
} else {
queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement);
}
if (Strings.isNullOrEmpty(queryStatement.getSql())
|| Strings.isNullOrEmpty(queryStatement.getSourceId())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
}
String querySql =
Objects.nonNull(queryStatement.getEnableLimitWrapper()) && queryStatement.getEnableLimitWrapper()
? String.format(SqlExecuteReq.LIMIT_WRAPPER,
queryStatement.getSql())
: queryStatement.getSql();
queryStatement.setSql(querySql);
}
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", parseSqlReq);
try {
if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) {
List<String[]> tables = new ArrayList<>();
Boolean isSingleTable = parseSqlReq.getTables().size() == 1;
for (MetricTable metricTable : parseSqlReq.getTables()) {
QueryStatement metricTableSql = parserSql(metricTable, isSingleTable, parseSqlReq, queryStatement);
if (isSingleTable && Objects.nonNull(metricTableSql.getViewSimplifySql())
&& !metricTableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(metricTableSql.getViewSimplifySql());
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
tables.add(new String[]{metricTable.getAlias(), metricTableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql = "";
if (parseSqlReq.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + parseSqlReq.getSql();
} else {
sql = parseSqlReq.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
}
} catch (Exception e) {
log.error("physicalSql error {}", e);
queryStatement.setErrMsg(e.getMessage());
}
return queryStatement;
}
public QueryStatement parser(QueryStatement queryStatement) {
return parser(queryStatement, AggOption.getAggregation(queryStatement.getMetricReq().isNativeQuery()));
}
public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg);
if (metricQueryReq.getRootPath().isEmpty()) {
queryStatement.setErrMsg("rootPath empty");
return queryStatement;
}
try {
return ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
}
return queryStatement;
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, ParseSqlReq parseSqlReq,
QueryStatement queryStatement) throws Exception {
MetricQueryReq metricReq = new MetricQueryReq();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
metricReq.setRootPath(parseSqlReq.getRootPath());
QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false);
tableSql.setMetricReq(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setModelIds(queryStatement.getModelIds());
tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) {
tableSql.setViewSql(parseSqlReq.getSql());
tableSql.setViewAlias(metricTable.getAlias());
}
tableSql = parser(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {
throw new Exception(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
tableSql.getErrMsg()));
}
queryStatement.setSourceId(tableSql.getSourceId());
return tableSql;
}
}

View File

@@ -1,156 +1,12 @@
package com.tencent.supersonic.headless.core.parser;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/**
* logical parse from ParseSqlReq or MetricReq
* A query parser takes semantic query request and generates SQL to be executed.
*/
@Component
@Slf4j
@Primary
public class QueryParser {
public interface QueryParser {
public QueryStatement parse(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
if (Objects.isNull(queryStatement.getParseSqlReq())) {
queryStatement.setParseSqlReq(new ParseSqlReq());
}
if (Objects.isNull(queryStatement.getMetricReq())) {
queryStatement.setMetricReq(new MetricQueryReq());
}
log.info("SemanticConverter before [{}]", queryStructReq);
for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) {
if (headlessConverter.accept(queryStatement)) {
log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.convert(queryStatement);
}
}
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(),
queryStatement.getMetricReq());
if (!queryStatement.getParseSqlReq().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getParseSqlReq(), queryStatement);
} else {
queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement);
}
if (Strings.isNullOrEmpty(queryStatement.getSql())
|| Strings.isNullOrEmpty(queryStatement.getSourceId())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
}
String querySql =
Objects.nonNull(queryStatement.getEnableLimitWrapper()) && queryStatement.getEnableLimitWrapper()
? String.format(SqlExecuteReq.LIMIT_WRAPPER,
queryStatement.getSql())
: queryStatement.getSql();
queryStatement.setSql(querySql);
return queryStatement;
}
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", parseSqlReq);
try {
if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) {
List<String[]> tables = new ArrayList<>();
Boolean isSingleTable = parseSqlReq.getTables().size() == 1;
for (MetricTable metricTable : parseSqlReq.getTables()) {
QueryStatement metricTableSql = parserSql(metricTable, isSingleTable, parseSqlReq, queryStatement);
if (isSingleTable && Objects.nonNull(metricTableSql.getViewSimplifySql())
&& !metricTableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(metricTableSql.getViewSimplifySql());
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
tables.add(new String[]{metricTable.getAlias(), metricTableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql = "";
if (parseSqlReq.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + parseSqlReq.getSql();
} else {
sql = parseSqlReq.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
}
} catch (Exception e) {
log.error("physicalSql error {}", e);
queryStatement.setErrMsg(e.getMessage());
}
return queryStatement;
}
public QueryStatement parser(QueryStatement queryStatement) {
return parser(queryStatement, AggOption.getAggregation(queryStatement.getMetricReq().isNativeQuery()));
}
public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg);
if (metricQueryReq.getRootPath().isEmpty()) {
queryStatement.setErrMsg("rootPath empty");
return queryStatement;
}
try {
return ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
}
return queryStatement;
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, ParseSqlReq parseSqlReq,
QueryStatement queryStatement) throws Exception {
MetricQueryReq metricReq = new MetricQueryReq();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
metricReq.setRootPath(parseSqlReq.getRootPath());
QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false);
tableSql.setMetricReq(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setModelIds(queryStatement.getModelIds());
tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) {
tableSql.setViewSql(parseSqlReq.getSql());
tableSql.setViewAlias(metricTable.getAlias());
}
tableSql = parser(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {
throw new Exception(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
tableSql.getErrMsg()));
}
queryStatement.setSourceId(tableSql.getSourceId());
return tableSql;
}
void parse(QueryStatement queryStatement) throws Exception;
}

View File

@@ -10,7 +10,6 @@ import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;

View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.parser;
package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;

View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.Param;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;

View File

@@ -0,0 +1,32 @@
package com.tencent.supersonic.headless.core.planner;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class DefaultQueryPlanner implements QueryPlanner {
public QueryExecutor plan(QueryStatement queryStatement) {
optimize(queryStatement);
return route(queryStatement);
}
private void optimize(QueryStatement queryStatement) {
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement);
}
}
public QueryExecutor route(QueryStatement queryStatement) {
for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
if (queryExecutor.accept(queryStatement)) {
return queryExecutor;
}
}
return null;
}
}

View File

@@ -14,11 +14,12 @@ import java.util.stream.Collectors;
* Remove the default metric added by the system when the query only has dimensions
*/
@Slf4j
@Component("DetailQuery")
public class DetailQuery implements QueryOptimizer {
@Component("DetailQueryOptimizer")
public class DetailQueryOptimizer implements QueryOptimizer {
@Override
public void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement) {
public void rewrite(QueryStatement queryStatement) {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
String sqlRaw = queryStatement.getSql().trim();
if (Strings.isNullOrEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");

View File

@@ -1,11 +1,10 @@
package com.tencent.supersonic.headless.core.planner;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
/**
* the interface that rewrites QueryStatement with some optimization rules
* A query optimizer rewrites QueryStatement with a set of optimization rules
*/
public interface QueryOptimizer {
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
void rewrite(QueryStatement queryStatement);
}

View File

@@ -2,31 +2,13 @@ package com.tencent.supersonic.headless.core.planner;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class QueryPlanner {
/**
* A query planner takes parsed QueryStatement and generates an optimized execution plan.
* It interacts with the optimizer to determine the most efficient way to execute the query.
*/
public interface QueryPlanner {
QueryExecutor plan(QueryStatement queryStatement);
public QueryExecutor plan(QueryStatement queryStatement) {
optimizer(queryStatement);
return route(queryStatement);
}
public void optimizer(QueryStatement queryStatement) {
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement.getQueryStructReq(), queryStatement);
}
}
public QueryExecutor route(QueryStatement queryStatement) {
for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
if (queryExecutor.accept(queryStatement)) {
return queryExecutor;
}
}
return null;
}
QueryExecutor route(QueryStatement queryStatement);
}

View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.headless.core.utils;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.planner.DetailQuery;
import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer;
import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.SqlParser;
import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser;
import com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter;
@@ -74,7 +74,7 @@ public class ComponentFactory {
}
private static void initQueryOptimizer() {
queryOptimizers.put("DetailQuery", getBean("DetailQuery", DetailQuery.class));
queryOptimizers.put("DetailQueryOptimizer", getBean("DetailQueryOptimizer", DetailQueryOptimizer.class));
}
private static void initQueryExecutors() {

View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.headless.core.manager;
package com.tencent.supersonic.headless.server.manager;
import com.tencent.supersonic.headless.api.enums.IdentifyType;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.manager;
package com.tencent.supersonic.headless.server.manager;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
@@ -9,11 +9,11 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.core.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.manager;
package com.tencent.supersonic.headless.server.manager;
import com.tencent.supersonic.headless.api.enums.DatasourceQuery;
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
@@ -10,11 +10,11 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;

View File

@@ -20,15 +20,15 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import java.util.ArrayList;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
import com.tencent.supersonic.headless.api.pojo.Field;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import lombok.AllArgsConstructor;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import lombok.AllArgsConstructor;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import lombok.Data;
import org.apache.commons.compress.utils.Lists;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
package com.tencent.supersonic.headless.server.pojo.yaml;
import lombok.Data;

View File

@@ -8,9 +8,9 @@ import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;

View File

@@ -12,9 +12,9 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import java.util.List;

View File

@@ -9,9 +9,9 @@ import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.DatabaseService;

View File

@@ -31,12 +31,12 @@ import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.core.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.core.manager.MetricYamlManager;
import com.tencent.supersonic.headless.core.manager.ModelYamlManager;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.server.manager.MetricYamlManager;
import com.tencent.supersonic.headless.server.manager.ModelYamlManager;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;

View File

@@ -37,6 +37,7 @@ import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.parser.DefaultQueryParser;
import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.planner.QueryPlanner;
@@ -44,7 +45,7 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission;
import com.tencent.supersonic.headless.server.annotation.StructDataPermission;
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
import com.tencent.supersonic.headless.server.cache.QueryCache;
import com.tencent.supersonic.headless.core.cache.QueryCache;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.service.AppService;
@@ -96,7 +97,7 @@ public class QueryServiceImpl implements QueryService {
AppService appService,
QueryCache queryCache,
SemanticSchemaManager semanticSchemaManager,
QueryParser queryParser,
DefaultQueryParser queryParser,
QueryPlanner queryPlanner) {
this.statUtils = statUtils;
this.queryUtils = queryUtils;
@@ -259,8 +260,8 @@ public class QueryServiceImpl implements QueryService {
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
queryStatement = queryParser.parse(queryStatement);
queryPlanner.optimizer(queryStatement);
queryParser.parse(queryStatement);
queryPlanner.plan(queryStatement);
sqlParsers.add(queryStatement);
}
log.info("multi sqlParser:{}", sqlParsers);
@@ -456,9 +457,9 @@ public class QueryServiceImpl implements QueryService {
}
private QueryStatement plan(QueryStatement queryStatement) throws Exception {
queryStatement = queryParser.parse(queryStatement);
queryParser.parse(queryStatement);
log.info("queryStatement:{}", queryStatement);
queryPlanner.optimizer(queryStatement);
queryPlanner.plan(queryStatement);
return queryStatement;
}
@@ -467,7 +468,7 @@ public class QueryServiceImpl implements QueryService {
log.info("[QueryStatement:{}]", queryStatement);
try {
//1 parse
queryStatement = queryParser.parse(queryStatement);
queryParser.parse(queryStatement);
//2 plan
QueryExecutor queryExecutor = queryPlanner.plan(queryStatement);
//3 execute

View File

@@ -14,7 +14,7 @@ import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.cache.CacheManager;
import com.tencent.supersonic.headless.core.cache.CacheManager;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import java.util.Arrays;

View File

@@ -8,13 +8,13 @@ import com.tencent.supersonic.headless.api.response.SqlParserResp;
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import lombok.extern.slf4j.Slf4j;