(improvement)[build] Use Spotless to customize the code formatting (#1750)

This commit is contained in:
lexluo09
2024-10-04 00:05:04 +08:00
committed by GitHub
parent 44d1cde34f
commit 71a9954be5
521 changed files with 7811 additions and 13046 deletions

View File

@@ -72,11 +72,8 @@ public abstract class BaseDbAdaptor implements DbAdaptor {
}
protected DatabaseMetaData getDatabaseMetaData(ConnectInfo connectionInfo) throws SQLException {
Connection connection =
DriverManager.getConnection(
connectionInfo.getUrl(),
connectionInfo.getUserName(),
connectionInfo.getPassword());
Connection connection = DriverManager.getConnection(connectionInfo.getUrl(),
connectionInfo.getUserName(), connectionInfo.getPassword());
return connection.getMetaData();
}
}

View File

@@ -13,11 +13,11 @@ public class ClickHouseAdaptor extends BaseDbAdaptor {
public String getDateFormat(String dateType, String dateFormat, String column) {
if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) {
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
return "toYYYYMM(toDate(parseDateTimeBestEffort(toString(%s))))"
.replace("%s", column);
return "toYYYYMM(toDate(parseDateTimeBestEffort(toString(%s))))".replace("%s",
column);
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))"
.replace("%s", column);
return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))".replace("%s",
column);
} else {
return "toDate(parseDateTimeBestEffort(toString(%s)))".replace("%s", column);
}

View File

@@ -9,18 +9,18 @@ public class H2Adaptor extends BaseDbAdaptor {
public String getDateFormat(String dateType, String dateFormat, String column) {
if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) {
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM')"
.replace("%s", column);
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM')".replace("%s",
column);
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
return "DATE_TRUNC('week',%s)".replace("%s", column);
} else {
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM-dd')"
.replace("%s", column);
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM-dd')".replace("%s",
column);
}
} else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) {
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyy-MM-dd'),'yyyy-MM') "
.replace("%s", column);
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyy-MM-dd'),'yyyy-MM') ".replace("%s",
column);
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
return "DATE_TRUNC('week',%s)".replace("%s", column);
} else {

View File

@@ -53,36 +53,30 @@ public class PostgresqlAdaptor extends BaseDbAdaptor {
functionMap.put("DAY".toLowerCase(), "TO_CHAR");
functionMap.put("YEAR".toLowerCase(), "TO_CHAR");
Map<String, UnaryOperator> functionCall = new HashMap<>();
functionCall.put(
"MONTH".toLowerCase(),
o -> {
if (Objects.nonNull(o) && o instanceof ExpressionList) {
ExpressionList expressionList = (ExpressionList) o;
expressionList.add(new StringValue("MM"));
return expressionList;
}
return o;
});
functionCall.put(
"DAY".toLowerCase(),
o -> {
if (Objects.nonNull(o) && o instanceof ExpressionList) {
ExpressionList expressionList = (ExpressionList) o;
expressionList.add(new StringValue("dd"));
return expressionList;
}
return o;
});
functionCall.put(
"YEAR".toLowerCase(),
o -> {
if (Objects.nonNull(o) && o instanceof ExpressionList) {
ExpressionList expressionList = (ExpressionList) o;
expressionList.add(new StringValue("YYYY"));
return expressionList;
}
return o;
});
functionCall.put("MONTH".toLowerCase(), o -> {
if (Objects.nonNull(o) && o instanceof ExpressionList) {
ExpressionList expressionList = (ExpressionList) o;
expressionList.add(new StringValue("MM"));
return expressionList;
}
return o;
});
functionCall.put("DAY".toLowerCase(), o -> {
if (Objects.nonNull(o) && o instanceof ExpressionList) {
ExpressionList expressionList = (ExpressionList) o;
expressionList.add(new StringValue("dd"));
return expressionList;
}
return o;
});
functionCall.put("YEAR".toLowerCase(), o -> {
if (Objects.nonNull(o) && o instanceof ExpressionList) {
ExpressionList expressionList = (ExpressionList) o;
expressionList.add(new StringValue("YYYY"));
return expressionList;
}
return o;
});
return SqlReplaceHelper.replaceFunction(sql, functionMap, functionCall);
}

View File

@@ -12,7 +12,8 @@ import java.util.concurrent.TimeUnit;
@Configuration
public class CaffeineCacheConfig {
@Autowired private CacheCommonConfig cacheCommonConfig;
@Autowired
private CacheCommonConfig cacheCommonConfig;
@Value("${s2.caffeine.initial.capacity:500}")
private Integer caffeineInitialCapacity;
@@ -23,19 +24,14 @@ public class CaffeineCacheConfig {
@Bean(name = "caffeineCache")
public Cache<String, Object> caffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(
cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES)
.initialCapacity(caffeineInitialCapacity)
.maximumSize(caffeineMaximumSize)
.build();
.expireAfterWrite(cacheCommonConfig.getCacheCommonExpireAfterWrite(),
TimeUnit.MINUTES)
.initialCapacity(caffeineInitialCapacity).maximumSize(caffeineMaximumSize).build();
}
@Bean(name = "searchCaffeineCache")
public Cache<Long, Object> searchCaffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(10000, TimeUnit.MINUTES)
.initialCapacity(caffeineInitialCapacity)
.maximumSize(caffeineMaximumSize)
.build();
return Caffeine.newBuilder().expireAfterWrite(10000, TimeUnit.MINUTES)
.initialCapacity(caffeineInitialCapacity).maximumSize(caffeineMaximumSize).build();
}
}

View File

@@ -12,7 +12,8 @@ import org.springframework.stereotype.Component;
@Slf4j
public class CaffeineCacheManager implements CacheManager {
@Autowired private CacheCommonConfig cacheCommonConfig;
@Autowired
private CacheCommonConfig cacheCommonConfig;
@Autowired
@Qualifier("caffeineCache")
@@ -37,13 +38,9 @@ public class CaffeineCacheManager implements CacheManager {
if (StringUtils.isEmpty(prefix)) {
prefix = "-1";
}
return Joiner.on(":")
.join(
cacheCommonConfig.getCacheCommonApp(),
cacheCommonConfig.getCacheCommonEnv(),
cacheCommonConfig.getCacheCommonVersion(),
prefix,
body);
return Joiner.on(":").join(cacheCommonConfig.getCacheCommonApp(),
cacheCommonConfig.getCacheCommonEnv(), cacheCommonConfig.getCacheCommonVersion(),
prefix, body);
}
@Override

View File

@@ -29,11 +29,10 @@ public class DefaultQueryCache implements QueryCache {
CacheCommonConfig cacheCommonConfig = ContextUtils.getBean(CacheCommonConfig.class);
if (cacheCommonConfig.getCacheEnable() && Objects.nonNull(value)) {
CompletableFuture.supplyAsync(() -> cacheManager.put(cacheKey, value))
.exceptionally(
exception -> {
log.warn("exception:", exception);
return null;
});
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
log.debug("put to cache, key: {}", cacheKey);
return true;
}
@@ -48,8 +47,8 @@ public class DefaultQueryCache implements QueryCache {
}
private String getKeyByModelIds(List<Long> modelIds) {
return String.join(
",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
return String.join(",",
modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
private boolean isCache(SemanticQueryReq semanticQueryReq) {

View File

@@ -56,13 +56,9 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
public static final String MATERIALIZATION_SYS_PARTITION = "sys_partition";
/** check if a materialization match the fields and partitions */
protected boolean check(
RelOptPlanner relOptPlanner,
RelBuilder relBuilder,
CalciteCatalogReader calciteCatalogReader,
Materialization materialization,
List<String> fields,
List<ImmutablePair<String, String>> partitions) {
protected boolean check(RelOptPlanner relOptPlanner, RelBuilder relBuilder,
CalciteCatalogReader calciteCatalogReader, Materialization materialization,
List<String> fields, List<ImmutablePair<String, String>> partitions) {
if (!materialization.isPartitioned()) {
return fields.stream().allMatch(f -> materialization.getColumns().contains(f));
}
@@ -85,8 +81,8 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
}
Materialization viewMaterialization = Materialization.builder().build();
viewMaterialization.setName(
String.format("%s.%s", MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW));
viewMaterialization
.setName(String.format("%s.%s", MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW));
viewMaterialization.setColumns(viewFieldList);
addMaterialization(calciteCatalogReader.getRootSchema(), viewMaterialization);
@@ -97,10 +93,8 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
queryMaterialization.setColumns(materializationFieldList);
addMaterialization(calciteCatalogReader.getRootSchema(), queryMaterialization);
RelNode replacement =
relBuilder
.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW))
.build();
RelNode replacement = relBuilder
.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_VIEW)).build();
RelBuilder viewBuilder =
relBuilder.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_SOURCE));
if (materialization.isPartitioned()) {
@@ -117,9 +111,8 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
RelBuilder checkBuilder =
relBuilder.scan(Arrays.asList(MATERIALIZATION_SYS_DB, MATERIALIZATION_SYS_SOURCE));
if (materialization.isPartitioned()) {
checkBuilder =
checkBuilder.filter(
getRexNode(checkBuilder, partitions, MATERIALIZATION_SYS_PARTITION));
checkBuilder = checkBuilder
.filter(getRexNode(checkBuilder, partitions, MATERIALIZATION_SYS_PARTITION));
}
RelNode checkRel = project(checkBuilder, queryFieldList).build();
relOptPlanner.setRoot(checkRel);
@@ -135,12 +128,9 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
protected CalciteCatalogReader getCalciteCatalogReader() {
CalciteCatalogReader calciteCatalogReader;
CalciteSchema viewSchema = SchemaBuilder.getMaterializationSchema();
calciteCatalogReader =
new CalciteCatalogReader(
CalciteSchema.from(viewSchema.plus()),
CalciteSchema.from(viewSchema.plus()).path(null),
Configuration.typeFactory,
new CalciteConnectionConfigImpl(new Properties()));
calciteCatalogReader = new CalciteCatalogReader(CalciteSchema.from(viewSchema.plus()),
CalciteSchema.from(viewSchema.plus()).path(null), Configuration.typeFactory,
new CalciteConnectionConfigImpl(new Properties()));
return calciteCatalogReader;
}
@@ -151,8 +141,8 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
return relOptPlanner;
}
protected RelBuilder builderMaterializationPlan(
CalciteCatalogReader calciteCatalogReader, RelOptPlanner relOptPlanner) {
protected RelBuilder builderMaterializationPlan(CalciteCatalogReader calciteCatalogReader,
RelOptPlanner relOptPlanner) {
relOptPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE);
relOptPlanner.addRelTraitDef(RelDistributionTraitDef.INSTANCE);
EnumerableRules.rules().forEach(relOptPlanner::addRule);
@@ -161,8 +151,8 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
return RelFactories.LOGICAL_BUILDER.create(relOptCluster, calciteCatalogReader);
}
protected void addMaterialization(
CalciteSchema dataSetSchema, Materialization materialization) {
protected void addMaterialization(CalciteSchema dataSetSchema,
Materialization materialization) {
String[] dbTable = materialization.getName().split("\\.");
String tb = dbTable[1].toLowerCase();
String db = dbTable[0].toLowerCase();
@@ -188,34 +178,28 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
protected Set<String> extractTableNames(RelNode relNode) {
Set<String> tableNames = new HashSet<>();
RelShuttle shuttle =
new RelHomogeneousShuttle() {
public RelNode visit(TableScan scan) {
RelOptTable table = scan.getTable();
tableNames.addAll(table.getQualifiedName());
return scan;
}
};
RelShuttle shuttle = new RelHomogeneousShuttle() {
public RelNode visit(TableScan scan) {
RelOptTable table = scan.getTable();
tableNames.addAll(table.getQualifiedName());
return scan;
}
};
relNode.accept(shuttle);
return tableNames;
}
protected RexNode getRexNodeByTimeRange(
RelBuilder relBuilder, TimeRange timeRange, String field) {
return relBuilder.call(
SqlStdOperatorTable.AND,
relBuilder.call(
SqlStdOperatorTable.GREATER_THAN_OR_EQUAL,
relBuilder.field(field),
protected RexNode getRexNodeByTimeRange(RelBuilder relBuilder, TimeRange timeRange,
String field) {
return relBuilder.call(SqlStdOperatorTable.AND,
relBuilder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, relBuilder.field(field),
relBuilder.literal(timeRange.getStart())),
relBuilder.call(
SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
relBuilder.field(field),
relBuilder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(field),
relBuilder.literal(timeRange.getEnd())));
}
protected RexNode getRexNode(
RelBuilder relBuilder, Materialization materialization, String viewField) {
protected RexNode getRexNode(RelBuilder relBuilder, Materialization materialization,
String viewField) {
RexNode rexNode = null;
for (String partition : materialization.getPartitions()) {
TimeRange timeRange = TimeRange.builder().start(partition).end(partition).build();
@@ -223,43 +207,26 @@ public abstract class AbstractAccelerator implements QueryAccelerator {
rexNode = getRexNodeByTimeRange(relBuilder, timeRange, viewField);
continue;
}
rexNode =
relBuilder.call(
SqlStdOperatorTable.OR,
rexNode,
getRexNodeByTimeRange(relBuilder, timeRange, viewField));
rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode,
getRexNodeByTimeRange(relBuilder, timeRange, viewField));
}
return rexNode;
}
protected RexNode getRexNode(
RelBuilder relBuilder,
List<ImmutablePair<String, String>> timeRanges,
String viewField) {
protected RexNode getRexNode(RelBuilder relBuilder,
List<ImmutablePair<String, String>> timeRanges, String viewField) {
RexNode rexNode = null;
for (ImmutablePair<String, String> timeRange : timeRanges) {
if (rexNode == null) {
rexNode =
getRexNodeByTimeRange(
relBuilder,
TimeRange.builder()
.start(timeRange.left)
.end(timeRange.right)
.build(),
viewField);
rexNode = getRexNodeByTimeRange(relBuilder,
TimeRange.builder().start(timeRange.left).end(timeRange.right).build(),
viewField);
continue;
}
rexNode =
relBuilder.call(
SqlStdOperatorTable.OR,
rexNode,
getRexNodeByTimeRange(
relBuilder,
TimeRange.builder()
.start(timeRange.left)
.end(timeRange.right)
.build(),
viewField));
rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode,
getRexNodeByTimeRange(relBuilder,
TimeRange.builder().start(timeRange.left).end(timeRange.right).build(),
viewField));
}
return rexNode;
}

View File

@@ -28,8 +28,8 @@ public class JdbcExecutor implements QueryExecutor {
SemanticQueryResp semanticQueryResp = queryAccelerator.query(queryStatement);
if (Objects.nonNull(semanticQueryResp)
&& !semanticQueryResp.getResultList().isEmpty()) {
log.info(
"query by Accelerator {}", queryAccelerator.getClass().getSimpleName());
log.info("query by Accelerator {}",
queryAccelerator.getClass().getSimpleName());
return semanticQueryResp;
}
}

View File

@@ -63,8 +63,8 @@ public class DuckDbSource {
protected void init(JdbcTemplate jdbcTemplate) {
jdbcTemplate.execute(
String.format("SET memory_limit = '%sGB';", executorConfig.getMemoryLimit()));
jdbcTemplate.execute(
String.format("SET temp_directory='%s';", executorConfig.getDuckDbTemp()));
jdbcTemplate
.execute(String.format("SET temp_directory='%s';", executorConfig.getDuckDbTemp()));
jdbcTemplate.execute(String.format("SET threads TO %s;", executorConfig.getThreads()));
jdbcTemplate.execute("SET enable_object_cache = true;");
}
@@ -82,23 +82,21 @@ public class DuckDbSource {
}
public void query(String sql, SemanticQueryResp queryResultWithColumns) {
duckDbJdbcTemplate.query(
sql,
rs -> {
if (null == rs) {
return queryResultWithColumns;
}
ResultSetMetaData metaData = rs.getMetaData();
List<QueryColumn> queryColumns = new ArrayList<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String key = metaData.getColumnLabel(i);
queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i)));
}
queryResultWithColumns.setColumns(queryColumns);
List<Map<String, Object>> resultList = buildResult(rs);
queryResultWithColumns.setResultList(resultList);
return queryResultWithColumns;
});
duckDbJdbcTemplate.query(sql, rs -> {
if (null == rs) {
return queryResultWithColumns;
}
ResultSetMetaData metaData = rs.getMetaData();
List<QueryColumn> queryColumns = new ArrayList<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String key = metaData.getColumnLabel(i);
queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i)));
}
queryResultWithColumns.setColumns(queryColumns);
List<Map<String, Object>> resultList = buildResult(rs);
queryResultWithColumns.setResultList(resultList);
return queryResultWithColumns;
});
}
public static List<Map<String, Object>> buildResult(ResultSet resultSet) {

View File

@@ -203,10 +203,8 @@ public class JdbcDataSource {
// default validation query
String driverName = druidDataSource.getDriverClassName();
if (driverName.indexOf("sqlserver") != -1
|| driverName.indexOf("mysql") != -1
|| driverName.indexOf("h2") != -1
|| driverName.indexOf("moonbox") != -1) {
if (driverName.indexOf("sqlserver") != -1 || driverName.indexOf("mysql") != -1
|| driverName.indexOf("h2") != -1 || driverName.indexOf("moonbox") != -1) {
druidDataSource.setValidationQuery("select 1");
}
@@ -242,12 +240,7 @@ public class JdbcDataSource {
}
private String getDataSourceKey(Database database) {
return JdbcDataSourceUtils.getKey(
database.getName(),
database.getUrl(),
database.getUsername(),
database.passwordDecrypt(),
"",
false);
return JdbcDataSourceUtils.getKey(database.getName(), database.getUrl(),
database.getUsername(), database.passwordDecrypt(), "", false);
}
}

View File

@@ -57,16 +57,12 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
headlessConverter.convert(queryStatement);
}
}
log.debug(
"SemanticConverter after {} {} {}",
queryParam,
queryStatement.getDataSetQueryParam(),
queryStatement.getMetricQueryParam());
log.debug("SemanticConverter after {} {} {}", queryParam,
queryStatement.getDataSetQueryParam(), queryStatement.getMetricQueryParam());
if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) {
doParse(queryStatement.getDataSetQueryParam(), queryStatement);
} else {
queryStatement
.getMetricQueryParam()
queryStatement.getMetricQueryParam()
.setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
doParse(queryStatement);
}
@@ -81,8 +77,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
}
}
public QueryStatement doParse(
DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) {
public QueryStatement doParse(DataSetQueryParam dataSetQueryParam,
QueryStatement queryStatement) {
log.info("parse dataSetQuery [{}] ", dataSetQueryParam);
SemanticModel semanticModel = queryStatement.getSemanticModel();
EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType());
@@ -91,9 +87,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
List<String[]> tables = new ArrayList<>();
boolean isSingleTable = dataSetQueryParam.getTables().size() == 1;
for (MetricTable metricTable : dataSetQueryParam.getTables()) {
QueryStatement tableSql =
parserSql(
metricTable, isSingleTable, dataSetQueryParam, queryStatement);
QueryStatement tableSql = parserSql(metricTable, isSingleTable,
dataSetQueryParam, queryStatement);
if (isSingleTable && StringUtils.isNotBlank(tableSql.getDataSetSimplifySql())) {
queryStatement.setSql(tableSql.getDataSetSimplifySql());
queryStatement.setDataSetQueryParam(dataSetQueryParam);
@@ -108,26 +103,13 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
tables.stream().map(table -> table[0]).collect(Collectors.toList());
List<String> parentSqlList =
tables.stream().map(table -> table[1]).collect(Collectors.toList());
sql =
SqlMergeWithUtils.mergeWith(
engineType,
dataSetQueryParam.getSql(),
parentSqlList,
parentWithNameList);
sql = SqlMergeWithUtils.mergeWith(engineType, dataSetQueryParam.getSql(),
parentSqlList, parentWithNameList);
} else {
sql = dataSetQueryParam.getSql();
for (String[] tb : tables) {
sql =
StringUtils.replace(
sql,
tb[0],
"("
+ tb[1]
+ ") "
+ (dataSetQueryParam.isWithAlias()
? ""
: tb[0]),
-1);
sql = StringUtils.replace(sql, tb[0], "(" + tb[1] + ") "
+ (dataSetQueryParam.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
@@ -143,8 +125,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
}
public QueryStatement doParse(QueryStatement queryStatement) {
return doParse(
queryStatement,
return doParse(queryStatement,
AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery()));
}
@@ -160,12 +141,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
return queryStatement;
}
private QueryStatement parserSql(
MetricTable metricTable,
Boolean isSingleMetricTable,
DataSetQueryParam dataSetQueryParam,
QueryStatement queryStatement)
throws Exception {
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable,
DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) throws Exception {
MetricQueryParam metricReq = new MetricQueryParam();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
@@ -184,10 +161,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
}
tableSql = doParse(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {
throw new Exception(
String.format(
"parser table [%s] error [%s]",
metricTable.getAlias(), tableSql.getErrMsg()));
throw new Exception(String.format("parser table [%s] error [%s]",
metricTable.getAlias(), tableSql.getErrMsg()));
}
return tableSql;
}

View File

@@ -27,11 +27,8 @@ public class DetailQueryOptimizer implements QueryOptimizer {
if (queryParam.getMetrics().size() == 0
&& !CollectionUtils.isEmpty(queryParam.getGroups())) {
String sqlForm = "select %s from ( %s ) src_no_metric";
String sql =
String.format(
sqlForm,
queryParam.getGroups().stream().collect(Collectors.joining(",")),
sqlRaw);
String sql = String.format(sqlForm,
queryParam.getGroups().stream().collect(Collectors.joining(",")), sqlRaw);
queryStatement.setSql(sql);
}
}
@@ -39,8 +36,7 @@ public class DetailQueryOptimizer implements QueryOptimizer {
}
public boolean isDetailQuery(QueryParam queryParam) {
return Objects.nonNull(queryParam)
&& queryParam.getQueryType().isNativeAggQuery()
return Objects.nonNull(queryParam) && queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(queryParam.getMetrics());
}
}

View File

@@ -41,14 +41,10 @@ public class CalciteQueryParser implements QueryParser {
&& Objects.nonNull(queryStatement.getDataSetAlias())
&& !queryStatement.getDataSetAlias().isEmpty()) {
// simplify model sql with query sql
String simplifySql =
aggBuilder.simplify(
getSqlByDataSet(
engineType,
aggBuilder.getSql(engineType),
queryStatement.getDataSetSql(),
queryStatement.getDataSetAlias()),
engineType);
String simplifySql = aggBuilder.simplify(
getSqlByDataSet(engineType, aggBuilder.getSql(engineType),
queryStatement.getDataSetSql(), queryStatement.getDataSetAlias()),
engineType);
if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) {
log.debug("simplifySql [{}]", simplifySql);
queryStatement.setDataSetSimplifySql(simplifySql);
@@ -56,8 +52,8 @@ public class CalciteQueryParser implements QueryParser {
}
}
private SemanticSchema getSemanticSchema(
SemanticModel semanticModel, QueryStatement queryStatement) {
private SemanticSchema getSemanticSchema(SemanticModel semanticModel,
QueryStatement queryStatement) {
SemanticSchema semanticSchema =
SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build();
semanticSchema.setSemanticModel(semanticModel);
@@ -66,20 +62,14 @@ public class CalciteQueryParser implements QueryParser {
semanticSchema.setMetric(semanticModel.getMetrics());
semanticSchema.setJoinRelations(semanticModel.getJoinRelations());
semanticSchema.setRuntimeOptions(
RuntimeOptions.builder()
.minMaxTime(queryStatement.getMinMaxTime())
.enableOptimize(queryStatement.getEnableOptimize())
.build());
RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime())
.enableOptimize(queryStatement.getEnableOptimize()).build());
return semanticSchema;
}
private String getSqlByDataSet(
EngineType engineType, String parentSql, String dataSetSql, String parentAlias)
throws SqlParseException {
return SqlMergeWithUtils.mergeWith(
engineType,
dataSetSql,
Collections.singletonList(parentSql),
Collections.singletonList(parentAlias));
private String getSqlByDataSet(EngineType engineType, String parentSql, String dataSetSql,
String parentAlias) throws SqlParseException {
return SqlMergeWithUtils.mergeWith(engineType, dataSetSql,
Collections.singletonList(parentSql), Collections.singletonList(parentAlias));
}
}

View File

@@ -56,7 +56,7 @@ public class AggPlanner implements Planner {
isAgg = getAgg(datasource.get(0));
sourceId = String.valueOf(datasource.get(0).getSourceId());
// build level by level
// build level by level
LinkedList<Renderer> builders = new LinkedList<>();
builders.add(new SourceRender());
builders.add(new FilterRender());
@@ -68,9 +68,8 @@ public class AggPlanner implements Planner {
Renderer renderer = it.next();
if (previous != null) {
previous.render(metricReq, datasource, scope, schema, !isAgg);
renderer.setTable(
previous.builderAs(
DataSourceNode.getNames(datasource) + "_" + String.valueOf(i)));
renderer.setTable(previous
.builderAs(DataSourceNode.getNames(datasource) + "_" + String.valueOf(i)));
i++;
}
previous = renderer;
@@ -88,10 +87,8 @@ public class AggPlanner implements Planner {
return AggOption.isAgg(aggOption);
}
// default by dataSource time aggregation
if (Objects.nonNull(dataSource.getAggTime())
&& !dataSource
.getAggTime()
.equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) {
if (Objects.nonNull(dataSource.getAggTime()) && !dataSource.getAggTime()
.equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) {
if (!metricReq.isNativeQuery()) {
return true;
}

View File

@@ -10,8 +10,7 @@ import lombok.NoArgsConstructor;
public class Identify {
public enum Type {
PRIMARY,
FOREIGN
PRIMARY, FOREIGN
}
private String name;

View File

@@ -15,10 +15,8 @@ public class Materialization {
* partition time type 1 - FULL, not use partition 2 - PARTITION , use time list 3 - ZIPPER,
* use [startDate, endDate] range time
*/
FULL("FULL"),
PARTITION("PARTITION"),
ZIPPER("ZIPPER"),
None("");
FULL("FULL"), PARTITION("PARTITION"), ZIPPER("ZIPPER"), None("");
private String name;
TimePartType(String name) {

View File

@@ -22,8 +22,7 @@ public class SemanticModel {
private Database database;
public List<Dimension> getDimensions() {
return dimensionMap.values().stream()
.flatMap(Collection::stream)
return dimensionMap.values().stream().flatMap(Collection::stream)
.collect(Collectors.toList());
}

View File

@@ -32,10 +32,7 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr
private RelDataType rowType;
private DataSourceTable(
String tableName,
List<String> fieldNames,
List<SqlTypeName> fieldTypes,
private DataSourceTable(String tableName, List<String> fieldNames, List<SqlTypeName> fieldTypes,
Statistic statistic) {
this.tableName = tableName;
this.fieldNames = fieldNames;
@@ -80,8 +77,8 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr
public RelNode toRel(RelOptTable.ToRelContext toRelContext, RelOptTable relOptTable) {
List<RelHint> hint = new ArrayList<>();
return new LogicalTableScan(
toRelContext.getCluster(), toRelContext.getCluster().traitSet(), hint, relOptTable);
return new LogicalTableScan(toRelContext.getCluster(), toRelContext.getCluster().traitSet(),
hint, relOptTable);
}
public static final class Builder {
@@ -128,8 +125,8 @@ public class DataSourceTable extends AbstractTable implements ScannableTable, Tr
throw new IllegalStateException("Table must have positive row count");
}
return new DataSourceTable(
tableName, fieldNames, fieldTypes, Statistics.of(rowCount, null));
return new DataSourceTable(tableName, fieldNames, fieldTypes,
Statistics.of(rowCount, null));
}
}
}

View File

@@ -31,50 +31,35 @@ public class SchemaBuilder {
Map<String, RelDataType> nameToTypeMap = new HashMap<>();
CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false);
rootSchema.add(schema.getSchemaKey(), schema);
Prepare.CatalogReader catalogReader =
new CalciteCatalogReader(
rootSchema,
Collections.singletonList(schema.getSchemaKey()),
Configuration.typeFactory,
Configuration.config);
Prepare.CatalogReader catalogReader = new CalciteCatalogReader(rootSchema,
Collections.singletonList(schema.getSchemaKey()), Configuration.typeFactory,
Configuration.config);
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
S2SQLSqlValidatorImpl s2SQLSqlValidator =
new S2SQLSqlValidatorImpl(
Configuration.operatorTable,
catalogReader,
Configuration.typeFactory,
Configuration.getValidatorConfig(engineType));
new S2SQLSqlValidatorImpl(Configuration.operatorTable, catalogReader,
Configuration.typeFactory, Configuration.getValidatorConfig(engineType));
return new ParameterScope(s2SQLSqlValidator, nameToTypeMap);
}
public static CalciteSchema getMaterializationSchema() {
CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false);
SchemaPlus schema = rootSchema.plus().add(MATERIALIZATION_SYS_DB, new AbstractSchema());
DataSourceTable srcTable =
DataSourceTable.newBuilder(MATERIALIZATION_SYS_SOURCE)
.addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE)
.addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT)
.withRowCount(1)
.build();
DataSourceTable srcTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_SOURCE)
.addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE)
.addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT).withRowCount(1)
.build();
schema.add(MATERIALIZATION_SYS_SOURCE, srcTable);
DataSourceTable dataSetTable =
DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW)
.addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE)
.addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT)
.withRowCount(1)
.build();
DataSourceTable dataSetTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW)
.addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE)
.addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT).withRowCount(1)
.build();
schema.add(MATERIALIZATION_SYS_VIEW, dataSetTable);
return rootSchema;
}
public static void addSourceView(
CalciteSchema dataSetSchema,
String dbSrc,
String tbSrc,
Set<String> dates,
Set<String> dimensions,
Set<String> metrics) {
public static void addSourceView(CalciteSchema dataSetSchema, String dbSrc, String tbSrc,
Set<String> dates, Set<String> dimensions, Set<String> metrics) {
String tb = tbSrc;
String db = dbSrc;
DataSourceTable.Builder builder = DataSourceTable.newBuilder(tb);

View File

@@ -28,38 +28,28 @@ public abstract class Renderer {
protected TableView tableView = new TableView();
public static Optional<Dimension> getDimensionByName(String name, DataSource datasource) {
return datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(name))
return datasource.getDimensions().stream().filter(d -> d.getName().equalsIgnoreCase(name))
.findFirst();
}
public static Optional<Measure> getMeasureByName(String name, DataSource datasource) {
return datasource.getMeasures().stream()
.filter(mm -> mm.getName().equalsIgnoreCase(name))
return datasource.getMeasures().stream().filter(mm -> mm.getName().equalsIgnoreCase(name))
.findFirst();
}
public static Optional<Metric> getMetricByName(String name, SemanticSchema schema) {
Optional<Metric> metric =
schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(name))
.findFirst();
Optional<Metric> metric = schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(name)).findFirst();
return metric;
}
public static Optional<Identify> getIdentifyByName(String name, DataSource datasource) {
return datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(name))
return datasource.getIdentifiers().stream().filter(i -> i.getName().equalsIgnoreCase(name))
.findFirst();
}
public static MetricNode buildMetricNode(
String metric,
DataSource datasource,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg,
String alias)
public static MetricNode buildMetricNode(String metric, DataSource datasource,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg, String alias)
throws Exception {
Optional<Metric> metricOpt = getMetricByName(metric, schema);
MetricNode metricNode = new MetricNode();
@@ -69,61 +59,38 @@ public abstract class Renderer {
for (Measure m : metricOpt.get().getMetricTypeParams().getMeasures()) {
Optional<Measure> measure = getMeasureByName(m.getName(), datasource);
if (measure.isPresent()) {
metricNode
.getNonAggNode()
.put(
measure.get().getName(),
MeasureNode.buildNonAgg(
alias, measure.get(), scope, engineType));
metricNode
.getAggNode()
.put(
measure.get().getName(),
MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType));
metricNode
.getAggFunction()
.put(measure.get().getName(), measure.get().getAgg());
metricNode.getNonAggNode().put(measure.get().getName(),
MeasureNode.buildNonAgg(alias, measure.get(), scope, engineType));
metricNode.getAggNode().put(measure.get().getName(),
MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType));
metricNode.getAggFunction().put(measure.get().getName(),
measure.get().getAgg());
} else {
metricNode
.getNonAggNode()
.put(m.getName(), MeasureNode.buildNonAgg(alias, m, scope, engineType));
metricNode
.getAggNode()
.put(m.getName(), MeasureNode.buildAgg(m, nonAgg, scope, engineType));
metricNode.getNonAggNode().put(m.getName(),
MeasureNode.buildNonAgg(alias, m, scope, engineType));
metricNode.getAggNode().put(m.getName(),
MeasureNode.buildAgg(m, nonAgg, scope, engineType));
metricNode.getAggFunction().put(m.getName(), m.getAgg());
}
if (m.getConstraint() != null && !m.getConstraint().isEmpty()) {
metricNode
.getMeasureFilter()
.put(
m.getName(),
SemanticNode.parse(m.getConstraint(), scope, engineType));
metricNode.getMeasureFilter().put(m.getName(),
SemanticNode.parse(m.getConstraint(), scope, engineType));
}
}
return metricNode;
}
Optional<Measure> measure = getMeasureByName(metric, datasource);
if (measure.isPresent()) {
metricNode
.getNonAggNode()
.put(
measure.get().getName(),
MeasureNode.buildNonAgg(alias, measure.get(), scope, engineType));
metricNode
.getAggNode()
.put(
measure.get().getName(),
MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType));
metricNode.getNonAggNode().put(measure.get().getName(),
MeasureNode.buildNonAgg(alias, measure.get(), scope, engineType));
metricNode.getAggNode().put(measure.get().getName(),
MeasureNode.buildAgg(measure.get(), nonAgg, scope, engineType));
metricNode.getAggFunction().put(measure.get().getName(), measure.get().getAgg());
if (measure.get().getConstraint() != null && !measure.get().getConstraint().isEmpty()) {
metricNode
.getMeasureFilter()
.put(
measure.get().getName(),
SemanticNode.parse(
measure.get().getConstraint(), scope, engineType));
metricNode.getMeasureFilter().put(measure.get().getName(),
SemanticNode.parse(measure.get().getConstraint(), scope, engineType));
}
}
return metricNode;
@@ -146,11 +113,6 @@ public abstract class Renderer {
return SemanticNode.buildAs(alias, tableView.build());
}
public abstract void render(
MetricQueryParam metricCommand,
List<DataSource> dataSources,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
throws Exception;
public abstract void render(MetricQueryParam metricCommand, List<DataSource> dataSources,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception;
}

View File

@@ -8,11 +8,8 @@ import org.apache.calcite.sql.validate.SqlValidatorImpl;
/** customize the SqlValidatorImpl */
public class S2SQLSqlValidatorImpl extends SqlValidatorImpl {
public S2SQLSqlValidatorImpl(
SqlOperatorTable opTab,
SqlValidatorCatalogReader catalogReader,
RelDataTypeFactory typeFactory,
Config config) {
public S2SQLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader,
RelDataTypeFactory typeFactory, Config config) {
super(opTab, catalogReader, typeFactory, config);
}
}

View File

@@ -39,29 +39,16 @@ public class TableView {
if (filter.size() > 0) {
filterNodeList = new SqlNodeList(filter, SqlParserPos.ZERO);
}
return new SqlSelect(
SqlParserPos.ZERO,
null,
new SqlNodeList(measure, SqlParserPos.ZERO),
table,
filterNodeList,
dimensionNodeList,
null,
null,
null,
order,
offset,
fetch,
return new SqlSelect(SqlParserPos.ZERO, null, new SqlNodeList(measure, SqlParserPos.ZERO),
table, filterNodeList, dimensionNodeList, null, null, null, order, offset, fetch,
null);
}
private List<SqlNode> getGroup(List<SqlNode> sqlNodeList) {
return sqlNodeList.stream()
.map(
s ->
(s.getKind().equals(SqlKind.AS)
? ((SqlBasicCall) s).getOperandList().get(0)
: s))
.map(s -> (s.getKind().equals(SqlKind.AS)
? ((SqlBasicCall) s).getOperandList().get(0)
: s))
.collect(Collectors.toList());
}
}

View File

@@ -8,33 +8,19 @@ import java.util.Objects;
public class AggFunctionNode extends SemanticNode {
public static SqlNode build(
String agg, String name, SqlValidatorScope scope, EngineType engineType)
throws Exception {
public static SqlNode build(String agg, String name, SqlValidatorScope scope,
EngineType engineType) throws Exception {
if (Objects.isNull(agg) || agg.isEmpty()) {
return parse(name, scope, engineType);
}
if (AggFunction.COUNT_DISTINCT.name().equalsIgnoreCase(agg)) {
return parse(
AggFunction.COUNT.name()
+ " ( "
+ AggFunction.DISTINCT.name()
+ " "
+ name
+ " ) ",
scope,
engineType);
return parse(AggFunction.COUNT.name() + " ( " + AggFunction.DISTINCT.name() + " " + name
+ " ) ", scope, engineType);
}
return parse(agg + " ( " + name + " ) ", scope, engineType);
}
public static enum AggFunction {
AVG,
COUNT_DISTINCT,
MAX,
MIN,
SUM,
COUNT,
DISTINCT
AVG, COUNT_DISTINCT, MAX, MIN, SUM, COUNT, DISTINCT
}
}

View File

@@ -46,9 +46,8 @@ public class DataSourceNode extends SemanticNode {
sqlTable = datasource.getSqlQuery();
} else if (datasource.getTableQuery() != null && !datasource.getTableQuery().isEmpty()) {
if (datasource.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) {
String fullTableName =
Arrays.stream(datasource.getTableQuery().split("\\."))
.collect(Collectors.joining(".public."));
String fullTableName = Arrays.stream(datasource.getTableQuery().split("\\."))
.collect(Collectors.joining(".public."));
sqlTable = "select * from " + fullTableName;
} else {
sqlTable = "select * from " + datasource.getTableQuery();
@@ -76,13 +75,8 @@ public class DataSourceNode extends SemanticNode {
}
}
private static void addSchemaTable(
SqlValidatorScope scope,
DataSource datasource,
String db,
String tb,
Set<String> fields)
throws Exception {
private static void addSchemaTable(SqlValidatorScope scope, DataSource datasource, String db,
String tb, Set<String> fields) throws Exception {
Set<String> dateInfo = new HashSet<>();
Set<String> dimensions = new HashSet<>();
Set<String> metrics = new HashSet<>();
@@ -99,13 +93,11 @@ public class DataSourceNode extends SemanticNode {
for (Measure m : datasource.getMeasures()) {
List<SqlNode> identifiers =
expand(SemanticNode.parse(m.getExpr(), scope, engineType), scope);
identifiers.stream()
.forEach(
i -> {
if (!dimensions.contains(i.toString())) {
metrics.add(i.toString());
}
});
identifiers.stream().forEach(i -> {
if (!dimensions.contains(i.toString())) {
metrics.add(i.toString());
}
});
if (!dimensions.contains(m.getName())) {
metrics.add(m.getName());
}
@@ -116,46 +108,32 @@ public class DataSourceNode extends SemanticNode {
log.info("add column {} {}", datasource.getName(), field);
}
}
SchemaBuilder.addSourceView(
scope.getValidator().getCatalogReader().getRootSchema(),
db,
tb,
dateInfo,
dimensions,
metrics);
SchemaBuilder.addSourceView(scope.getValidator().getCatalogReader().getRootSchema(), db, tb,
dateInfo, dimensions, metrics);
}
public static SqlNode buildExtend(
DataSource datasource, Map<String, String> exprList, SqlValidatorScope scope)
throws Exception {
public static SqlNode buildExtend(DataSource datasource, Map<String, String> exprList,
SqlValidatorScope scope) throws Exception {
if (CollectionUtils.isEmpty(exprList)) {
return build(datasource, scope);
}
EngineType engineType = EngineType.fromString(datasource.getType());
SqlNode dataSet =
new SqlBasicCall(
new LateralViewExplodeNode(exprList),
Arrays.asList(
build(datasource, scope),
new SqlNodeList(
getExtendField(exprList, scope, engineType),
SqlParserPos.ZERO)),
SqlParserPos.ZERO);
SqlNode dataSet = new SqlBasicCall(new LateralViewExplodeNode(exprList),
Arrays.asList(build(datasource, scope), new SqlNodeList(
getExtendField(exprList, scope, engineType), SqlParserPos.ZERO)),
SqlParserPos.ZERO);
return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, dataSet);
}
public static List<SqlNode> getExtendField(
Map<String, String> exprList, SqlValidatorScope scope, EngineType engineType)
throws Exception {
public static List<SqlNode> getExtendField(Map<String, String> exprList,
SqlValidatorScope scope, EngineType engineType) throws Exception {
List<SqlNode> sqlNodeList = new ArrayList<>();
for (String expr : exprList.keySet()) {
sqlNodeList.add(parse(expr, scope, engineType));
sqlNodeList.add(
new SqlDataTypeSpec(
new SqlUserDefinedTypeNameSpec(
expr + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX,
SqlParserPos.ZERO),
SqlParserPos.ZERO));
sqlNodeList.add(new SqlDataTypeSpec(
new SqlUserDefinedTypeNameSpec(expr + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX,
SqlParserPos.ZERO),
SqlParserPos.ZERO));
}
return sqlNodeList;
}
@@ -172,45 +150,31 @@ public class DataSourceNode extends SemanticNode {
return dataSourceList.stream().map(d -> d.getName()).collect(Collectors.joining("_"));
}
public static void getQueryDimensionMeasure(
SemanticSchema schema,
MetricQueryParam metricCommand,
Set<String> queryDimension,
List<String> measures) {
queryDimension.addAll(
metricCommand.getDimensions().stream()
.map(
d ->
d.contains(Constants.DIMENSION_IDENTIFY)
? d.split(Constants.DIMENSION_IDENTIFY)[1]
: d)
.collect(Collectors.toSet()));
public static void getQueryDimensionMeasure(SemanticSchema schema,
MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures) {
queryDimension.addAll(metricCommand.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY)
? d.split(Constants.DIMENSION_IDENTIFY)[1]
: d)
.collect(Collectors.toSet()));
Set<String> schemaMetricName =
schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet());
schema.getMetrics().stream()
.filter(m -> metricCommand.getMetrics().contains(m.getName()))
.forEach(
m ->
m.getMetricTypeParams().getMeasures().stream()
.forEach(mm -> measures.add(mm.getName())));
metricCommand.getMetrics().stream()
.filter(m -> !schemaMetricName.contains(m))
schema.getMetrics().stream().filter(m -> metricCommand.getMetrics().contains(m.getName()))
.forEach(m -> m.getMetricTypeParams().getMeasures().stream()
.forEach(mm -> measures.add(mm.getName())));
metricCommand.getMetrics().stream().filter(m -> !schemaMetricName.contains(m))
.forEach(m -> measures.add(m));
}
public static void mergeQueryFilterDimensionMeasure(
SemanticSchema schema,
MetricQueryParam metricCommand,
Set<String> queryDimension,
List<String> measures,
SqlValidatorScope scope)
throws Exception {
public static void mergeQueryFilterDimensionMeasure(SemanticSchema schema,
MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures,
SqlValidatorScope scope) throws Exception {
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
Set<String> filterConditions = new HashSet<>();
FilterNode.getFilterField(
parse(metricCommand.getWhere(), scope, engineType), filterConditions);
FilterNode.getFilterField(parse(metricCommand.getWhere(), scope, engineType),
filterConditions);
Set<String> queryMeasures = new HashSet<>(measures);
Set<String> schemaMetricName =
schema.getMetrics().stream().map(m -> m.getName()).collect(Collectors.toSet());
@@ -218,11 +182,8 @@ public class DataSourceNode extends SemanticNode {
if (schemaMetricName.contains(filterCondition)) {
schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(filterCondition))
.forEach(
m ->
m.getMetricTypeParams().getMeasures().stream()
.forEach(
mm -> queryMeasures.add(mm.getName())));
.forEach(m -> m.getMetricTypeParams().getMeasures().stream()
.forEach(mm -> queryMeasures.add(mm.getName())));
continue;
}
queryDimension.add(filterCondition);
@@ -232,9 +193,8 @@ public class DataSourceNode extends SemanticNode {
}
}
public static List<DataSource> getMatchDataSources(
SqlValidatorScope scope, SemanticSchema schema, MetricQueryParam metricCommand)
throws Exception {
public static List<DataSource> getMatchDataSources(SqlValidatorScope scope,
SemanticSchema schema, MetricQueryParam metricCommand) throws Exception {
List<DataSource> dataSources = new ArrayList<>();
// check by metric
@@ -245,18 +205,14 @@ public class DataSourceNode extends SemanticNode {
// one , match measure count
Map<String, Integer> dataSourceMeasures = new HashMap<>();
for (Map.Entry<String, DataSource> entry : schema.getDatasource().entrySet()) {
Set<String> sourceMeasure =
entry.getValue().getMeasures().stream()
.map(mm -> mm.getName())
.collect(Collectors.toSet());
Set<String> sourceMeasure = entry.getValue().getMeasures().stream()
.map(mm -> mm.getName()).collect(Collectors.toSet());
sourceMeasure.retainAll(measures);
dataSourceMeasures.put(entry.getKey(), sourceMeasure.size());
}
log.info("dataSourceMeasures [{}]", dataSourceMeasures);
Optional<Map.Entry<String, Integer>> base =
dataSourceMeasures.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
.findFirst();
Optional<Map.Entry<String, Integer>> base = dataSourceMeasures.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst();
if (base.isPresent()) {
baseDataSource = schema.getDatasource().get(base.get().getKey());
dataSources.add(baseDataSource);
@@ -264,14 +220,10 @@ public class DataSourceNode extends SemanticNode {
// second , check match all dimension and metric
if (baseDataSource != null) {
Set<String> filterMeasure = new HashSet<>();
Set<String> sourceMeasure =
baseDataSource.getMeasures().stream()
.map(mm -> mm.getName())
.collect(Collectors.toSet());
Set<String> dimension =
baseDataSource.getDimensions().stream()
.map(dd -> dd.getName())
.collect(Collectors.toSet());
Set<String> sourceMeasure = baseDataSource.getMeasures().stream()
.map(mm -> mm.getName()).collect(Collectors.toSet());
Set<String> dimension = baseDataSource.getDimensions().stream().map(dd -> dd.getName())
.collect(Collectors.toSet());
baseDataSource.getIdentifiers().stream().forEach(i -> dimension.add(i.getName()));
if (schema.getDimension().containsKey(baseDataSource.getName())) {
schema.getDimension().get(baseDataSource.getName()).stream()
@@ -281,43 +233,31 @@ public class DataSourceNode extends SemanticNode {
filterMeasure.addAll(dimension);
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
mergeQueryFilterDimensionMeasure(
schema, metricCommand, queryDimension, measures, scope);
boolean isAllMatch =
checkMatch(
sourceMeasure,
queryDimension,
measures,
dimension,
metricCommand,
scope,
engineType);
mergeQueryFilterDimensionMeasure(schema, metricCommand, queryDimension, measures,
scope);
boolean isAllMatch = checkMatch(sourceMeasure, queryDimension, measures, dimension,
metricCommand, scope, engineType);
if (isAllMatch) {
log.debug("baseDataSource match all ");
return dataSources;
}
// find all dataSource has the same identifiers
List<DataSource> linkDataSources =
getLinkDataSourcesByJoinRelation(
queryDimension, measures, baseDataSource, schema);
List<DataSource> linkDataSources = getLinkDataSourcesByJoinRelation(queryDimension,
measures, baseDataSource, schema);
if (CollectionUtils.isEmpty(linkDataSources)) {
log.debug("baseDataSource get by identifiers ");
Set<String> baseIdentifiers =
baseDataSource.getIdentifiers().stream()
.map(i -> i.getName())
.collect(Collectors.toSet());
Set<String> baseIdentifiers = baseDataSource.getIdentifiers().stream()
.map(i -> i.getName()).collect(Collectors.toSet());
if (baseIdentifiers.isEmpty()) {
throw new Exception(
"datasource error : " + baseDataSource.getName() + " miss identifier");
}
linkDataSources =
getLinkDataSources(
baseIdentifiers, queryDimension, measures, baseDataSource, schema);
linkDataSources = getLinkDataSources(baseIdentifiers, queryDimension, measures,
baseDataSource, schema);
if (linkDataSources.isEmpty()) {
throw new Exception(
String.format(
"not find the match datasource : dimension[%s],measure[%s]",
queryDimension, measures));
throw new Exception(String.format(
"not find the match datasource : dimension[%s],measure[%s]",
queryDimension, measures));
}
}
log.debug("linkDataSources {}", linkDataSources);
@@ -328,15 +268,9 @@ public class DataSourceNode extends SemanticNode {
return dataSources;
}
private static boolean checkMatch(
Set<String> sourceMeasure,
Set<String> queryDimension,
List<String> measures,
Set<String> dimension,
MetricQueryParam metricCommand,
SqlValidatorScope scope,
EngineType engineType)
throws Exception {
private static boolean checkMatch(Set<String> sourceMeasure, Set<String> queryDimension,
List<String> measures, Set<String> dimension, MetricQueryParam metricCommand,
SqlValidatorScope scope, EngineType engineType) throws Exception {
boolean isAllMatch = true;
sourceMeasure.retainAll(measures);
if (sourceMeasure.size() < measures.size()) {
@@ -367,11 +301,8 @@ public class DataSourceNode extends SemanticNode {
return isAllMatch;
}
private static List<DataSource> getLinkDataSourcesByJoinRelation(
Set<String> queryDimension,
List<String> measures,
DataSource baseDataSource,
SemanticSchema schema) {
private static List<DataSource> getLinkDataSourcesByJoinRelation(Set<String> queryDimension,
List<String> measures, DataSource baseDataSource, SemanticSchema schema) {
Set<String> linkDataSourceName = new HashSet<>();
List<DataSource> linkDataSources = new ArrayList<>();
Set<String> before = new HashSet<>();
@@ -379,13 +310,9 @@ public class DataSourceNode extends SemanticNode {
if (!CollectionUtils.isEmpty(schema.getJoinRelations())) {
Set<Long> visitJoinRelations = new HashSet<>();
List<JoinRelation> sortedJoinRelation = new ArrayList<>();
sortJoinRelation(
schema.getJoinRelations(),
baseDataSource.getName(),
visitJoinRelations,
sortedJoinRelation);
schema.getJoinRelations().stream()
.filter(j -> !visitJoinRelations.contains(j.getId()))
sortJoinRelation(schema.getJoinRelations(), baseDataSource.getName(),
visitJoinRelations, sortedJoinRelation);
schema.getJoinRelations().stream().filter(j -> !visitJoinRelations.contains(j.getId()))
.forEach(j -> sortedJoinRelation.add(j));
for (JoinRelation joinRelation : sortedJoinRelation) {
if (!before.contains(joinRelation.getLeft())
@@ -394,34 +321,26 @@ public class DataSourceNode extends SemanticNode {
}
boolean isMatch = false;
boolean isRight = before.contains(joinRelation.getLeft());
DataSource other =
isRight
? schema.getDatasource().get(joinRelation.getRight())
: schema.getDatasource().get(joinRelation.getLeft());
DataSource other = isRight ? schema.getDatasource().get(joinRelation.getRight())
: schema.getDatasource().get(joinRelation.getLeft());
if (!queryDimension.isEmpty()) {
Set<String> linkDimension =
other.getDimensions().stream()
.map(dd -> dd.getName())
.collect(Collectors.toSet());
Set<String> linkDimension = other.getDimensions().stream()
.map(dd -> dd.getName()).collect(Collectors.toSet());
other.getIdentifiers().stream().forEach(i -> linkDimension.add(i.getName()));
linkDimension.retainAll(queryDimension);
if (!linkDimension.isEmpty()) {
isMatch = true;
}
}
Set<String> linkMeasure =
other.getMeasures().stream()
.map(mm -> mm.getName())
.collect(Collectors.toSet());
Set<String> linkMeasure = other.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet());
linkMeasure.retainAll(measures);
if (!linkMeasure.isEmpty()) {
isMatch = true;
}
if (!isMatch && schema.getDimension().containsKey(other.getName())) {
Set<String> linkDimension =
schema.getDimension().get(other.getName()).stream()
.map(dd -> dd.getName())
.collect(Collectors.toSet());
Set<String> linkDimension = schema.getDimension().get(other.getName()).stream()
.map(dd -> dd.getName()).collect(Collectors.toSet());
linkDimension.retainAll(queryDimension);
if (!linkDimension.isEmpty()) {
isMatch = true;
@@ -444,41 +363,30 @@ public class DataSourceNode extends SemanticNode {
orders.put(joinRelation.getRight(), 1L);
}
}
orders.entrySet().stream()
.sorted(Map.Entry.comparingByValue())
.forEach(
d -> {
linkDataSources.add(schema.getDatasource().get(d.getKey()));
});
orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> {
linkDataSources.add(schema.getDatasource().get(d.getKey()));
});
}
return linkDataSources;
}
private static void sortJoinRelation(
List<JoinRelation> joinRelations,
String next,
Set<Long> visited,
List<JoinRelation> sortedJoins) {
private static void sortJoinRelation(List<JoinRelation> joinRelations, String next,
Set<Long> visited, List<JoinRelation> sortedJoins) {
for (JoinRelation link : joinRelations) {
if (!visited.contains(link.getId())) {
if (link.getLeft().equals(next) || link.getRight().equals(next)) {
visited.add(link.getId());
sortedJoins.add(link);
sortJoinRelation(
joinRelations,
link.getLeft().equals(next) ? link.getRight() : link.getLeft(),
visited,
sortJoinRelation(joinRelations,
link.getLeft().equals(next) ? link.getRight() : link.getLeft(), visited,
sortedJoins);
}
}
}
}
private static List<DataSource> getLinkDataSources(
Set<String> baseIdentifiers,
Set<String> queryDimension,
List<String> measures,
DataSource baseDataSource,
private static List<DataSource> getLinkDataSources(Set<String> baseIdentifiers,
Set<String> queryDimension, List<String> measures, DataSource baseDataSource,
SemanticSchema schema) {
Set<String> linkDataSourceName = new HashSet<>();
List<DataSource> linkDataSources = new ArrayList<>();
@@ -486,18 +394,13 @@ public class DataSourceNode extends SemanticNode {
if (entry.getKey().equalsIgnoreCase(baseDataSource.getName())) {
continue;
}
Long identifierNum =
entry.getValue().getIdentifiers().stream()
.map(i -> i.getName())
.filter(i -> baseIdentifiers.contains(i))
.count();
Long identifierNum = entry.getValue().getIdentifiers().stream().map(i -> i.getName())
.filter(i -> baseIdentifiers.contains(i)).count();
if (identifierNum > 0) {
boolean isMatch = false;
if (!queryDimension.isEmpty()) {
Set<String> linkDimension =
entry.getValue().getDimensions().stream()
.map(dd -> dd.getName())
.collect(Collectors.toSet());
Set<String> linkDimension = entry.getValue().getDimensions().stream()
.map(dd -> dd.getName()).collect(Collectors.toSet());
entry.getValue().getIdentifiers().stream()
.forEach(i -> linkDimension.add(i.getName()));
linkDimension.retainAll(queryDimension);
@@ -506,10 +409,8 @@ public class DataSourceNode extends SemanticNode {
}
}
if (!measures.isEmpty()) {
Set<String> linkMeasure =
entry.getValue().getMeasures().stream()
.map(mm -> mm.getName())
.collect(Collectors.toSet());
Set<String> linkMeasure = entry.getValue().getMeasures().stream()
.map(mm -> mm.getName()).collect(Collectors.toSet());
linkMeasure.retainAll(measures);
if (!linkMeasure.isEmpty()) {
isMatch = true;
@@ -522,10 +423,8 @@ public class DataSourceNode extends SemanticNode {
}
for (Map.Entry<String, List<Dimension>> entry : schema.getDimension().entrySet()) {
if (!queryDimension.isEmpty()) {
Set<String> linkDimension =
entry.getValue().stream()
.map(dd -> dd.getName())
.collect(Collectors.toSet());
Set<String> linkDimension = entry.getValue().stream().map(dd -> dd.getName())
.collect(Collectors.toSet());
linkDimension.retainAll(queryDimension);
if (!linkDimension.isEmpty()) {
linkDataSourceName.add(entry.getKey());

View File

@@ -17,25 +17,24 @@ public class DimensionNode extends SemanticNode {
return buildAs(dimension.getName(), sqlNode);
}
public static List<SqlNode> expand(
Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception {
public static List<SqlNode> expand(Dimension dimension, SqlValidatorScope scope,
EngineType engineType) throws Exception {
SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType);
return expand(sqlNode, scope);
}
public static SqlNode buildName(
Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception {
public static SqlNode buildName(Dimension dimension, SqlValidatorScope scope,
EngineType engineType) throws Exception {
return parse(dimension.getName(), scope, engineType);
}
public static SqlNode buildExp(
Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception {
public static SqlNode buildExp(Dimension dimension, SqlValidatorScope scope,
EngineType engineType) throws Exception {
return parse(dimension.getExpr(), scope, engineType);
}
public static SqlNode buildNameAs(
String alias, Dimension dimension, SqlValidatorScope scope, EngineType engineType)
throws Exception {
public static SqlNode buildNameAs(String alias, Dimension dimension, SqlValidatorScope scope,
EngineType engineType) throws Exception {
if ("".equals(alias)) {
return buildName(dimension, scope, engineType);
}
@@ -43,16 +42,13 @@ public class DimensionNode extends SemanticNode {
return buildAs(alias, sqlNode);
}
public static SqlNode buildArray(
Dimension dimension, SqlValidatorScope scope, EngineType engineType) throws Exception {
public static SqlNode buildArray(Dimension dimension, SqlValidatorScope scope,
EngineType engineType) throws Exception {
if (Objects.nonNull(dimension.getDataType()) && dimension.getDataType().isArray()) {
SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType);
if (isIdentifier(sqlNode)) {
return buildAs(
dimension.getName(),
parse(
dimension.getExpr() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX,
scope,
return buildAs(dimension.getName(),
parse(dimension.getExpr() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, scope,
engineType));
}
throw new Exception("array dimension expr should only identify");

View File

@@ -18,10 +18,8 @@ public class IdentifyNode extends SemanticNode {
}
public static Set<String> getIdentifyNames(List<Identify> identifies, Identify.Type type) {
return identifies.stream()
.filter(i -> type.name().equalsIgnoreCase(i.getType()))
.map(i -> i.getName())
.collect(Collectors.toSet());
return identifies.stream().filter(i -> type.name().equalsIgnoreCase(i.getType()))
.map(i -> i.getName()).collect(Collectors.toSet());
}
public static boolean isForeign(String name, List<Identify> identifies) {

View File

@@ -7,29 +7,25 @@ import org.apache.calcite.sql.validate.SqlValidatorScope;
public class MeasureNode extends SemanticNode {
public static SqlNode buildNonAgg(
String alias, Measure measure, SqlValidatorScope scope, EngineType engineType)
throws Exception {
public static SqlNode buildNonAgg(String alias, Measure measure, SqlValidatorScope scope,
EngineType engineType) throws Exception {
return buildAs(measure.getName(), getExpr(measure, alias, scope, engineType));
}
public static SqlNode buildAgg(
Measure measure, boolean noAgg, SqlValidatorScope scope, EngineType engineType)
throws Exception {
public static SqlNode buildAgg(Measure measure, boolean noAgg, SqlValidatorScope scope,
EngineType engineType) throws Exception {
if ((measure.getAgg() == null || measure.getAgg().isEmpty()) || noAgg) {
return parse(measure.getName(), scope, engineType);
}
return buildAs(
measure.getName(),
return buildAs(measure.getName(),
AggFunctionNode.build(measure.getAgg(), measure.getName(), scope, engineType));
}
private static SqlNode getExpr(
Measure measure, String alias, SqlValidatorScope scope, EngineType enginType)
throws Exception {
private static SqlNode getExpr(Measure measure, String alias, SqlValidatorScope scope,
EngineType enginType) throws Exception {
if (measure.getExpr() == null) {
return parse(
(alias.isEmpty() ? "" : alias + ".") + measure.getName(), scope, enginType);
return parse((alias.isEmpty() ? "" : alias + ".") + measure.getName(), scope,
enginType);
}
return parse(measure.getExpr(), scope, enginType);
}

View File

@@ -22,8 +22,7 @@ public class MetricNode extends SemanticNode {
public static SqlNode build(Metric metric, SqlValidatorScope scope, EngineType engineType)
throws Exception {
if (metric.getMetricTypeParams() == null
|| metric.getMetricTypeParams().getExpr() == null
if (metric.getMetricTypeParams() == null || metric.getMetricTypeParams().getExpr() == null
|| metric.getMetricTypeParams().getExpr().isEmpty()) {
return parse(metric.getName(), scope, engineType);
}
@@ -32,10 +31,8 @@ public class MetricNode extends SemanticNode {
}
public static Boolean isMetricField(String name, SemanticSchema schema) {
Optional<Metric> metric =
schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(name))
.findFirst();
Optional<Metric> metric = schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(name)).findFirst();
return metric.isPresent() && metric.get().getMetricTypeParams().isFieldMetric();
}

View File

@@ -65,7 +65,7 @@ public abstract class SemanticNode {
AGGREGATION_KIND.add(SqlKind.SUM);
AGGREGATION_KIND.add(SqlKind.MAX);
AGGREGATION_KIND.add(SqlKind.MIN);
AGGREGATION_KIND.add(SqlKind.OTHER_FUNCTION); // more
AGGREGATION_KIND.add(SqlKind.OTHER_FUNCTION); // more
AGGREGATION_FUNC.add("sum");
AGGREGATION_FUNC.add("count");
AGGREGATION_FUNC.add("max");
@@ -75,11 +75,9 @@ public abstract class SemanticNode {
public static SqlNode parse(String expression, SqlValidatorScope scope, EngineType engineType)
throws Exception {
SqlValidatorWithHints sqlValidatorWithHints =
Configuration.getSqlValidatorWithHints(
scope.getValidator().getCatalogReader().getRootSchema(), engineType);
if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType)
.getReservedAndKeyWords()
SqlValidatorWithHints sqlValidatorWithHints = Configuration.getSqlValidatorWithHints(
scope.getValidator().getCatalogReader().getRootSchema(), engineType);
if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType).getReservedAndKeyWords()
.contains(expression.toUpperCase())) {
expression = String.format("`%s`", expression);
}
@@ -93,10 +91,8 @@ public abstract class SemanticNode {
public static SqlNode buildAs(String asName, SqlNode sqlNode) throws Exception {
SqlAsOperator sqlAsOperator = new SqlAsOperator();
SqlIdentifier sqlIdentifier = new SqlIdentifier(asName, SqlParserPos.ZERO);
return new SqlBasicCall(
sqlAsOperator,
new ArrayList<>(Arrays.asList(sqlNode, sqlIdentifier)),
SqlParserPos.ZERO);
return new SqlBasicCall(sqlAsOperator,
new ArrayList<>(Arrays.asList(sqlNode, sqlIdentifier)), SqlParserPos.ZERO);
}
public static String getSql(SqlNode sqlNode, EngineType engineType) {
@@ -154,17 +150,10 @@ public abstract class SemanticNode {
if (table instanceof SqlSelect) {
SqlSelect tableSelect = (SqlSelect) table;
return tableSelect.getSelectList().stream()
.map(
s ->
(s instanceof SqlIdentifier)
? ((SqlIdentifier) s).names.get(0)
: (((s instanceof SqlBasicCall)
&& s.getKind().equals(SqlKind.AS))
? ((SqlBasicCall) s)
.getOperandList()
.get(1)
.toString()
: ""))
.map(s -> (s instanceof SqlIdentifier) ? ((SqlIdentifier) s).names.get(0)
: (((s instanceof SqlBasicCall) && s.getKind().equals(SqlKind.AS))
? ((SqlBasicCall) s).getOperandList().get(1).toString()
: ""))
.collect(Collectors.toSet());
}
return new HashSet<>();
@@ -192,10 +181,8 @@ public abstract class SemanticNode {
case AS:
SqlBasicCall sqlBasicCall = (SqlBasicCall) sqlNode;
if (sqlBasicCall.getOperandList().get(0).getKind().equals(SqlKind.IDENTIFIER)) {
addTableName(
sqlBasicCall.getOperandList().get(0).toString(),
sqlBasicCall.getOperandList().get(1).toString(),
parseInfo);
addTableName(sqlBasicCall.getOperandList().get(0).toString(),
sqlBasicCall.getOperandList().get(1).toString(), parseInfo);
} else {
sqlVisit(sqlBasicCall.getOperandList().get(0), parseInfo);
}
@@ -211,12 +198,9 @@ public abstract class SemanticNode {
}
break;
case UNION:
((SqlBasicCall) sqlNode)
.getOperandList()
.forEach(
node -> {
sqlVisit(node, parseInfo);
});
((SqlBasicCall) sqlNode).getOperandList().forEach(node -> {
sqlVisit(node, parseInfo);
});
break;
case WITH:
SqlWith sqlWith = (SqlWith) sqlNode;
@@ -233,12 +217,9 @@ public abstract class SemanticNode {
}
SqlSelect sqlSelect = (SqlSelect) select;
SqlNodeList selectList = sqlSelect.getSelectList();
selectList
.getList()
.forEach(
list -> {
fieldVisit(list, parseInfo, "");
});
selectList.getList().forEach(list -> {
fieldVisit(list, parseInfo, "");
});
fromVisit(sqlSelect.getFrom(), parseInfo);
if (sqlSelect.hasWhere()) {
whereVisit((SqlBasicCall) sqlSelect.getWhere(), parseInfo);
@@ -248,17 +229,16 @@ public abstract class SemanticNode {
}
SqlNodeList group = sqlSelect.getGroup();
if (group != null) {
group.forEach(
groupField -> {
if (groupHints.contains(groupField.toString())) {
int groupIdx = Integer.valueOf(groupField.toString()) - 1;
if (selectList.getList().size() > groupIdx) {
fieldVisit(selectList.get(groupIdx), parseInfo, "");
}
} else {
fieldVisit(groupField, parseInfo, "");
}
});
group.forEach(groupField -> {
if (groupHints.contains(groupField.toString())) {
int groupIdx = Integer.valueOf(groupField.toString()) - 1;
if (selectList.getList().size() > groupIdx) {
fieldVisit(selectList.get(groupIdx), parseInfo, "");
}
} else {
fieldVisit(groupField, parseInfo, "");
}
});
}
}
@@ -266,17 +246,15 @@ public abstract class SemanticNode {
if (where == null) {
return;
}
if (where.operandCount() == 2
&& where.operand(0).getKind().equals(SqlKind.IDENTIFIER)
if (where.operandCount() == 2 && where.operand(0).getKind().equals(SqlKind.IDENTIFIER)
&& where.operand(1).getKind().equals(SqlKind.LITERAL)) {
fieldVisit(where.operand(0), parseInfo, "");
return;
}
// 子查询
if (where.operandCount() == 2
&& (where.operand(0).getKind().equals(SqlKind.IDENTIFIER)
&& (where.operand(1).getKind().equals(SqlKind.SELECT)
|| where.operand(1).getKind().equals(SqlKind.ORDER_BY)))) {
if (where.operandCount() == 2 && (where.operand(0).getKind().equals(SqlKind.IDENTIFIER)
&& (where.operand(1).getKind().equals(SqlKind.SELECT)
|| where.operand(1).getKind().equals(SqlKind.ORDER_BY)))) {
fieldVisit(where.operand(0), parseInfo, "");
sqlVisit((SqlNode) (where.operand(1)), parseInfo);
return;
@@ -331,12 +309,9 @@ public abstract class SemanticNode {
}
}
if (field instanceof SqlNodeList) {
((SqlNodeList) field)
.getList()
.forEach(
node -> {
fieldVisit(node, parseInfo, "");
});
((SqlNodeList) field).getList().forEach(node -> {
fieldVisit(node, parseInfo, "");
});
}
}
@@ -421,10 +396,7 @@ public abstract class SemanticNode {
return parseInfo;
}
public static SqlNode optimize(
SqlValidatorScope scope,
SemanticSchema schema,
SqlNode sqlNode,
public static SqlNode optimize(SqlValidatorScope scope, SemanticSchema schema, SqlNode sqlNode,
EngineType engineType) {
try {
HepProgramBuilder hepProgramBuilder = new HepProgramBuilder();
@@ -433,16 +405,13 @@ public abstract class SemanticNode {
new FilterToGroupScanRule(FilterToGroupScanRule.DEFAULT, schema));
RelOptPlanner relOptPlanner = new HepPlanner(hepProgramBuilder.build());
RelToSqlConverter converter = new RelToSqlConverter(sqlDialect);
SqlValidator sqlValidator =
Configuration.getSqlValidator(
scope.getValidator().getCatalogReader().getRootSchema(), engineType);
SqlToRelConverter sqlToRelConverter =
Configuration.getSqlToRelConverter(
scope, sqlValidator, relOptPlanner, engineType);
SqlValidator sqlValidator = Configuration.getSqlValidator(
scope.getValidator().getCatalogReader().getRootSchema(), engineType);
SqlToRelConverter sqlToRelConverter = Configuration.getSqlToRelConverter(scope,
sqlValidator, relOptPlanner, engineType);
RelNode sqlRel =
sqlToRelConverter.convertQuery(sqlValidator.validate(sqlNode), false, true).rel;
log.debug(
"RelNode optimize {}",
log.debug("RelNode optimize {}",
SemanticNode.getSql(converter.visitRoot(sqlRel).asStatement(), engineType));
relOptPlanner.setRoot(sqlRel);
RelNode relNode = relOptPlanner.findBestExp();

View File

@@ -30,29 +30,15 @@ import java.util.Optional;
public class FilterToGroupScanRule extends RelRule<Config> implements TransformationRule {
public static FilterTableScanRule.Config DEFAULT =
FilterTableScanRule.Config.DEFAULT
.withOperandSupplier(
(b0) -> {
return b0.operand(LogicalFilter.class)
.oneInput(
(b1) -> {
return b1.operand(LogicalProject.class)
.oneInput(
(b2) -> {
return b2.operand(
LogicalAggregate
.class)
.oneInput(
(b3) -> {
return b3.operand(
LogicalProject
.class)
.anyInputs();
});
});
});
})
.as(FilterTableScanRule.Config.class);
FilterTableScanRule.Config.DEFAULT.withOperandSupplier((b0) -> {
return b0.operand(LogicalFilter.class).oneInput((b1) -> {
return b1.operand(LogicalProject.class).oneInput((b2) -> {
return b2.operand(LogicalAggregate.class).oneInput((b3) -> {
return b3.operand(LogicalProject.class).anyInputs();
});
});
});
}).as(FilterTableScanRule.Config.class);
private SemanticSchema semanticSchema;
@@ -75,19 +61,16 @@ public class FilterToGroupScanRule extends RelRule<Config> implements Transforma
Project project0 = (Project) call.rel(1);
Project project1 = (Project) call.rel(3);
Aggregate logicalAggregate = (Aggregate) call.rel(2);
Optional<Pair<RexNode, String>> isIn =
project1.getNamedProjects().stream()
.filter(i -> i.right.equalsIgnoreCase(minMax.getLeft()))
.findFirst();
Optional<Pair<RexNode, String>> isIn = project1.getNamedProjects().stream()
.filter(i -> i.right.equalsIgnoreCase(minMax.getLeft())).findFirst();
if (!isIn.isPresent()) {
return;
}
RelBuilder relBuilder = call.builder();
relBuilder.push(project1);
RexNode addPartitionCondition =
getRexNodeByTimeRange(
relBuilder, minMax.getLeft(), minMax.getMiddle(), minMax.getRight());
RexNode addPartitionCondition = getRexNodeByTimeRange(relBuilder, minMax.getLeft(),
minMax.getMiddle(), minMax.getRight());
relBuilder.filter(new RexNode[] {addPartitionCondition});
relBuilder.project(project1.getProjects());
ImmutableBitSet newGroupSet = logicalAggregate.getGroupSet();
@@ -97,13 +80,8 @@ public class FilterToGroupScanRule extends RelRule<Config> implements Transforma
Iterator var = logicalAggregate.getAggCallList().iterator();
while (var.hasNext()) {
AggregateCall aggCall = (AggregateCall) var.next();
newAggCalls.add(
aggCall.adaptTo(
project1,
aggCall.getArgList(),
aggCall.filterArg,
groupCount,
newGroupCount));
newAggCalls.add(aggCall.adaptTo(project1, aggCall.getArgList(), aggCall.filterArg,
groupCount, newGroupCount));
}
relBuilder.aggregate(relBuilder.groupKey(newGroupSet), newAggCalls);
relBuilder.project(project0.getProjects());
@@ -111,17 +89,12 @@ public class FilterToGroupScanRule extends RelRule<Config> implements Transforma
call.transformTo(relBuilder.build());
}
private RexNode getRexNodeByTimeRange(
RelBuilder relBuilder, String dateField, String start, String end) {
return relBuilder.call(
SqlStdOperatorTable.AND,
relBuilder.call(
SqlStdOperatorTable.GREATER_THAN_OR_EQUAL,
relBuilder.field(dateField),
relBuilder.literal(start)),
relBuilder.call(
SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
relBuilder.field(dateField),
private RexNode getRexNodeByTimeRange(RelBuilder relBuilder, String dateField, String start,
String end) {
return relBuilder.call(SqlStdOperatorTable.AND,
relBuilder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL,
relBuilder.field(dateField), relBuilder.literal(start)),
relBuilder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(dateField),
relBuilder.literal(end)));
}
}

View File

@@ -27,13 +27,8 @@ import java.util.stream.Collectors;
public class FilterRender extends Renderer {
@Override
public void render(
MetricQueryParam metricCommand,
List<DataSource> dataSources,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
throws Exception {
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception {
TableView tableView = super.tableView;
SqlNode filterNode = null;
List<String> queryMetrics = new ArrayList<>(metricCommand.getMetrics());
@@ -49,14 +44,8 @@ public class FilterRender extends Renderer {
Set<String> dimensions = new HashSet<>();
Set<String> metrics = new HashSet<>();
for (DataSource dataSource : dataSources) {
SourceRender.whereDimMetric(
fieldWhere,
metricCommand.getMetrics(),
metricCommand.getDimensions(),
dataSource,
schema,
dimensions,
metrics);
SourceRender.whereDimMetric(fieldWhere, metricCommand.getMetrics(),
metricCommand.getDimensions(), dataSource, schema, dimensions, metrics);
}
queryMetrics.addAll(metrics);
queryDimensions.addAll(dimensions);
@@ -71,8 +60,7 @@ public class FilterRender extends Renderer {
continue;
}
if (optionalMetric.isPresent()) {
tableView
.getMeasure()
tableView.getMeasure()
.add(MetricNode.build(optionalMetric.get(), scope, engineType));
} else {
tableView.getMeasure().add(SemanticNode.parse(metric, scope, engineType));
@@ -80,9 +68,8 @@ public class FilterRender extends Renderer {
}
if (filterNode != null) {
TableView filterView = new TableView();
filterView.setTable(
SemanticNode.buildAs(
Constants.DATASOURCE_TABLE_FILTER_PREFIX, tableView.build()));
filterView.setTable(SemanticNode.buildAs(Constants.DATASOURCE_TABLE_FILTER_PREFIX,
tableView.build()));
filterView.getFilter().add(filterNode);
filterView.getMeasure().add(SqlIdentifier.star(SqlParserPos.ZERO));
super.tableView = filterView;

View File

@@ -48,13 +48,8 @@ import java.util.stream.Collectors;
public class JoinRender extends Renderer {
@Override
public void render(
MetricQueryParam metricCommand,
List<DataSource> dataSources,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
throws Exception {
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere();
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -82,14 +77,8 @@ public class JoinRender extends Renderer {
final Set<String> filterMetrics = new HashSet<>();
final List<String> queryDimension = new ArrayList<>();
final List<String> queryMetrics = new ArrayList<>();
SourceRender.whereDimMetric(
fieldWhere,
queryMetrics,
queryDimension,
dataSource,
schema,
filterDimensions,
filterMetrics);
SourceRender.whereDimMetric(fieldWhere, queryMetrics, queryDimension, dataSource,
schema, filterDimensions, filterMetrics);
List<String> reqMetric = new ArrayList<>(metricCommand.getMetrics());
reqMetric.addAll(filterMetrics);
reqMetric = uniqList(reqMetric);
@@ -98,33 +87,14 @@ public class JoinRender extends Renderer {
reqDimension.addAll(filterDimensions);
reqDimension = uniqList(reqDimension);
Set<String> sourceMeasure =
dataSource.getMeasures().stream()
.map(mm -> mm.getName())
.collect(Collectors.toSet());
doMetric(
innerSelect,
filterView,
queryMetrics,
reqMetric,
dataSource,
sourceMeasure,
scope,
schema,
nonAgg);
Set<String> dimension =
dataSource.getDimensions().stream()
.map(dd -> dd.getName())
.collect(Collectors.toSet());
doDimension(
innerSelect,
filterDimension,
queryDimension,
reqDimension,
dataSource,
dimension,
scope,
schema);
Set<String> sourceMeasure = dataSource.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet());
doMetric(innerSelect, filterView, queryMetrics, reqMetric, dataSource, sourceMeasure,
scope, schema, nonAgg);
Set<String> dimension = dataSource.getDimensions().stream().map(dd -> dd.getName())
.collect(Collectors.toSet());
doDimension(innerSelect, filterDimension, queryDimension, reqDimension, dataSource,
dimension, scope, schema);
List<String> primary = new ArrayList<>();
for (Identify identify : dataSource.getIdentifiers()) {
primary.add(identify.getName());
@@ -135,16 +105,8 @@ public class JoinRender extends Renderer {
List<String> dataSourceWhere = new ArrayList<>(fieldWhere);
addZipperField(dataSource, dataSourceWhere);
TableView tableView =
SourceRender.renderOne(
"",
dataSourceWhere,
queryMetrics,
queryDimension,
metricCommand.getWhere(),
dataSources.get(i),
scope,
schema,
true);
SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension,
metricCommand.getWhere(), dataSources.get(i), scope, schema, true);
log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString()));
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
tableView.setAlias(alias);
@@ -165,8 +127,8 @@ public class JoinRender extends Renderer {
innerView.getMeasure().add(entry.getValue());
}
innerView.setTable(left);
filterView.setTable(
SemanticNode.buildAs(Constants.JOIN_TABLE_OUT_PREFIX, innerView.build()));
filterView
.setTable(SemanticNode.buildAs(Constants.JOIN_TABLE_OUT_PREFIX, innerView.build()));
if (!filterDimension.isEmpty()) {
for (String d : getQueryDimension(filterDimension, queryAllDimension, whereFields)) {
if (nonAgg) {
@@ -179,17 +141,10 @@ public class JoinRender extends Renderer {
super.tableView = filterView;
}
private void doMetric(
Map<String, SqlNode> innerSelect,
TableView filterView,
List<String> queryMetrics,
List<String> reqMetrics,
DataSource dataSource,
Set<String> sourceMeasure,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
throws Exception {
private void doMetric(Map<String, SqlNode> innerSelect, TableView filterView,
List<String> queryMetrics, List<String> reqMetrics, DataSource dataSource,
Set<String> sourceMeasure, SqlValidatorScope scope, SemanticSchema schema,
boolean nonAgg) throws Exception {
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -200,38 +155,21 @@ public class JoinRender extends Renderer {
if (!metricNode.getNonAggNode().isEmpty()) {
for (String measure : metricNode.getNonAggNode().keySet()) {
innerSelect.put(
measure,
SemanticNode.buildAs(
measure,
SemanticNode.parse(
alias + "." + measure, scope, engineType)));
innerSelect.put(measure, SemanticNode.buildAs(measure,
SemanticNode.parse(alias + "." + measure, scope, engineType)));
}
}
if (metricNode.getAggFunction() != null && !metricNode.getAggFunction().isEmpty()) {
for (Map.Entry<String, String> entry : metricNode.getAggFunction().entrySet()) {
if (metricNode.getNonAggNode().containsKey(entry.getKey())) {
if (nonAgg) {
filterView
.getMeasure()
.add(
SemanticNode.buildAs(
entry.getKey(),
SemanticNode.parse(
entry.getKey(),
scope,
engineType)));
filterView.getMeasure().add(SemanticNode.buildAs(entry.getKey(),
SemanticNode.parse(entry.getKey(), scope, engineType)));
} else {
filterView
.getMeasure()
.add(
SemanticNode.buildAs(
entry.getKey(),
AggFunctionNode.build(
entry.getValue(),
entry.getKey(),
scope,
engineType)));
filterView.getMeasure()
.add(SemanticNode.buildAs(entry.getKey(),
AggFunctionNode.build(entry.getValue(),
entry.getKey(), scope, engineType)));
}
}
}
@@ -240,15 +178,9 @@ public class JoinRender extends Renderer {
}
}
private void doDimension(
Map<String, SqlNode> innerSelect,
Set<String> filterDimension,
List<String> queryDimension,
List<String> reqDimensions,
DataSource dataSource,
Set<String> dimension,
SqlValidatorScope scope,
SemanticSchema schema)
private void doDimension(Map<String, SqlNode> innerSelect, Set<String> filterDimension,
List<String> queryDimension, List<String> reqDimensions, DataSource dataSource,
Set<String> dimension, SqlValidatorScope scope, SemanticSchema schema)
throws Exception {
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
EngineType engineType =
@@ -257,44 +189,32 @@ public class JoinRender extends Renderer {
if (getMatchDimension(schema, dimension, dataSource, d, queryDimension)) {
if (d.contains(Constants.DIMENSION_IDENTIFY)) {
String[] identifyDimension = d.split(Constants.DIMENSION_IDENTIFY);
innerSelect.put(
d,
SemanticNode.buildAs(
d,
SemanticNode.parse(
alias + "." + identifyDimension[1],
scope,
engineType)));
innerSelect.put(d, SemanticNode.buildAs(d, SemanticNode
.parse(alias + "." + identifyDimension[1], scope, engineType)));
} else {
innerSelect.put(
d,
SemanticNode.buildAs(
d, SemanticNode.parse(alias + "." + d, scope, engineType)));
innerSelect.put(d, SemanticNode.buildAs(d,
SemanticNode.parse(alias + "." + d, scope, engineType)));
}
filterDimension.add(d);
}
}
}
private Set<String> getQueryDimension(
Set<String> filterDimension, Set<String> queryAllDimension, Set<String> whereFields) {
private Set<String> getQueryDimension(Set<String> filterDimension,
Set<String> queryAllDimension, Set<String> whereFields) {
return filterDimension.stream()
.filter(d -> queryAllDimension.contains(d) || whereFields.contains(d))
.collect(Collectors.toSet());
}
private boolean getMatchMetric(
SemanticSchema schema, Set<String> sourceMeasure, String m, List<String> queryMetrics) {
Optional<Metric> metric =
schema.getMetrics().stream()
.filter(mm -> mm.getName().equalsIgnoreCase(m))
.findFirst();
private boolean getMatchMetric(SemanticSchema schema, Set<String> sourceMeasure, String m,
List<String> queryMetrics) {
Optional<Metric> metric = schema.getMetrics().stream()
.filter(mm -> mm.getName().equalsIgnoreCase(m)).findFirst();
boolean isAdd = false;
if (metric.isPresent()) {
Set<String> metricMeasures =
metric.get().getMetricTypeParams().getMeasures().stream()
.map(me -> me.getName())
.collect(Collectors.toSet());
Set<String> metricMeasures = metric.get().getMetricTypeParams().getMeasures().stream()
.map(me -> me.getName()).collect(Collectors.toSet());
if (sourceMeasure.containsAll(metricMeasures)) {
isAdd = true;
}
@@ -308,12 +228,8 @@ public class JoinRender extends Renderer {
return isAdd;
}
private boolean getMatchDimension(
SemanticSchema schema,
Set<String> sourceDimension,
DataSource dataSource,
String d,
List<String> queryDimension) {
private boolean getMatchDimension(SemanticSchema schema, Set<String> sourceDimension,
DataSource dataSource, String d, List<String> queryDimension) {
String oriDimension = d;
boolean isAdd = false;
if (d.contains(Constants.DIMENSION_IDENTIFY)) {
@@ -345,15 +261,9 @@ public class JoinRender extends Renderer {
return SemanticNode.getTable(tableView.getTable());
}
private SqlNode buildJoin(
SqlNode left,
TableView leftTable,
TableView tableView,
Map<String, String> before,
DataSource dataSource,
SemanticSchema schema,
SqlValidatorScope scope)
throws Exception {
private SqlNode buildJoin(SqlNode left, TableView leftTable, TableView tableView,
Map<String, String> before, DataSource dataSource, SemanticSchema schema,
SqlValidatorScope scope) throws Exception {
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
SqlNode condition =
@@ -367,53 +277,37 @@ public class JoinRender extends Renderer {
condition = joinRelationCondition;
}
if (Materialization.TimePartType.ZIPPER.equals(leftTable.getDataSource().getTimePartType())
|| Materialization.TimePartType.ZIPPER.equals(
tableView.getDataSource().getTimePartType())) {
|| Materialization.TimePartType.ZIPPER
.equals(tableView.getDataSource().getTimePartType())) {
SqlNode zipperCondition =
getZipperCondition(leftTable, tableView, dataSource, schema, scope);
if (Objects.nonNull(joinRelationCondition)) {
condition =
new SqlBasicCall(
SqlStdOperatorTable.AND,
new ArrayList<>(
Arrays.asList(zipperCondition, joinRelationCondition)),
SqlParserPos.ZERO,
null);
condition = new SqlBasicCall(SqlStdOperatorTable.AND,
new ArrayList<>(Arrays.asList(zipperCondition, joinRelationCondition)),
SqlParserPos.ZERO, null);
} else {
condition = zipperCondition;
}
}
return new SqlJoin(
SqlParserPos.ZERO,
left,
SqlLiteral.createBoolean(false, SqlParserPos.ZERO),
sqlLiteral,
return new SqlJoin(SqlParserPos.ZERO, left,
SqlLiteral.createBoolean(false, SqlParserPos.ZERO), sqlLiteral,
SemanticNode.buildAs(tableView.getAlias(), getTable(tableView, scope)),
SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO),
condition);
SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO), condition);
}
private JoinRelation getMatchJoinRelation(
Map<String, String> before, TableView tableView, SemanticSchema schema) {
private JoinRelation getMatchJoinRelation(Map<String, String> before, TableView tableView,
SemanticSchema schema) {
JoinRelation matchJoinRelation = JoinRelation.builder().build();
if (!CollectionUtils.isEmpty(schema.getJoinRelations())) {
for (JoinRelation joinRelation : schema.getJoinRelations()) {
if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataSource().getName())
&& before.containsKey(joinRelation.getLeft())) {
matchJoinRelation.setJoinCondition(
joinRelation.getJoinCondition().stream()
.map(
r ->
Triple.of(
before.get(joinRelation.getLeft())
+ "."
+ r.getLeft(),
r.getMiddle(),
tableView.getAlias()
+ "."
+ r.getRight()))
.collect(Collectors.toList()));
matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream()
.map(r -> Triple.of(
before.get(joinRelation.getLeft()) + "." + r.getLeft(),
r.getMiddle(), tableView.getAlias() + "." + r.getRight()))
.collect(Collectors.toList()));
matchJoinRelation.setJoinType(joinRelation.getJoinType());
}
}
@@ -421,46 +315,29 @@ public class JoinRender extends Renderer {
return matchJoinRelation;
}
private SqlNode getCondition(
JoinRelation joinRelation, SqlValidatorScope scope, EngineType engineType)
throws Exception {
private SqlNode getCondition(JoinRelation joinRelation, SqlValidatorScope scope,
EngineType engineType) throws Exception {
SqlNode condition = null;
for (Triple<String, String, String> con : joinRelation.getJoinCondition()) {
List<SqlNode> ons = new ArrayList<>();
ons.add(SemanticNode.parse(con.getLeft(), scope, engineType));
ons.add(SemanticNode.parse(con.getRight(), scope, engineType));
if (Objects.isNull(condition)) {
condition =
new SqlBasicCall(
SemanticNode.getBinaryOperator(con.getMiddle()),
ons,
SqlParserPos.ZERO,
null);
condition = new SqlBasicCall(SemanticNode.getBinaryOperator(con.getMiddle()), ons,
SqlParserPos.ZERO, null);
continue;
}
SqlNode addCondition =
new SqlBasicCall(
SemanticNode.getBinaryOperator(con.getMiddle()),
ons,
SqlParserPos.ZERO,
null);
condition =
new SqlBasicCall(
SqlStdOperatorTable.AND,
new ArrayList<>(Arrays.asList(condition, addCondition)),
SqlParserPos.ZERO,
null);
SqlNode addCondition = new SqlBasicCall(SemanticNode.getBinaryOperator(con.getMiddle()),
ons, SqlParserPos.ZERO, null);
condition = new SqlBasicCall(SqlStdOperatorTable.AND,
new ArrayList<>(Arrays.asList(condition, addCondition)), SqlParserPos.ZERO,
null);
}
return condition;
}
private SqlNode getCondition(
TableView left,
TableView right,
DataSource dataSource,
SemanticSchema schema,
SqlValidatorScope scope,
EngineType engineType)
private SqlNode getCondition(TableView left, TableView right, DataSource dataSource,
SemanticSchema schema, SqlValidatorScope scope, EngineType engineType)
throws Exception {
Set<String> selectLeft = SemanticNode.getSelect(left.getTable());
@@ -491,22 +368,15 @@ public class JoinRender extends Renderer {
}
SqlNode addCondition =
new SqlBasicCall(SqlStdOperatorTable.EQUALS, ons, SqlParserPos.ZERO, null);
condition =
new SqlBasicCall(
SqlStdOperatorTable.AND,
new ArrayList<>(Arrays.asList(condition, addCondition)),
SqlParserPos.ZERO,
null);
condition = new SqlBasicCall(SqlStdOperatorTable.AND,
new ArrayList<>(Arrays.asList(condition, addCondition)), SqlParserPos.ZERO,
null);
}
return condition;
}
private static void joinOrder(
int cnt,
String id,
Map<String, Set<String>> next,
Queue<String> orders,
Map<String, Boolean> visited) {
private static void joinOrder(int cnt, String id, Map<String, Set<String>> next,
Queue<String> orders, Map<String, Boolean> visited) {
visited.put(id, true);
orders.add(id);
if (orders.size() >= cnt) {
@@ -528,97 +398,62 @@ public class JoinRender extends Renderer {
if (Materialization.TimePartType.ZIPPER.equals(dataSource.getTimePartType())) {
dataSource.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.forEach(
t -> {
if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END)
&& !fields.contains(t.getName())) {
fields.add(t.getName());
}
if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START)
&& !fields.contains(t.getName())) {
fields.add(t.getName());
}
});
.forEach(t -> {
if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END)
&& !fields.contains(t.getName())) {
fields.add(t.getName());
}
if (t.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START)
&& !fields.contains(t.getName())) {
fields.add(t.getName());
}
});
}
}
private SqlNode getZipperCondition(
TableView left,
TableView right,
DataSource dataSource,
SemanticSchema schema,
SqlValidatorScope scope)
throws Exception {
private SqlNode getZipperCondition(TableView left, TableView right, DataSource dataSource,
SemanticSchema schema, SqlValidatorScope scope) throws Exception {
if (Materialization.TimePartType.ZIPPER.equals(left.getDataSource().getTimePartType())
&& Materialization.TimePartType.ZIPPER.equals(
right.getDataSource().getTimePartType())) {
&& Materialization.TimePartType.ZIPPER
.equals(right.getDataSource().getTimePartType())) {
throw new Exception("not support two zipper table");
}
SqlNode condition = null;
Optional<Dimension> leftTime =
left.getDataSource().getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
Optional<Dimension> rightTime =
right.getDataSource().getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
Optional<Dimension> leftTime = left.getDataSource().getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
Optional<Dimension> rightTime = right.getDataSource().getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
if (leftTime.isPresent() && rightTime.isPresent()) {
String startTime = "";
String endTime = "";
String dateTime = "";
Optional<Dimension> startTimeOp =
(Materialization.TimePartType.ZIPPER.equals(
left.getDataSource().getTimePartType())
? left
: right)
.getDataSource().getDimensions().stream()
.filter(
d ->
Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(
d.getType()))
.filter(
d ->
d.getName()
.startsWith(
Constants
.MATERIALIZATION_ZIPPER_START))
.findFirst();
Optional<Dimension> endTimeOp =
(Materialization.TimePartType.ZIPPER.equals(
left.getDataSource().getTimePartType())
? left
: right)
.getDataSource().getDimensions().stream()
.filter(
d ->
Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(
d.getType()))
.filter(
d ->
d.getName()
.startsWith(
Constants
.MATERIALIZATION_ZIPPER_END))
.findFirst();
Optional<Dimension> startTimeOp = (Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? left : right).getDataSource()
.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME
.equalsIgnoreCase(d.getType()))
.filter(d -> d.getName()
.startsWith(Constants.MATERIALIZATION_ZIPPER_START))
.findFirst();
Optional<Dimension> endTimeOp = (Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? left : right).getDataSource()
.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME
.equalsIgnoreCase(d.getType()))
.filter(d -> d.getName()
.startsWith(Constants.MATERIALIZATION_ZIPPER_END))
.findFirst();
if (startTimeOp.isPresent() && endTimeOp.isPresent()) {
TableView zipper =
Materialization.TimePartType.ZIPPER.equals(
left.getDataSource().getTimePartType())
? left
: right;
TableView partMetric =
Materialization.TimePartType.ZIPPER.equals(
left.getDataSource().getTimePartType())
? right
: left;
Optional<Dimension> partTime =
Materialization.TimePartType.ZIPPER.equals(
left.getDataSource().getTimePartType())
? rightTime
: leftTime;
TableView zipper = Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? left : right;
TableView partMetric = Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? right : left;
Optional<Dimension> partTime = Materialization.TimePartType.ZIPPER
.equals(left.getDataSource().getTimePartType()) ? rightTime : leftTime;
startTime = zipper.getAlias() + "." + startTimeOp.get().getName();
endTime = zipper.getAlias() + "." + endTimeOp.get().getName();
dateTime = partMetric.getAlias() + "." + partTime.get().getName();
@@ -626,36 +461,18 @@ public class JoinRender extends Renderer {
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
ArrayList<SqlNode> operandList =
new ArrayList<>(
Arrays.asList(
SemanticNode.parse(endTime, scope, engineType),
SemanticNode.parse(dateTime, scope, engineType)));
condition =
new SqlBasicCall(
SqlStdOperatorTable.AND,
new ArrayList<SqlNode>(
Arrays.asList(
new SqlBasicCall(
SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
new ArrayList<SqlNode>(
Arrays.asList(
SemanticNode.parse(
startTime,
scope,
engineType),
SemanticNode.parse(
dateTime,
scope,
engineType))),
SqlParserPos.ZERO,
null),
new SqlBasicCall(
SqlStdOperatorTable.GREATER_THAN,
operandList,
SqlParserPos.ZERO,
null))),
SqlParserPos.ZERO,
null);
new ArrayList<>(Arrays.asList(SemanticNode.parse(endTime, scope, engineType),
SemanticNode.parse(dateTime, scope, engineType)));
condition = new SqlBasicCall(SqlStdOperatorTable.AND,
new ArrayList<SqlNode>(Arrays.asList(
new SqlBasicCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
new ArrayList<SqlNode>(Arrays.asList(
SemanticNode.parse(startTime, scope, engineType),
SemanticNode.parse(dateTime, scope, engineType))),
SqlParserPos.ZERO, null),
new SqlBasicCall(SqlStdOperatorTable.GREATER_THAN, operandList,
SqlParserPos.ZERO, null))),
SqlParserPos.ZERO, null);
}
return condition;
}

View File

@@ -23,13 +23,8 @@ import java.util.List;
public class OutputRender extends Renderer {
@Override
public void render(
MetricQueryParam metricCommand,
List<DataSource> dataSources,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
throws Exception {
public void render(MetricQueryParam metricCommand, List<DataSource> dataSources,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception {
TableView selectDataSet = super.tableView;
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -53,12 +48,9 @@ public class OutputRender extends Renderer {
List<SqlNode> orderList = new ArrayList<>();
for (ColumnOrder columnOrder : metricCommand.getOrder()) {
if (SqlStdOperatorTable.DESC.getName().equalsIgnoreCase(columnOrder.getOrder())) {
orderList.add(
SqlStdOperatorTable.DESC.createCall(
SqlParserPos.ZERO,
new SqlNode[] {
SemanticNode.parse(columnOrder.getCol(), scope, engineType)
}));
orderList.add(SqlStdOperatorTable.DESC.createCall(SqlParserPos.ZERO,
new SqlNode[] {SemanticNode.parse(columnOrder.getCol(), scope,
engineType)}));
} else {
orderList.add(SemanticNode.parse(columnOrder.getCol(), scope, engineType));
}

View File

@@ -42,16 +42,9 @@ import static com.tencent.supersonic.headless.core.translator.calcite.s2sql.Cons
@Slf4j
public class SourceRender extends Renderer {
public static TableView renderOne(
String alias,
List<String> fieldWheres,
List<String> reqMetrics,
List<String> reqDimensions,
String queryWhere,
DataSource datasource,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
public static TableView renderOne(String alias, List<String> fieldWheres,
List<String> reqMetrics, List<String> reqDimensions, String queryWhere,
DataSource datasource, SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg)
throws Exception {
TableView dataSet = new TableView();
@@ -63,29 +56,14 @@ public class SourceRender extends Renderer {
if (!fieldWhere.isEmpty()) {
Set<String> dimensions = new HashSet<>();
Set<String> metrics = new HashSet<>();
whereDimMetric(
fieldWhere,
queryMetrics,
queryDimensions,
datasource,
schema,
dimensions,
metrics);
whereDimMetric(fieldWhere, queryMetrics, queryDimensions, datasource, schema,
dimensions, metrics);
queryMetrics.addAll(metrics);
queryMetrics = uniqList(queryMetrics);
queryDimensions.addAll(dimensions);
queryDimensions = uniqList(queryDimensions);
mergeWhere(
fieldWhere,
dataSet,
output,
queryMetrics,
queryDimensions,
extendFields,
datasource,
scope,
schema,
nonAgg);
mergeWhere(fieldWhere, dataSet, output, queryMetrics, queryDimensions, extendFields,
datasource, scope, schema, nonAgg);
}
addTimeDimension(datasource, queryDimensions);
for (String metric : queryMetrics) {
@@ -109,18 +87,11 @@ public class SourceRender extends Renderer {
&& queryDimensions.contains(dimension.split(Constants.DIMENSION_IDENTIFY)[1])) {
continue;
}
buildDimension(
dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension : "",
buildDimension(dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension : "",
dimension.contains(Constants.DIMENSION_IDENTIFY)
? dimension.split(Constants.DIMENSION_IDENTIFY)[1]
: dimension,
datasource,
schema,
nonAgg,
extendFields,
dataSet,
output,
scope);
datasource, schema, nonAgg, extendFields, dataSet, output, scope);
}
output.setMeasure(deduplicateNode(output.getMeasure()));
@@ -129,12 +100,8 @@ public class SourceRender extends Renderer {
SqlNode tableNode = DataSourceNode.buildExtend(datasource, extendFields, scope);
dataSet.setTable(tableNode);
output.setTable(
SemanticNode.buildAs(
Constants.DATASOURCE_TABLE_OUT_PREFIX
+ datasource.getName()
+ "_"
+ UUID.randomUUID().toString().substring(32),
dataSet.build()));
SemanticNode.buildAs(Constants.DATASOURCE_TABLE_OUT_PREFIX + datasource.getName()
+ "_" + UUID.randomUUID().toString().substring(32), dataSet.build()));
return output;
}
@@ -148,8 +115,7 @@ public class SourceRender extends Renderer {
return uniqueElements;
}
private static boolean containsElement(
List<SqlNode> list, SqlNode element) { // 检查List<SqlNode>中是否含有某element
private static boolean containsElement(List<SqlNode> list, SqlNode element) { // 检查List<SqlNode>中是否含有某element
for (SqlNode i : list) {
if (i.equalsDeep(element, Litmus.IGNORE)) {
return true;
@@ -158,17 +124,9 @@ public class SourceRender extends Renderer {
return false;
}
private static void buildDimension(
String alias,
String dimension,
DataSource datasource,
SemanticSchema schema,
boolean nonAgg,
Map<String, String> extendFields,
TableView dataSet,
TableView output,
SqlValidatorScope scope)
throws Exception {
private static void buildDimension(String alias, String dimension, DataSource datasource,
SemanticSchema schema, boolean nonAgg, Map<String, String> extendFields,
TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception {
List<Dimension> dimensionList = schema.getDimension().get(datasource.getName());
EngineType engineType =
EngineType.fromString(schema.getSemanticModel().getDatabase().getType());
@@ -197,10 +155,8 @@ public class SourceRender extends Renderer {
}
}
if (!isAdd) {
Optional<Identify> identify =
datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(dimension))
.findFirst();
Optional<Identify> identify = datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(dimension)).findFirst();
if (identify.isPresent()) {
if (nonAgg) {
dataSet.getMeasure()
@@ -238,24 +194,17 @@ public class SourceRender extends Renderer {
if (dimension.getDataType().isArray()) {
if (Objects.nonNull(dimension.getExt())
&& dimension.getExt().containsKey(DIMENSION_DELIMITER)) {
extendFields.put(
dimension.getExpr(), (String) dimension.getExt().get(DIMENSION_DELIMITER));
extendFields.put(dimension.getExpr(),
(String) dimension.getExt().get(DIMENSION_DELIMITER));
} else {
extendFields.put(dimension.getExpr(), "");
}
}
}
private static List<SqlNode> getWhereMeasure(
List<String> fields,
List<String> queryMetrics,
List<String> queryDimensions,
Map<String, String> extendFields,
DataSource datasource,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
throws Exception {
private static List<SqlNode> getWhereMeasure(List<String> fields, List<String> queryMetrics,
List<String> queryDimensions, Map<String, String> extendFields, DataSource datasource,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception {
Iterator<String> iterator = fields.iterator();
List<SqlNode> whereNode = new ArrayList<>();
EngineType engineType =
@@ -295,40 +244,19 @@ public class SourceRender extends Renderer {
return whereNode;
}
private static void mergeWhere(
List<String> fields,
TableView dataSet,
TableView outputSet,
List<String> queryMetrics,
List<String> queryDimensions,
Map<String, String> extendFields,
DataSource datasource,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
throws Exception {
List<SqlNode> whereNode =
getWhereMeasure(
fields,
queryMetrics,
queryDimensions,
extendFields,
datasource,
scope,
schema,
nonAgg);
private static void mergeWhere(List<String> fields, TableView dataSet, TableView outputSet,
List<String> queryMetrics, List<String> queryDimensions,
Map<String, String> extendFields, DataSource datasource, SqlValidatorScope scope,
SemanticSchema schema, boolean nonAgg) throws Exception {
List<SqlNode> whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions,
extendFields, datasource, scope, schema, nonAgg);
dataSet.getMeasure().addAll(whereNode);
// getWhere(outputSet,fields,queryMetrics,queryDimensions,datasource,scope,schema);
}
public static void whereDimMetric(
List<String> fields,
List<String> queryMetrics,
List<String> queryDimensions,
DataSource datasource,
SemanticSchema schema,
Set<String> dimensions,
Set<String> metrics) {
public static void whereDimMetric(List<String> fields, List<String> queryMetrics,
List<String> queryDimensions, DataSource datasource, SemanticSchema schema,
Set<String> dimensions, Set<String> metrics) {
for (String field : fields) {
if (queryDimensions.contains(field) || queryMetrics.contains(field)) {
continue;
@@ -341,59 +269,40 @@ public class SourceRender extends Renderer {
}
}
private static void addField(
String field,
String oriField,
DataSource datasource,
SemanticSchema schema,
Set<String> dimensions,
Set<String> metrics) {
Optional<Dimension> dimension =
datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(field))
.findFirst();
private static void addField(String field, String oriField, DataSource datasource,
SemanticSchema schema, Set<String> dimensions, Set<String> metrics) {
Optional<Dimension> dimension = datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
if (dimension.isPresent()) {
dimensions.add(oriField);
return;
}
Optional<Identify> identify =
datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(field))
.findFirst();
Optional<Identify> identify = datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(field)).findFirst();
if (identify.isPresent()) {
dimensions.add(oriField);
return;
}
if (schema.getDimension().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim =
schema.getDimension().get(datasource.getName()).stream()
.filter(d -> d.getName().equalsIgnoreCase(field))
.findFirst();
Optional<Dimension> dataSourceDim = schema.getDimension().get(datasource.getName())
.stream().filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
if (dataSourceDim.isPresent()) {
dimensions.add(oriField);
return;
}
}
Optional<Measure> metric =
datasource.getMeasures().stream()
.filter(m -> m.getName().equalsIgnoreCase(field))
.findFirst();
Optional<Measure> metric = datasource.getMeasures().stream()
.filter(m -> m.getName().equalsIgnoreCase(field)).findFirst();
if (metric.isPresent()) {
metrics.add(oriField);
return;
}
Optional<Metric> datasourceMetric =
schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(field))
.findFirst();
Optional<Metric> datasourceMetric = schema.getMetrics().stream()
.filter(m -> m.getName().equalsIgnoreCase(field)).findFirst();
if (datasourceMetric.isPresent()) {
Set<String> measures =
datasourceMetric.get().getMetricTypeParams().getMeasures().stream()
.map(m -> m.getName())
.collect(Collectors.toSet());
if (datasource.getMeasures().stream()
.map(m -> m.getName())
.collect(Collectors.toSet())
Set<String> measures = datasourceMetric.get().getMetricTypeParams().getMeasures()
.stream().map(m -> m.getName()).collect(Collectors.toSet());
if (datasource.getMeasures().stream().map(m -> m.getName()).collect(Collectors.toSet())
.containsAll(measures)) {
metrics.add(oriField);
return;
@@ -402,25 +311,19 @@ public class SourceRender extends Renderer {
}
public static boolean isDimension(String name, DataSource datasource, SemanticSchema schema) {
Optional<Dimension> dimension =
datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(name))
.findFirst();
Optional<Dimension> dimension = datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(name)).findFirst();
if (dimension.isPresent()) {
return true;
}
Optional<Identify> identify =
datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(name))
.findFirst();
Optional<Identify> identify = datasource.getIdentifiers().stream()
.filter(i -> i.getName().equalsIgnoreCase(name)).findFirst();
if (identify.isPresent()) {
return true;
}
if (schema.getDimension().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim =
schema.getDimension().get(datasource.getName()).stream()
.filter(d -> d.getName().equalsIgnoreCase(name))
.findFirst();
Optional<Dimension> dataSourceDim = schema.getDimension().get(datasource.getName())
.stream().filter(d -> d.getName().equalsIgnoreCase(name)).findFirst();
if (dataSourceDim.isPresent()) {
return true;
}
@@ -430,30 +333,14 @@ public class SourceRender extends Renderer {
private static void addTimeDimension(DataSource dataSource, List<String> queryDimension) {
if (Materialization.TimePartType.ZIPPER.equals(dataSource.getTimePartType())) {
Optional<Dimension> startTimeOp =
dataSource.getDimensions().stream()
.filter(
d ->
Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(
d.getType()))
.filter(
d ->
d.getName()
.startsWith(
Constants.MATERIALIZATION_ZIPPER_START))
.findFirst();
Optional<Dimension> endTimeOp =
dataSource.getDimensions().stream()
.filter(
d ->
Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(
d.getType()))
.filter(
d ->
d.getName()
.startsWith(
Constants.MATERIALIZATION_ZIPPER_END))
.findFirst();
Optional<Dimension> startTimeOp = dataSource.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_START))
.findFirst();
Optional<Dimension> endTimeOp = dataSource.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.filter(d -> d.getName().startsWith(Constants.MATERIALIZATION_ZIPPER_END))
.findFirst();
if (startTimeOp.isPresent() && !queryDimension.contains(startTimeOp.get().getName())) {
queryDimension.add(startTimeOp.get().getName());
}
@@ -461,26 +348,17 @@ public class SourceRender extends Renderer {
queryDimension.add(endTimeOp.get().getName());
}
} else {
Optional<Dimension> timeOp =
dataSource.getDimensions().stream()
.filter(
d ->
Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(
d.getType()))
.findFirst();
Optional<Dimension> timeOp = dataSource.getDimensions().stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
if (timeOp.isPresent() && !queryDimension.contains(timeOp.get().getName())) {
queryDimension.add(timeOp.get().getName());
}
}
}
public void render(
MetricQueryParam metricQueryParam,
List<DataSource> dataSources,
SqlValidatorScope scope,
SemanticSchema schema,
boolean nonAgg)
throws Exception {
public void render(MetricQueryParam metricQueryParam, List<DataSource> dataSources,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricQueryParam.getWhere();
Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>();
@@ -493,17 +371,9 @@ public class SourceRender extends Renderer {
}
if (dataSources.size() == 1) {
DataSource dataSource = dataSources.get(0);
super.tableView =
renderOne(
"",
fieldWhere,
metricQueryParam.getMetrics(),
metricQueryParam.getDimensions(),
metricQueryParam.getWhere(),
dataSource,
scope,
schema,
nonAgg);
super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(),
metricQueryParam.getDimensions(), metricQueryParam.getWhere(), dataSource,
scope, schema, nonAgg);
return;
}
JoinRender joinRender = new JoinRender();

View File

@@ -33,9 +33,8 @@ public class CalculateAggConverter implements QueryConverter {
String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql);
}
public DataSetQueryParam generateSqlCommend(
QueryStatement queryStatement, EngineType engineTypeEnum, String version)
throws Exception {
public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement,
EngineType engineTypeEnum, String version) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
QueryParam queryParam = queryStatement.getQueryParam();
// 同环比
@@ -53,24 +52,16 @@ public class CalculateAggConverter implements QueryConverter {
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql =
String.format(
"select %s from %s %s %s %s",
sqlGenerateUtils.getSelect(queryParam),
metricTableName,
sqlGenerateUtils.getGroupBy(queryParam),
sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryParam));
String sql = String.format("select %s from %s %s %s %s",
sqlGenerateUtils.getSelect(queryParam), metricTableName,
sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryParam));
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false);
sql =
String.format(
"select %s from %s t0 %s %s %s",
sqlGenerateUtils.getSelect(queryParam),
metricTableName,
sqlGenerateUtils.getGroupBy(queryParam),
sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryParam));
sql = String.format("select %s from %s t0 %s %s %s",
sqlGenerateUtils.getSelect(queryParam), metricTableName,
sqlGenerateUtils.getGroupBy(queryParam),
sqlGenerateUtils.getOrderBy(queryParam), sqlGenerateUtils.getLimit(queryParam));
}
sqlCommand.setSql(sql);
return sqlCommand;
@@ -107,32 +98,25 @@ public class CalculateAggConverter implements QueryConverter {
@Override
public void convert(QueryStatement queryStatement) throws Exception {
Database database = queryStatement.getSemanticModel().getDatabase();
DataSetQueryParam dataSetQueryParam =
generateSqlCommend(
queryStatement,
EngineType.fromString(database.getType().toUpperCase()),
database.getVersion());
DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement,
EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
queryStatement.setDataSetQueryParam(dataSetQueryParam);
}
/** Ratio */
public boolean isRatioAccept(QueryParam queryParam) {
Long ratioFuncNum =
queryParam.getAggregators().stream()
.filter(
f ->
(f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)
|| f.getFunc().equals(AggOperatorEnum.RATIO_OVER)))
.count();
Long ratioFuncNum = queryParam.getAggregators().stream()
.filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)
|| f.getFunc().equals(AggOperatorEnum.RATIO_OVER)))
.count();
if (ratioFuncNum > 0) {
return true;
}
return false;
}
public DataSetQueryParam generateRatioSqlCommand(
QueryStatement queryStatement, EngineType engineTypeEnum, String version)
throws Exception {
public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement,
EngineType engineTypeEnum, String version) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
QueryParam queryParam = queryStatement.getQueryParam();
check(queryParam);
@@ -161,21 +145,11 @@ public class CalculateAggConverter implements QueryConverter {
sqlCommand.setSupportWith(false);
}
if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) {
sql =
new MysqlEngineSql()
.sql(
queryParam,
isOver,
sqlCommand.isSupportWith(),
metricTableName);
sql = new MysqlEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(),
metricTableName);
} else {
sql =
new CkEngineSql()
.sql(
queryParam,
isOver,
sqlCommand.isSupportWith(),
metricTableName);
sql = new CkEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(),
metricTableName);
}
break;
default:
@@ -187,27 +161,17 @@ public class CalculateAggConverter implements QueryConverter {
public class H2EngineSql implements EngineSql {
public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr =
queryParam.getAggregators().stream()
.map(
f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format(
"( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s",
f.getColumn(),
f.getColumn(),
f.getColumn(),
f.getColumn(),
f.getFunc().getOperator(),
f.getColumn());
} else {
return f.getColumn();
}
})
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups())
? aggStr
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getFunc().getOperator(), f.getColumn());
} else {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
}
@@ -227,48 +191,31 @@ public class CalculateAggConverter implements QueryConverter {
return "";
}
public String getJoinOn(
QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = getTimeSpan(queryParam, isOver, true);
String aggStr =
queryParam.getAggregators().stream()
.map(
f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam
.getDateInfo()
.getPeriod()
.equals(DatePeriodEnum.MONTH)) {
return String.format(
"%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ",
aliasRight + timeDim,
aliasLeft + timeDim,
timeSpan,
aliasRight + timeDim);
}
if (queryParam
.getDateInfo()
.getPeriod()
.equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format(
" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ",
getTimeSpan(queryParam, isOver, false),
aliasLeft + timeDim,
aliasRight + timeDim);
}
return String.format(
"%s = TIMESTAMPADD(%s,%s) ",
aliasLeft + timeDim,
timeSpan,
aliasRight + timeDim);
} else {
return f.getColumn();
}
})
.collect(Collectors.joining(" and "));
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ",
aliasRight + timeDim, aliasLeft + timeDim, timeSpan,
aliasRight + timeDim);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ",
getTimeSpan(queryParam, isOver, false), aliasLeft + timeDim,
aliasRight + timeDim);
}
return String.format("%s = TIMESTAMPADD(%s,%s) ", aliasLeft + timeDim, timeSpan,
aliasRight + timeDim);
} else {
return f.getColumn();
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
@@ -276,71 +223,48 @@ public class CalculateAggConverter implements QueryConverter {
}
groups.add(aliasLeft + group + " = " + aliasRight + group);
}
return CollectionUtils.isEmpty(groups)
? aggStr
return CollectionUtils.isEmpty(groups) ? aggStr
: String.join(" and ", groups) + " and " + aggStr + " ";
}
@Override
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
String sql =
String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver),
getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."),
metricSql,
metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryParam),
getLimit(queryParam));
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
return sql;
}
}
public class CkEngineSql extends MysqlEngineSql {
public String getJoinOn(
QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr =
queryParam.getAggregators().stream()
.map(
f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam
.getDateInfo()
.getPeriod()
.equals(DatePeriodEnum.MONTH)) {
return String.format(
"toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ",
aliasLeft + timeDim,
aliasRight + timeDim,
timeSpan);
}
if (queryParam
.getDateInfo()
.getPeriod()
.equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format(
"toMonday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim,
getTimeSpan(queryParam, isOver, false),
aliasRight + timeDim);
}
return String.format(
"%s = date_add(%s,%s) ",
aliasLeft + timeDim,
aliasRight + timeDim,
timeSpan);
} else {
return f.getColumn();
}
})
.collect(Collectors.joining(" and "));
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false),
aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim,
aliasRight + timeDim, timeSpan);
} else {
return f.getColumn();
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
@@ -348,8 +272,7 @@ public class CalculateAggConverter implements QueryConverter {
}
groups.add(aliasLeft + group + " = " + aliasRight + group);
}
return CollectionUtils.isEmpty(groups)
? aggStr
return CollectionUtils.isEmpty(groups) ? aggStr
: String.join(" and ", groups) + " and " + aggStr + " ";
}
@@ -358,25 +281,17 @@ public class CalculateAggConverter implements QueryConverter {
if (!asWith) {
return String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver),
getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."),
metricSql,
metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryParam),
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
}
return String.format(
",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s "
+ "from t0 left join t1 on %s ) metric_tb_src %s %s ",
metricSql,
metricSql,
getOverSelect(queryParam, isOver),
getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."),
getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryParam),
metricSql, metricSql, getOverSelect(queryParam, isOver),
getAllSelect(queryParam, "t0."), getAllJoinSelect(queryParam, "t1."),
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
}
}
@@ -400,72 +315,44 @@ public class CalculateAggConverter implements QueryConverter {
}
public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr =
queryParam.getAggregators().stream()
.map(
f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format(
"if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s",
f.getColumn(),
f.getColumn(),
f.getColumn(),
f.getColumn(),
f.getColumn(),
f.getFunc().getOperator(),
f.getColumn());
} else {
return f.getColumn();
}
})
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups())
? aggStr
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getColumn(), f.getFunc().getOperator(), f.getColumn());
} else {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
}
public String getJoinOn(
QueryParam queryParam, boolean isOver, String aliasLeft, String aliasRight) {
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr =
queryParam.getAggregators().stream()
.map(
f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam
.getDateInfo()
.getPeriod()
.equals(DatePeriodEnum.MONTH)) {
return String.format(
"%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ",
aliasLeft + timeDim,
aliasRight + timeDim,
timeSpan);
}
if (queryParam
.getDateInfo()
.getPeriod()
.equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format(
"to_monday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim,
getTimeSpan(queryParam, isOver, false),
aliasRight + timeDim);
}
return String.format(
"%s = date_add(%s,%s) ",
aliasLeft + timeDim,
aliasRight + timeDim,
timeSpan);
} else {
return f.getColumn();
}
})
.collect(Collectors.joining(" and "));
String aggStr = queryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false),
aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim,
aliasRight + timeDim, timeSpan);
} else {
return f.getColumn();
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
@@ -473,38 +360,26 @@ public class CalculateAggConverter implements QueryConverter {
}
groups.add(aliasLeft + group + " = " + aliasRight + group);
}
return CollectionUtils.isEmpty(groups)
? aggStr
return CollectionUtils.isEmpty(groups) ? aggStr
: String.join(" and ", groups) + " and " + aggStr + " ";
}
@Override
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
String sql =
String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver),
getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."),
metricSql,
metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."),
getOrderBy(queryParam),
getLimit(queryParam));
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
return sql;
}
}
private String getAllJoinSelect(QueryParam queryParam, String alias) {
String aggStr =
queryParam.getAggregators().stream()
.map(
f ->
getSelectField(f, alias)
+ " as "
+ getSelectField(f, "")
+ "_roll")
.collect(Collectors.joining(","));
String aggStr = queryParam.getAggregators().stream()
.map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "") + "_roll")
.collect(Collectors.joining(","));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
groups.add(alias + group + " as " + group + "_roll");
@@ -514,8 +389,7 @@ public class CalculateAggConverter implements QueryConverter {
private String getGroupDimWithOutTime(QueryParam queryParam) {
String timeDim = getTimeDim(queryParam);
return queryParam.getGroups().stream()
.filter(f -> !f.equalsIgnoreCase(timeDim))
return queryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(","));
}
@@ -532,12 +406,9 @@ public class CalculateAggConverter implements QueryConverter {
}
private String getAllSelect(QueryParam queryParam, String alias) {
String aggStr =
queryParam.getAggregators().stream()
.map(f -> getSelectField(f, alias))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups())
? aggStr
String aggStr = queryParam.getAggregators().stream().map(f -> getSelectField(f, alias))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: alias + String.join("," + alias, queryParam.getGroups()) + "," + aggStr;
}
@@ -562,22 +433,16 @@ public class CalculateAggConverter implements QueryConverter {
}
private boolean isOverRatio(QueryParam queryParam) {
Long overCt =
queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER))
.count();
Long overCt = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
return overCt > 0;
}
private void check(QueryParam queryParam) throws Exception {
Long ratioOverNum =
queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER))
.count();
Long ratioRollNum =
queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL))
.count();
Long ratioOverNum = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
Long ratioRollNum = queryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count();
if (ratioOverNum > 0 && ratioRollNum > 0) {
throw new Exception("not support over ratio and roll ratio together ");
}

View File

@@ -34,18 +34,16 @@ public class DefaultDimValueConverter implements QueryConverter {
@Override
public void convert(QueryStatement queryStatement) {
List<Dimension> dimensions =
queryStatement.getSemanticModel().getDimensions().stream()
.filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues()))
.collect(Collectors.toList());
List<Dimension> dimensions = queryStatement.getSemanticModel().getDimensions().stream()
.filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
String sql = queryStatement.getDataSetQueryParam().getSql();
List<String> whereFields =
SqlSelectHelper.getWhereFields(sql).stream()
.filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
List<String> whereFields = SqlSelectHelper.getWhereFields(sql).stream()
.filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) {
return;
}

View File

@@ -42,8 +42,8 @@ public class ParserDefaultConverter implements QueryConverter {
BeanUtils.copyProperties(metricReq, metricQueryParam);
}
public MetricQueryParam generateSqlCommand(
QueryParam queryParam, QueryStatement queryStatement) {
public MetricQueryParam generateSqlCommand(QueryParam queryParam,
QueryStatement queryStatement) {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
MetricQueryParam metricQueryParam = new MetricQueryParam();
metricQueryParam.setMetrics(queryParam.getMetrics());
@@ -52,10 +52,9 @@ public class ParserDefaultConverter implements QueryConverter {
log.info("in generateSqlCommend, complete where:{}", where);
metricQueryParam.setWhere(where);
metricQueryParam.setOrder(
queryParam.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection()))
.collect(Collectors.toList()));
metricQueryParam.setOrder(queryParam.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection()))
.collect(Collectors.toList()));
metricQueryParam.setLimit(queryParam.getLimit());
// support detail query

View File

@@ -33,19 +33,14 @@ public class SqlVariableParseConverter implements QueryConverter {
return;
}
for (ModelResp modelResp : modelResps) {
if (ModelDefineType.SQL_QUERY
.getName()
if (ModelDefineType.SQL_QUERY.getName()
.equalsIgnoreCase(modelResp.getModelDetail().getQueryType())) {
String sqlParsed =
SqlVariableParseUtils.parse(
modelResp.getModelDetail().getSqlQuery(),
SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(),
modelResp.getModelDetail().getSqlVariables(),
queryStatement.getQueryParam().getParams());
DataSource dataSource =
queryStatement
.getSemanticModel()
.getDatasourceMap()
.get(modelResp.getBizName());
DataSource dataSource = queryStatement.getSemanticModel().getDatasourceMap()
.get(modelResp.getBizName());
dataSource.setSqlQuery(sqlParsed);
}
}

View File

@@ -120,15 +120,13 @@ public class ComponentFactory {
}
private static <T> List<T> init(Class<T> factoryType, List list) {
list.addAll(
SpringFactoriesLoader.loadFactories(
factoryType, Thread.currentThread().getContextClassLoader()));
list.addAll(SpringFactoriesLoader.loadFactories(factoryType,
Thread.currentThread().getContextClassLoader()));
return list;
}
private static <T> T init(Class<T> factoryType) {
return SpringFactoriesLoader.loadFactories(
factoryType, Thread.currentThread().getContextClassLoader())
.get(0);
return SpringFactoriesLoader
.loadFactories(factoryType, Thread.currentThread().getContextClassLoader()).get(0);
}
}

View File

@@ -15,11 +15,8 @@ import java.util.stream.Collectors;
/** transform query results to return the users */
public class DataTransformUtils {
public static List<Map<String, Object>> transform(
List<Map<String, Object>> originalData,
String metric,
List<String> groups,
DateConf dateConf) {
public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData,
String metric, List<String> groups, DateConf dateConf) {
List<String> dateList = dateConf.getDateList();
List<Map<String, Object>> transposedData = new ArrayList<>();
for (Map<String, Object> originalRow : originalData) {
@@ -29,14 +26,12 @@ public class DataTransformUtils {
transposedRow.put(key, originalRow.get(key));
}
}
transposedRow.put(
String.valueOf(originalRow.get(getTimeDimension(dateConf))),
transposedRow.put(String.valueOf(originalRow.get(getTimeDimension(dateConf))),
originalRow.get(metric));
transposedData.add(transposedRow);
}
Map<String, List<Map<String, Object>>> dataMerge =
transposedData.stream()
.collect(Collectors.groupingBy(row -> getRowKey(row, groups)));
Map<String, List<Map<String, Object>>> dataMerge = transposedData.stream()
.collect(Collectors.groupingBy(row -> getRowKey(row, groups)));
List<Map<String, Object>> resultData = Lists.newArrayList();
for (List<Map<String, Object>> data : dataMerge.values()) {
Map<String, Object> rowData = new HashMap<>();

View File

@@ -31,7 +31,8 @@ import static com.tencent.supersonic.common.pojo.Constants.SPACE;
@Slf4j
public class JdbcDataSourceUtils {
@Getter private static Set releaseSourceSet = new HashSet();
@Getter
private static Set releaseSourceSet = new HashSet();
private JdbcDataSource jdbcDataSource;
public JdbcDataSourceUtils(JdbcDataSource jdbcDataSource) {
@@ -46,9 +47,8 @@ public class JdbcDataSourceUtils {
log.error(e.toString(), e);
return false;
}
try (Connection con =
DriverManager.getConnection(
database.getUrl(), database.getUsername(), database.passwordDecrypt())) {
try (Connection con = DriverManager.getConnection(database.getUrl(), database.getUsername(),
database.passwordDecrypt())) {
return con != null;
} catch (SQLException e) {
log.error(e.toString(), e);
@@ -116,8 +116,7 @@ public class JdbcDataSourceUtils {
log.error("e", e);
}
if (!StringUtils.isEmpty(className)
&& !className.contains("com.sun.proxy")
if (!StringUtils.isEmpty(className) && !className.contains("com.sun.proxy")
&& !className.contains("net.sf.cglib.proxy")) {
return className;
}
@@ -129,13 +128,8 @@ public class JdbcDataSourceUtils {
throw new RuntimeException("Not supported data type: jdbcUrl=" + jdbcUrl);
}
public static String getKey(
String name,
String jdbcUrl,
String username,
String password,
String version,
boolean isExt) {
public static String getKey(String name, String jdbcUrl, String username, String password,
String version, boolean isExt) {
StringBuilder sb = new StringBuilder();
@@ -165,10 +159,8 @@ public class JdbcDataSourceUtils {
return dataSource.getConnection();
} catch (Exception e) {
log.error("Get connection error, jdbcUrl:{}, e:{}", database.getUrl(), e);
throw new RuntimeException(
"Get connection error, jdbcUrl:"
+ database.getUrl()
+ " you can try again later or reset datasource");
throw new RuntimeException("Get connection error, jdbcUrl:" + database.getUrl()
+ " you can try again later or reset datasource");
}
}
return conn;
@@ -176,7 +168,7 @@ public class JdbcDataSourceUtils {
private Connection getConnectionWithRetry(Database database) {
int rc = 1;
for (; ; ) {
for (;;) {
if (rc > 3) {
return null;

View File

@@ -11,14 +11,8 @@ import java.util.stream.Collectors;
/** tools functions to duckDb query */
public class JdbcDuckDbUtils {
public static void attachMysql(
DuckDbSource duckDbSource,
String host,
Integer port,
String user,
String password,
String database)
throws Exception {
public static void attachMysql(DuckDbSource duckDbSource, String host, Integer port,
String user, String password, String database) throws Exception {
try {
duckDbSource.execute("INSTALL mysql");
duckDbSource.execute("load mysql");
@@ -40,25 +34,20 @@ public class JdbcDuckDbUtils {
if (!queryResultWithColumns.getResultList().isEmpty()) {
return queryResultWithColumns.getResultList().stream()
.filter(l -> l.containsKey("name") && Objects.nonNull(l.get("name")))
.map(l -> (String) l.get("name"))
.collect(Collectors.toList());
.map(l -> (String) l.get("name")).collect(Collectors.toList());
}
return new ArrayList<>();
}
public static List<String> getParquetPartition(
DuckDbSource duckDbSource, String parquetPath, String partitionName) throws Exception {
public static List<String> getParquetPartition(DuckDbSource duckDbSource, String parquetPath,
String partitionName) throws Exception {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
duckDbSource.query(
String.format(
"SELECT distinct %s as partition FROM read_parquet('%s')",
partitionName, parquetPath),
queryResultWithColumns);
duckDbSource.query(String.format("SELECT distinct %s as partition FROM read_parquet('%s')",
partitionName, parquetPath), queryResultWithColumns);
if (!queryResultWithColumns.getResultList().isEmpty()) {
return queryResultWithColumns.getResultList().stream()
.filter(l -> l.containsKey("partition") && Objects.nonNull(l.get("partition")))
.map(l -> (String) l.get("partition"))
.collect(Collectors.toList());
.map(l -> (String) l.get("partition")).collect(Collectors.toList());
}
return new ArrayList<>();
}

View File

@@ -19,24 +19,21 @@ public class SchemaMatchHelper {
}
Set<String> metricDimensionDetectWordSet =
matches.stream()
.filter(SchemaMatchHelper::isMetricOrDimension)
.map(SchemaElementMatch::getDetectWord)
.collect(Collectors.toSet());
matches.stream().filter(SchemaMatchHelper::isMetricOrDimension)
.map(SchemaElementMatch::getDetectWord).collect(Collectors.toSet());
matches.removeIf(
elementMatch -> {
if (!isMetricOrDimension(elementMatch)) {
return false;
}
for (String detectWord : metricDimensionDetectWordSet) {
if (detectWord.startsWith(elementMatch.getDetectWord())
&& detectWord.length() > elementMatch.getDetectWord().length()) {
return true;
}
}
return false;
});
matches.removeIf(elementMatch -> {
if (!isMetricOrDimension(elementMatch)) {
return false;
}
for (String detectWord : metricDimensionDetectWordSet) {
if (detectWord.startsWith(elementMatch.getDetectWord())
&& detectWord.length() > elementMatch.getDetectWord().length()) {
return true;
}
}
return false;
});
}
private static boolean isMetricOrDimension(SchemaElementMatch elementMatch) {

View File

@@ -54,9 +54,7 @@ public class SqlGenerateUtils {
private final ExecutorConfig executorConfig;
public SqlGenerateUtils(
SqlFilterUtils sqlFilterUtils,
DateModeUtils dateModeUtils,
public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils, DateModeUtils dateModeUtils,
ExecutorConfig executorConfig) {
this.sqlFilterUtils = sqlFilterUtils;
this.dateModeUtils = dateModeUtils;
@@ -95,22 +93,16 @@ public class SqlGenerateUtils {
}
public String getSelect(QueryParam queryParam) {
String aggStr =
queryParam.getAggregators().stream()
.map(this::getSelectField)
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups())
? aggStr
String aggStr = queryParam.getAggregators().stream().map(this::getSelectField)
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
}
public String getSelect(QueryParam queryParam, Map<String, String> deriveMetrics) {
String aggStr =
queryParam.getAggregators().stream()
.map(a -> getSelectField(a, deriveMetrics))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups())
? aggStr
String aggStr = queryParam.getAggregators().stream()
.map(a -> getSelectField(a, deriveMetrics)).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
}
@@ -121,20 +113,12 @@ public class SqlGenerateUtils {
if (CollectionUtils.isEmpty(agg.getArgs())) {
return agg.getFunc() + "( " + agg.getColumn() + " ) AS " + agg.getColumn() + " ";
}
return agg.getFunc()
+ "( "
return agg.getFunc() + "( "
+ agg.getArgs().stream()
.map(
arg ->
arg.equals(agg.getColumn())
? arg
: (StringUtils.isNumeric(arg)
? arg
: ("'" + arg + "'")))
.map(arg -> arg.equals(agg.getColumn()) ? arg
: (StringUtils.isNumeric(arg) ? arg : ("'" + arg + "'")))
.collect(Collectors.joining(","))
+ " ) AS "
+ agg.getColumn()
+ " ";
+ " ) AS " + agg.getColumn() + " ";
}
public String getSelectField(final Aggregator agg, Map<String, String> deriveMetrics) {
@@ -155,10 +139,9 @@ public class SqlGenerateUtils {
if (CollectionUtils.isEmpty(queryParam.getOrders())) {
return "";
}
return "order by "
+ queryParam.getOrders().stream()
.map(order -> " " + order.getColumn() + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
return "order by " + queryParam.getOrders().stream()
.map(order -> " " + order.getColumn() + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String getOrderBy(QueryParam queryParam, Map<String, String> deriveMetrics) {
@@ -169,18 +152,11 @@ public class SqlGenerateUtils {
.anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
return getOrderBy(queryParam);
}
return "order by "
+ queryParam.getOrders().stream()
.map(
order ->
" "
+ (deriveMetrics.containsKey(order.getColumn())
? deriveMetrics.get(order.getColumn())
: order.getColumn())
+ " "
+ order.getDirection()
+ " ")
.collect(Collectors.joining(","));
return "order by " + queryParam.getOrders().stream()
.map(order -> " " + (deriveMetrics.containsKey(order.getColumn())
? deriveMetrics.get(order.getColumn())
: order.getColumn()) + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String generateWhere(QueryParam queryParam, ItemDateResp itemDateResp) {
@@ -190,8 +166,8 @@ public class SqlGenerateUtils {
return mergeDateWhereClause(queryParam, whereClauseFromFilter, whereFromDate);
}
private String mergeDateWhereClause(
QueryParam queryParam, String whereClauseFromFilter, String whereFromDate) {
private String mergeDateWhereClause(QueryParam queryParam, String whereClauseFromFilter,
String whereFromDate) {
if (StringUtils.isNotEmpty(whereFromDate)
&& StringUtils.isNotEmpty(whereClauseFromFilter)) {
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
@@ -209,9 +185,8 @@ public class SqlGenerateUtils {
}
public String getDateWhereClause(DateConf dateInfo, ItemDateResp dateDate) {
if (Objects.isNull(dateDate)
|| StringUtils.isEmpty(dateDate.getStartDate())
&& StringUtils.isEmpty(dateDate.getEndDate())) {
if (Objects.isNull(dateDate) || StringUtils.isEmpty(dateDate.getStartDate())
&& StringUtils.isEmpty(dateDate.getEndDate())) {
if (dateInfo.getDateMode().equals(DateConf.DateMode.LIST)) {
return dateModeUtils.listDateStr(dateInfo);
}
@@ -228,8 +203,8 @@ public class SqlGenerateUtils {
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
}
public Triple<String, String, String> getBeginEndTime(
QueryParam queryParam, ItemDateResp dataDate) {
public Triple<String, String, String> getBeginEndTime(QueryParam queryParam,
ItemDateResp dataDate) {
if (Objects.isNull(queryParam.getDateInfo())) {
return Triple.of("", "", "");
}
@@ -243,16 +218,13 @@ public class SqlGenerateUtils {
case BETWEEN:
return Triple.of(dateInfo, dateConf.getStartDate(), dateConf.getEndDate());
case LIST:
return Triple.of(
dateInfo,
Collections.min(dateConf.getDateList()),
return Triple.of(dateInfo, Collections.min(dateConf.getDateList()),
Collections.max(dateConf.getDateList()));
case RECENT:
LocalDate dateMax = LocalDate.now().minusDays(1);
LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1);
if (Objects.isNull(dataDate)) {
return Triple.of(
dateInfo,
return Triple.of(dateInfo,
dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)),
dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT)));
}
@@ -270,11 +242,8 @@ public class SqlGenerateUtils {
dateModeUtils.recentMonth(dataDate, dateConf);
Optional<String> minBegins =
rets.stream().map(i -> i.left).sorted().findFirst();
Optional<String> maxBegins =
rets.stream()
.map(i -> i.right)
.sorted(Comparator.reverseOrder())
.findFirst();
Optional<String> maxBegins = rets.stream().map(i -> i.right)
.sorted(Comparator.reverseOrder()).findFirst();
if (minBegins.isPresent() && maxBegins.isPresent()) {
return Triple.of(dateInfo, minBegins.get(), maxBegins.get());
}
@@ -290,13 +259,11 @@ public class SqlGenerateUtils {
}
public boolean isSupportWith(EngineType engineTypeEnum, String version) {
if (engineTypeEnum.equals(EngineType.MYSQL)
&& Objects.nonNull(version)
if (engineTypeEnum.equals(EngineType.MYSQL) && Objects.nonNull(version)
&& version.startsWith(executorConfig.getMysqlLowVersion())) {
return false;
}
if (engineTypeEnum.equals(EngineType.CLICKHOUSE)
&& Objects.nonNull(version)
if (engineTypeEnum.equals(EngineType.CLICKHOUSE) && Objects.nonNull(version)
&& StringUtil.compareVersion(version, executorConfig.getCkLowVersion()) < 0) {
return false;
}
@@ -307,44 +274,28 @@ public class SqlGenerateUtils {
return modelBizName + UNDERLINE + executorConfig.getInternalMetricNameSuffix();
}
public String generateDerivedMetric(
final List<MetricSchemaResp> metricResps,
final Set<String> allFields,
final Map<String, Measure> allMeasures,
final List<DimSchemaResp> dimensionResps,
final String expression,
final MetricDefineType metricDefineType,
AggOption aggOption,
Set<String> visitedMetric,
Set<String> measures,
Set<String> dimensions) {
public String generateDerivedMetric(final List<MetricSchemaResp> metricResps,
final Set<String> allFields, final Map<String, Measure> allMeasures,
final List<DimSchemaResp> dimensionResps, final String expression,
final MetricDefineType metricDefineType, AggOption aggOption, Set<String> visitedMetric,
Set<String> measures, Set<String> dimensions) {
Set<String> fields = SqlSelectHelper.getColumnFromExpr(expression);
if (!CollectionUtils.isEmpty(fields)) {
Map<String, String> replace = new HashMap<>();
for (String field : fields) {
switch (metricDefineType) {
case METRIC:
Optional<MetricSchemaResp> metricItem =
metricResps.stream()
.filter(m -> m.getBizName().equalsIgnoreCase(field))
.findFirst();
Optional<MetricSchemaResp> metricItem = metricResps.stream()
.filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
if (metricItem.isPresent()) {
if (visitedMetric.contains(field)) {
break;
}
replace.put(
field,
generateDerivedMetric(
metricResps,
allFields,
allMeasures,
dimensionResps,
getExpr(metricItem.get()),
metricItem.get().getMetricDefineType(),
aggOption,
visitedMetric,
measures,
dimensions));
replace.put(field,
generateDerivedMetric(metricResps, allFields, allMeasures,
dimensionResps, getExpr(metricItem.get()),
metricItem.get().getMetricDefineType(), aggOption,
visitedMetric, measures, dimensions));
visitedMetric.add(field);
}
break;
@@ -356,10 +307,8 @@ public class SqlGenerateUtils {
break;
case FIELD:
if (allFields.contains(field)) {
Optional<DimSchemaResp> dimensionItem =
dimensionResps.stream()
.filter(d -> d.getBizName().equals(field))
.findFirst();
Optional<DimSchemaResp> dimensionItem = dimensionResps.stream()
.filter(d -> d.getBizName().equals(field)).findFirst();
if (dimensionItem.isPresent()) {
dimensions.add(field);
} else {
@@ -382,17 +331,11 @@ public class SqlGenerateUtils {
public String getExpr(Measure measure, AggOption aggOption) {
if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) {
return AggOption.NATIVE.equals(aggOption)
? measure.getBizName()
: AggOperatorEnum.COUNT.getOperator()
+ " ( "
+ AggOperatorEnum.DISTINCT
+ " "
+ measure.getBizName()
+ " ) ";
return AggOption.NATIVE.equals(aggOption) ? measure.getBizName()
: AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " "
+ measure.getBizName() + " ) ";
}
return AggOption.NATIVE.equals(aggOption)
? measure.getBizName()
return AggOption.NATIVE.equals(aggOption) ? measure.getBizName()
: measure.getAgg() + " ( " + measure.getBizName() + " ) ";
}

View File

@@ -36,9 +36,11 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
@Component
public class SqlUtils {
@Getter private Database database;
@Getter
private Database database;
@Autowired private JdbcDataSource jdbcDataSource;
@Autowired
private JdbcDataSource jdbcDataSource;
@Value("${s2.source.result-limit:1000000}")
private int resultLimit;
@@ -46,9 +48,11 @@ public class SqlUtils {
@Value("${s2.source.enable-query-log:false}")
private boolean isQueryLogEnable;
@Getter private DataType dataTypeEnum;
@Getter
private DataType dataTypeEnum;
@Getter private JdbcDataSourceUtils jdbcDataSourceUtils;
@Getter
private JdbcDataSourceUtils jdbcDataSourceUtils;
public SqlUtils() {}
@@ -60,14 +64,10 @@ public class SqlUtils {
public SqlUtils init(Database database) {
return SqlUtilsBuilder.getBuilder()
.withName(database.getId() + AT_SYMBOL + database.getName())
.withType(database.getType())
.withJdbcUrl(database.getUrl())
.withUsername(database.getUsername())
.withPassword(database.getPassword())
.withJdbcDataSource(this.jdbcDataSource)
.withResultLimit(this.resultLimit)
.withIsQueryLogEnable(this.isQueryLogEnable)
.build();
.withType(database.getType()).withJdbcUrl(database.getUrl())
.withUsername(database.getUsername()).withPassword(database.getPassword())
.withJdbcDataSource(this.jdbcDataSource).withResultLimit(this.resultLimit)
.withIsQueryLogEnable(this.isQueryLogEnable).build();
}
public List<Map<String, Object>> execute(String sql) throws ServerException {
@@ -105,27 +105,25 @@ public class SqlUtils {
getResult(sql, queryResultWithColumns, jdbcTemplate());
}
private SemanticQueryResp getResult(
String sql, SemanticQueryResp queryResultWithColumns, JdbcTemplate jdbcTemplate) {
jdbcTemplate.query(
sql,
rs -> {
if (null == rs) {
return queryResultWithColumns;
}
private SemanticQueryResp getResult(String sql, SemanticQueryResp queryResultWithColumns,
JdbcTemplate jdbcTemplate) {
jdbcTemplate.query(sql, rs -> {
if (null == rs) {
return queryResultWithColumns;
}
ResultSetMetaData metaData = rs.getMetaData();
List<QueryColumn> queryColumns = new ArrayList<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String key = metaData.getColumnLabel(i);
queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i)));
}
queryResultWithColumns.setColumns(queryColumns);
ResultSetMetaData metaData = rs.getMetaData();
List<QueryColumn> queryColumns = new ArrayList<>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String key = metaData.getColumnLabel(i);
queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i)));
}
queryResultWithColumns.setColumns(queryColumns);
List<Map<String, Object>> resultList = getAllData(rs, queryColumns);
queryResultWithColumns.setResultList(resultList);
return queryResultWithColumns;
});
List<Map<String, Object>> resultList = getAllData(rs, queryColumns);
queryResultWithColumns.setResultList(resultList);
return queryResultWithColumns;
});
return queryResultWithColumns;
}
@@ -226,14 +224,8 @@ public class SqlUtils {
}
public SqlUtils build() {
Database database =
Database.builder()
.name(this.name)
.type(this.type)
.url(this.jdbcUrl)
.username(this.username)
.password(this.password)
.build();
Database database = Database.builder().name(this.name).type(this.type).url(this.jdbcUrl)
.username(this.username).password(this.password).build();
SqlUtils sqlUtils = new SqlUtils(database);
sqlUtils.jdbcDataSource = this.jdbcDataSource;

View File

@@ -38,38 +38,32 @@ public class SqlVariableParseUtils {
return sql;
}
// 1. handle default variable value
sqlVariables.forEach(
variable -> {
variables.put(
variable.getName().trim(),
getValues(variable.getValueType(), variable.getDefaultValues()));
});
sqlVariables.forEach(variable -> {
variables.put(variable.getName().trim(),
getValues(variable.getValueType(), variable.getDefaultValues()));
});
// override by variable param
if (!CollectionUtils.isEmpty(params)) {
Map<String, List<SqlVariable>> map =
sqlVariables.stream().collect(Collectors.groupingBy(SqlVariable::getName));
params.forEach(
p -> {
if (map.containsKey(p.getName())) {
List<SqlVariable> list = map.get(p.getName());
if (!CollectionUtils.isEmpty(list)) {
SqlVariable v = list.get(list.size() - 1);
variables.put(
p.getName().trim(),
getValue(v.getValueType(), p.getValue()));
}
}
});
params.forEach(p -> {
if (map.containsKey(p.getName())) {
List<SqlVariable> list = map.get(p.getName());
if (!CollectionUtils.isEmpty(list)) {
SqlVariable v = list.get(list.size() - 1);
variables.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
}
}
});
}
variables.forEach(
(k, v) -> {
if (v instanceof List && ((List) v).size() > 0) {
v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString();
}
variables.put(k, v);
});
variables.forEach((k, v) -> {
if (v instanceof List && ((List) v).size() > 0) {
v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString();
}
variables.put(k, v);
});
return parse(sql, variables);
}
@@ -88,17 +82,12 @@ public class SqlVariableParseUtils {
if (null != valueType) {
switch (valueType) {
case STRING:
return values.stream()
.map(String::valueOf)
.map(
s ->
s.startsWith(APOSTROPHE) && s.endsWith(APOSTROPHE)
? s
: String.join(EMPTY, APOSTROPHE, s, APOSTROPHE))
return values.stream().map(String::valueOf)
.map(s -> s.startsWith(APOSTROPHE) && s.endsWith(APOSTROPHE) ? s
: String.join(EMPTY, APOSTROPHE, s, APOSTROPHE))
.collect(Collectors.toList());
case EXPR:
values.stream()
.map(String::valueOf)
values.stream().map(String::valueOf)
.forEach(SqlVariableParseUtils::checkSensitiveSql);
return values.stream().map(String::valueOf).collect(Collectors.toList());
case NUMBER:
@@ -115,11 +104,8 @@ public class SqlVariableParseUtils {
if (null != valueType) {
switch (valueType) {
case STRING:
return String.join(
EMPTY,
value.startsWith(APOSTROPHE) ? EMPTY : APOSTROPHE,
value,
value.endsWith(APOSTROPHE) ? EMPTY : APOSTROPHE);
return String.join(EMPTY, value.startsWith(APOSTROPHE) ? EMPTY : APOSTROPHE,
value, value.endsWith(APOSTROPHE) ? EMPTY : APOSTROPHE);
case NUMBER:
case EXPR:
default:

View File

@@ -17,8 +17,7 @@ public class SysTimeDimensionBuilder {
// Defines the regular expression pattern for the time keyword
private static final Pattern TIME_KEYWORD_PATTERN =
Pattern.compile(
"\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b",
Pattern.compile("\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b",
Pattern.CASE_INSENSITIVE);
public static void addSysTimeDimension(List<Dim> dims, DbAdaptor engineAdaptor) {
@@ -39,9 +38,8 @@ public class SysTimeDimensionBuilder {
Dim dim = new Dim();
dim.setBizName(TimeDimensionEnum.DAY.getName());
dim.setType(DimensionType.partition_time.name());
dim.setExpr(
generateTimeExpr(
timeDim, TimeDimensionEnum.DAY.name().toLowerCase(), engineAdaptor));
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.DAY.name().toLowerCase());
typeParams.setIsPrimary("true");
@@ -53,9 +51,8 @@ public class SysTimeDimensionBuilder {
Dim dim = new Dim();
dim.setBizName(TimeDimensionEnum.WEEK.getName());
dim.setType(DimensionType.partition_time.name());
dim.setExpr(
generateTimeExpr(
timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(), engineAdaptor));
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.WEEK.name().toLowerCase());
typeParams.setIsPrimary("false");
@@ -67,9 +64,8 @@ public class SysTimeDimensionBuilder {
Dim dim = new Dim();
dim.setBizName(TimeDimensionEnum.MONTH.getName());
dim.setType(DimensionType.partition_time.name());
dim.setExpr(
generateTimeExpr(
timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(), engineAdaptor));
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.MONTH.name().toLowerCase());
typeParams.setIsPrimary("false");

View File

@@ -11,462 +11,316 @@ public class CalciteSqlParserTest {
@Test
public void testCalciteSqlParser() throws Exception {
String json =
"{\n"
+ " \"dataSetId\": 1,\n"
+ " \"sql\": \"\",\n"
+ " \"sourceId\": \"\",\n"
+ " \"errMsg\": \"\",\n"
+ " \"metricQueryParam\": {\n"
+ " \"metrics\": [\n"
+ " \"pv\"\n"
+ " ],\n"
+ " \"dimensions\": [\n"
+ " \"sys_imp_date\"\n"
+ " ],\n"
+ " \"nativeQuery\": false\n"
+ " },\n"
+ " \"status\": 0,\n"
+ " \"isS2SQL\": false,\n"
+ " \"enableOptimize\": true,\n"
+ " \"minMaxTime\": {\n"
+ " \"left\": \"sys_imp_date\",\n"
+ " \"middle\": \"2024-03-24\",\n"
+ " \"right\": \"2024-03-18\"\n"
+ " },\n"
+ " \"dataSetSql\": \"SELECT sys_imp_date, SUM(pv) AS pv FROM t_1 WHERE "
+ "sys_imp_date >= '2024-03-18' AND sys_imp_date <= '2024-03-24' GROUP BY sys_imp_date LIMIT 365\",\n"
+ " \"dataSetAlias\": \"t_1\",\n"
+ " \"dataSetSimplifySql\": \"\",\n"
+ " \"enableLimitWrapper\": false,\n"
+ " \"semanticModel\": {\n"
+ " \"schemaKey\": \"VIEW_1\",\n"
+ " \"metrics\": [\n"
+ " {\n"
+ " \"name\": \"pv\",\n"
+ " \"owners\": [\n"
+ " \"admin\"\n"
+ " ],\n"
+ " \"type\": \"ATOMIC\",\n"
+ " \"metricTypeParams\": {\n"
+ " \"measures\": [\n"
+ " {\n"
+ " \"name\": \"s2_pv_uv_statis_pv\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"constraint\": \"\"\n"
+ " }\n"
+ " ],\n"
+ " \"isFieldMetric\": false,\n"
+ " \"expr\": \"s2_pv_uv_statis_pv\"\n"
+ " }\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"uv\",\n"
+ " \"owners\": [\n"
+ " \"admin\"\n"
+ " ],\n"
+ " \"type\": \"DERIVED\",\n"
+ " \"metricTypeParams\": {\n"
+ " \"measures\": [\n"
+ " {\n"
+ " \"name\": \"user_id\",\n"
+ " \"expr\": \"user_id\"\n"
+ " }\n"
+ " ],\n"
+ " \"isFieldMetric\": true,\n"
+ " \"expr\": \"user_id\"\n"
+ " }\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"pv_avg\",\n"
+ " \"owners\": [\n"
+ " \"admin\"\n"
+ " ],\n"
+ " \"type\": \"DERIVED\",\n"
+ " \"metricTypeParams\": {\n"
+ " \"measures\": [\n"
+ " {\n"
+ " \"name\": \"pv\",\n"
+ " \"expr\": \"pv\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"uv\",\n"
+ " \"expr\": \"uv\"\n"
+ " }\n"
+ " ],\n"
+ " \"isFieldMetric\": true,\n"
+ " \"expr\": \"pv\"\n"
+ " }\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"stay_hours\",\n"
+ " \"owners\": [\n"
+ " \"admin\"\n"
+ " ],\n"
+ " \"type\": \"ATOMIC\",\n"
+ " \"metricTypeParams\": {\n"
+ " \"measures\": [\n"
+ " {\n"
+ " \"name\": \"s2_stay_time_statis_stay_hours\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"constraint\": \"\"\n"
+ " }\n"
+ " ],\n"
+ " \"isFieldMetric\": false,\n"
+ " \"expr\": \"s2_stay_time_statis_stay_hours\"\n"
+ " }\n"
+ " }\n"
+ " ],\n"
+ " \"datasourceMap\": {\n"
+ " \"user_department\": {\n"
+ " \"id\": 1,\n"
+ " \"name\": \"user_department\",\n"
+ " \"sourceId\": 1,\n"
+ " \"type\": \"h2\",\n"
+ " \"sqlQuery\": \"select user_name,department from s2_user_department\",\n"
+ " \"identifiers\": [\n"
+ " {\n"
+ " \"name\": \"user_name\",\n"
+ " \"type\": \"primary\"\n"
+ " }\n"
+ " ],\n"
+ " \"dimensions\": [\n"
+ " {\n"
+ " \"name\": \"department\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"department\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"department\"\n"
+ " }\n"
+ " ],\n"
+ " \"measures\": [\n"
+ " {\n"
+ " \"name\": \"user_department_internal_cnt\",\n"
+ " \"agg\": \"count\",\n"
+ " \"expr\": \"user_name\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"user_name\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"user_name\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"department\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"department\"\n"
+ " }\n"
+ " ],\n"
+ " \"aggTime\": \"none\"\n"
+ " },\n"
+ " \"s2_pv_uv_statis\": {\n"
+ " \"id\": 2,\n"
+ " \"name\": \"s2_pv_uv_statis\",\n"
+ " \"sourceId\": 1,\n"
+ " \"type\": \"h2\",\n"
+ " \"sqlQuery\": \"SELECT imp_date, user_name, page, 1 as pv, user_name as user_id "
+ "FROM s2_pv_uv_statis\",\n"
+ " \"identifiers\": [\n"
+ " {\n"
+ " \"name\": \"user_name\",\n"
+ " \"type\": \"primary\"\n"
+ " }\n"
+ " ],\n"
+ " \"dimensions\": [\n"
+ " {\n"
+ " \"name\": \"imp_date\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"imp_date\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"true\",\n"
+ " \"timeGranularity\": \"day\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"imp_date\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"page\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"page\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"page\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"sys_imp_date\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"imp_date\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"true\",\n"
+ " \"timeGranularity\": \"day\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_date\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"sys_imp_week\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"DATE_TRUNC('week',imp_date)\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"false\",\n"
+ " \"timeGranularity\": \"week\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_week\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"sys_imp_month\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"FORMATDATETIME(PARSEDATETIME"
+ "(imp_date, 'yyyy-MM-dd'),'yyyy-MM') \",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"false\",\n"
+ " \"timeGranularity\": \"month\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_month\"\n"
+ " }\n"
+ " ],\n"
+ " \"measures\": [\n"
+ " {\n"
+ " \"name\": \"s2_pv_uv_statis_pv\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"expr\": \"pv\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"s2_pv_uv_statis_user_id\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"expr\": \"user_id\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"s2_pv_uv_statis_internal_cnt\",\n"
+ " \"agg\": \"count\",\n"
+ " \"expr\": \"user_name\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"user_name\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"user_name\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"imp_date\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"imp_date\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"page\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"page\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"pv\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"pv\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"user_id\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"user_id\"\n"
+ " }\n"
+ " ],\n"
+ " \"aggTime\": \"day\"\n"
+ " },\n"
+ " \"s2_stay_time_statis\": {\n"
+ " \"id\": 3,\n"
+ " \"name\": \"s2_stay_time_statis\",\n"
+ " \"sourceId\": 1,\n"
+ " \"type\": \"h2\",\n"
+ " \"sqlQuery\": \"select imp_date,user_name,stay_hours"
+ ",page from s2_stay_time_statis\",\n"
+ " \"identifiers\": [\n"
+ " {\n"
+ " \"name\": \"user_name\",\n"
+ " \"type\": \"primary\"\n"
+ " }\n"
+ " ],\n"
+ " \"dimensions\": [\n"
+ " {\n"
+ " \"name\": \"imp_date\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"imp_date\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"true\",\n"
+ " \"timeGranularity\": \"day\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"imp_date\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"page\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"page\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"page\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"sys_imp_date\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"imp_date\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"true\",\n"
+ " \"timeGranularity\": \"day\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_date\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"sys_imp_week\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"DATE_TRUNC('week',imp_date)\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"false\",\n"
+ " \"timeGranularity\": \"week\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_week\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"sys_imp_month\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"FORMATDATETIME(PARSEDATETIME"
+ "(imp_date, 'yyyy-MM-dd'),'yyyy-MM') \",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"false\",\n"
+ " \"timeGranularity\": \"month\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_month\"\n"
+ " }\n"
+ " ],\n"
+ " \"measures\": [\n"
+ " {\n"
+ " \"name\": \"s2_stay_time_statis_stay_hours\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"expr\": \"stay_hours\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"s2_stay_time_statis_internal_cnt\",\n"
+ " \"agg\": \"count\",\n"
+ " \"expr\": \"user_name\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"user_name\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"user_name\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"imp_date\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"imp_date\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"page\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"page\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"stay_hours\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"stay_hours\"\n"
+ " }\n"
+ " ],\n"
+ " \"aggTime\": \"day\"\n"
+ " }\n"
+ " },\n"
+ " \"dimensionMap\": {\n"
+ " \"user_department\": [\n"
+ " {\n"
+ " \"name\": \"department\",\n"
+ " \"owners\": \"admin\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"department\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"department\"\n"
+ " }\n"
+ " ],\n"
+ " \"s2_pv_uv_statis\": [\n"
+ " ],\n"
+ " \"s2_stay_time_statis\": [\n"
+ " {\n"
+ " \"name\": \"page\",\n"
+ " \"owners\": \"admin\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"page\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"page\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"materializationList\": [\n"
+ " ],\n"
+ " \"joinRelations\": [\n"
+ " {\n"
+ " \"id\": 1,\n"
+ " \"left\": \"user_department\",\n"
+ " \"right\": \"s2_pv_uv_statis\",\n"
+ " \"joinType\": \"left join\",\n"
+ " \"joinCondition\": [\n"
+ " {\n"
+ " \"left\": \"user_name\",\n"
+ " \"middle\": \"=\",\n"
+ " \"right\": \"user_name\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " {\n"
+ " \"id\": 2,\n"
+ " \"left\": \"user_department\",\n"
+ " \"right\": \"s2_stay_time_statis\",\n"
+ " \"joinType\": \"left join\",\n"
+ " \"joinCondition\": [\n"
+ " {\n"
+ " \"left\": \"user_name\",\n"
+ " \"middle\": \"=\",\n"
+ " \"right\": \"user_name\"\n"
+ " }\n"
+ " ]\n"
+ " }\n"
+ " ],\n"
+ " \"database\": {\n"
+ " \"id\": 1,\n"
+ " \"name\": \"数据实例\",\n"
+ " \"description\": \"样例数据库实例\",\n"
+ " \"url\": \"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false\",\n"
+ " \"username\": \"root\",\n"
+ " \"password\": \"semantic\",\n"
+ " \"type\": \"h2\",\n"
+ " \"connectInfo\": {\n"
+ " \"url\": \"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false\",\n"
+ " \"userName\": \"root\",\n"
+ " \"password\": \"semantic\"\n"
+ " },\n"
+ " \"admins\": [\n"
+ " ],\n"
+ " \"viewers\": [\n"
+ " ],\n"
+ " \"createdBy\": \"admin\",\n"
+ " \"updatedBy\": \"admin\",\n"
+ " \"createdAt\": 1711367511146,\n"
+ " \"updatedAt\": 1711367511146\n"
+ " }\n"
+ " }\n"
+ "}";
String json = "{\n" + " \"dataSetId\": 1,\n" + " \"sql\": \"\",\n"
+ " \"sourceId\": \"\",\n" + " \"errMsg\": \"\",\n"
+ " \"metricQueryParam\": {\n" + " \"metrics\": [\n"
+ " \"pv\"\n" + " ],\n" + " \"dimensions\": [\n"
+ " \"sys_imp_date\"\n" + " ],\n"
+ " \"nativeQuery\": false\n" + " },\n" + " \"status\": 0,\n"
+ " \"isS2SQL\": false,\n" + " \"enableOptimize\": true,\n"
+ " \"minMaxTime\": {\n" + " \"left\": \"sys_imp_date\",\n"
+ " \"middle\": \"2024-03-24\",\n" + " \"right\": \"2024-03-18\"\n"
+ " },\n"
+ " \"dataSetSql\": \"SELECT sys_imp_date, SUM(pv) AS pv FROM t_1 WHERE "
+ "sys_imp_date >= '2024-03-18' AND sys_imp_date <= '2024-03-24' GROUP BY sys_imp_date LIMIT 365\",\n"
+ " \"dataSetAlias\": \"t_1\",\n" + " \"dataSetSimplifySql\": \"\",\n"
+ " \"enableLimitWrapper\": false,\n" + " \"semanticModel\": {\n"
+ " \"schemaKey\": \"VIEW_1\",\n" + " \"metrics\": [\n"
+ " {\n" + " \"name\": \"pv\",\n"
+ " \"owners\": [\n" + " \"admin\"\n"
+ " ],\n" + " \"type\": \"ATOMIC\",\n"
+ " \"metricTypeParams\": {\n"
+ " \"measures\": [\n" + " {\n"
+ " \"name\": \"s2_pv_uv_statis_pv\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"constraint\": \"\"\n"
+ " }\n" + " ],\n"
+ " \"isFieldMetric\": false,\n"
+ " \"expr\": \"s2_pv_uv_statis_pv\"\n" + " }\n"
+ " },\n" + " {\n" + " \"name\": \"uv\",\n"
+ " \"owners\": [\n" + " \"admin\"\n"
+ " ],\n" + " \"type\": \"DERIVED\",\n"
+ " \"metricTypeParams\": {\n"
+ " \"measures\": [\n" + " {\n"
+ " \"name\": \"user_id\",\n"
+ " \"expr\": \"user_id\"\n"
+ " }\n" + " ],\n"
+ " \"isFieldMetric\": true,\n"
+ " \"expr\": \"user_id\"\n" + " }\n"
+ " },\n" + " {\n" + " \"name\": \"pv_avg\",\n"
+ " \"owners\": [\n" + " \"admin\"\n"
+ " ],\n" + " \"type\": \"DERIVED\",\n"
+ " \"metricTypeParams\": {\n"
+ " \"measures\": [\n" + " {\n"
+ " \"name\": \"pv\",\n"
+ " \"expr\": \"pv\"\n" + " },\n"
+ " {\n" + " \"name\": \"uv\",\n"
+ " \"expr\": \"uv\"\n" + " }\n"
+ " ],\n" + " \"isFieldMetric\": true,\n"
+ " \"expr\": \"pv\"\n" + " }\n"
+ " },\n" + " {\n"
+ " \"name\": \"stay_hours\",\n" + " \"owners\": [\n"
+ " \"admin\"\n" + " ],\n"
+ " \"type\": \"ATOMIC\",\n"
+ " \"metricTypeParams\": {\n"
+ " \"measures\": [\n" + " {\n"
+ " \"name\": \"s2_stay_time_statis_stay_hours\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"constraint\": \"\"\n"
+ " }\n" + " ],\n"
+ " \"isFieldMetric\": false,\n"
+ " \"expr\": \"s2_stay_time_statis_stay_hours\"\n"
+ " }\n" + " }\n" + " ],\n"
+ " \"datasourceMap\": {\n" + " \"user_department\": {\n"
+ " \"id\": 1,\n"
+ " \"name\": \"user_department\",\n"
+ " \"sourceId\": 1,\n" + " \"type\": \"h2\",\n"
+ " \"sqlQuery\": \"select user_name,department from s2_user_department\",\n"
+ " \"identifiers\": [\n" + " {\n"
+ " \"name\": \"user_name\",\n"
+ " \"type\": \"primary\"\n" + " }\n"
+ " ],\n" + " \"dimensions\": [\n"
+ " {\n" + " \"name\": \"department\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"department\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"department\"\n"
+ " }\n" + " ],\n"
+ " \"measures\": [\n" + " {\n"
+ " \"name\": \"user_department_internal_cnt\",\n"
+ " \"agg\": \"count\",\n"
+ " \"expr\": \"user_name\"\n" + " },\n"
+ " {\n" + " \"name\": \"user_name\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"user_name\"\n" + " },\n"
+ " {\n" + " \"name\": \"department\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"department\"\n" + " }\n"
+ " ],\n" + " \"aggTime\": \"none\"\n"
+ " },\n" + " \"s2_pv_uv_statis\": {\n"
+ " \"id\": 2,\n"
+ " \"name\": \"s2_pv_uv_statis\",\n"
+ " \"sourceId\": 1,\n" + " \"type\": \"h2\",\n"
+ " \"sqlQuery\": \"SELECT imp_date, user_name, page, 1 as pv, user_name as user_id "
+ "FROM s2_pv_uv_statis\",\n" + " \"identifiers\": [\n"
+ " {\n" + " \"name\": \"user_name\",\n"
+ " \"type\": \"primary\"\n" + " }\n"
+ " ],\n" + " \"dimensions\": [\n"
+ " {\n" + " \"name\": \"imp_date\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"imp_date\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"true\",\n"
+ " \"timeGranularity\": \"day\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"imp_date\"\n" + " },\n"
+ " {\n" + " \"name\": \"page\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"page\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"page\"\n" + " },\n"
+ " {\n"
+ " \"name\": \"sys_imp_date\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"imp_date\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"true\",\n"
+ " \"timeGranularity\": \"day\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_date\"\n"
+ " },\n" + " {\n"
+ " \"name\": \"sys_imp_week\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"DATE_TRUNC('week',imp_date)\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"false\",\n"
+ " \"timeGranularity\": \"week\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_week\"\n"
+ " },\n" + " {\n"
+ " \"name\": \"sys_imp_month\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"FORMATDATETIME(PARSEDATETIME"
+ "(imp_date, 'yyyy-MM-dd'),'yyyy-MM') \",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"false\",\n"
+ " \"timeGranularity\": \"month\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_month\"\n"
+ " }\n" + " ],\n"
+ " \"measures\": [\n" + " {\n"
+ " \"name\": \"s2_pv_uv_statis_pv\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"expr\": \"pv\"\n" + " },\n"
+ " {\n"
+ " \"name\": \"s2_pv_uv_statis_user_id\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"expr\": \"user_id\"\n" + " },\n"
+ " {\n"
+ " \"name\": \"s2_pv_uv_statis_internal_cnt\",\n"
+ " \"agg\": \"count\",\n"
+ " \"expr\": \"user_name\"\n" + " },\n"
+ " {\n" + " \"name\": \"user_name\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"user_name\"\n" + " },\n"
+ " {\n" + " \"name\": \"imp_date\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"imp_date\"\n" + " },\n"
+ " {\n" + " \"name\": \"page\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"page\"\n" + " },\n"
+ " {\n" + " \"name\": \"pv\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"pv\"\n" + " },\n"
+ " {\n" + " \"name\": \"user_id\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"user_id\"\n" + " }\n"
+ " ],\n" + " \"aggTime\": \"day\"\n"
+ " },\n" + " \"s2_stay_time_statis\": {\n"
+ " \"id\": 3,\n"
+ " \"name\": \"s2_stay_time_statis\",\n"
+ " \"sourceId\": 1,\n" + " \"type\": \"h2\",\n"
+ " \"sqlQuery\": \"select imp_date,user_name,stay_hours"
+ ",page from s2_stay_time_statis\",\n" + " \"identifiers\": [\n"
+ " {\n" + " \"name\": \"user_name\",\n"
+ " \"type\": \"primary\"\n" + " }\n"
+ " ],\n" + " \"dimensions\": [\n"
+ " {\n" + " \"name\": \"imp_date\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"imp_date\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"true\",\n"
+ " \"timeGranularity\": \"day\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"imp_date\"\n" + " },\n"
+ " {\n" + " \"name\": \"page\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"page\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"page\"\n" + " },\n"
+ " {\n"
+ " \"name\": \"sys_imp_date\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"imp_date\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"true\",\n"
+ " \"timeGranularity\": \"day\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_date\"\n"
+ " },\n" + " {\n"
+ " \"name\": \"sys_imp_week\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"DATE_TRUNC('week',imp_date)\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"false\",\n"
+ " \"timeGranularity\": \"week\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_week\"\n"
+ " },\n" + " {\n"
+ " \"name\": \"sys_imp_month\",\n"
+ " \"type\": \"time\",\n"
+ " \"expr\": \"FORMATDATETIME(PARSEDATETIME"
+ "(imp_date, 'yyyy-MM-dd'),'yyyy-MM') \",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " \"isPrimary\": \"false\",\n"
+ " \"timeGranularity\": \"month\"\n"
+ " },\n"
+ " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"sys_imp_month\"\n"
+ " }\n" + " ],\n"
+ " \"measures\": [\n" + " {\n"
+ " \"name\": \"s2_stay_time_statis_stay_hours\",\n"
+ " \"agg\": \"SUM\",\n"
+ " \"expr\": \"stay_hours\"\n" + " },\n"
+ " {\n"
+ " \"name\": \"s2_stay_time_statis_internal_cnt\",\n"
+ " \"agg\": \"count\",\n"
+ " \"expr\": \"user_name\"\n" + " },\n"
+ " {\n" + " \"name\": \"user_name\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"user_name\"\n" + " },\n"
+ " {\n" + " \"name\": \"imp_date\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"imp_date\"\n" + " },\n"
+ " {\n" + " \"name\": \"page\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"page\"\n" + " },\n"
+ " {\n" + " \"name\": \"stay_hours\",\n"
+ " \"agg\": \"\",\n"
+ " \"expr\": \"stay_hours\"\n" + " }\n"
+ " ],\n" + " \"aggTime\": \"day\"\n"
+ " }\n" + " },\n" + " \"dimensionMap\": {\n"
+ " \"user_department\": [\n" + " {\n"
+ " \"name\": \"department\",\n"
+ " \"owners\": \"admin\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"department\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n" + " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"department\"\n" + " }\n"
+ " ],\n" + " \"s2_pv_uv_statis\": [\n" + " ],\n"
+ " \"s2_stay_time_statis\": [\n" + " {\n"
+ " \"name\": \"page\",\n"
+ " \"owners\": \"admin\",\n"
+ " \"type\": \"categorical\",\n"
+ " \"expr\": \"page\",\n"
+ " \"dimensionTimeTypeParams\": {\n"
+ " },\n" + " \"dataType\": \"UNKNOWN\",\n"
+ " \"bizName\": \"page\"\n" + " }\n"
+ " ]\n" + " },\n" + " \"materializationList\": [\n"
+ " ],\n" + " \"joinRelations\": [\n" + " {\n"
+ " \"id\": 1,\n"
+ " \"left\": \"user_department\",\n"
+ " \"right\": \"s2_pv_uv_statis\",\n"
+ " \"joinType\": \"left join\",\n"
+ " \"joinCondition\": [\n" + " {\n"
+ " \"left\": \"user_name\",\n"
+ " \"middle\": \"=\",\n"
+ " \"right\": \"user_name\"\n" + " }\n"
+ " ]\n" + " },\n" + " {\n"
+ " \"id\": 2,\n"
+ " \"left\": \"user_department\",\n"
+ " \"right\": \"s2_stay_time_statis\",\n"
+ " \"joinType\": \"left join\",\n"
+ " \"joinCondition\": [\n" + " {\n"
+ " \"left\": \"user_name\",\n"
+ " \"middle\": \"=\",\n"
+ " \"right\": \"user_name\"\n" + " }\n"
+ " ]\n" + " }\n" + " ],\n"
+ " \"database\": {\n" + " \"id\": 1,\n"
+ " \"name\": \"数据实例\",\n"
+ " \"description\": \"样例数据库实例\",\n"
+ " \"url\": \"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false\",\n"
+ " \"username\": \"root\",\n"
+ " \"password\": \"semantic\",\n" + " \"type\": \"h2\",\n"
+ " \"connectInfo\": {\n"
+ " \"url\": \"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false\",\n"
+ " \"userName\": \"root\",\n"
+ " \"password\": \"semantic\"\n" + " },\n"
+ " \"admins\": [\n" + " ],\n"
+ " \"viewers\": [\n" + " ],\n"
+ " \"createdBy\": \"admin\",\n"
+ " \"updatedBy\": \"admin\",\n"
+ " \"createdAt\": 1711367511146,\n"
+ " \"updatedAt\": 1711367511146\n" + " }\n" + " }\n" + "}";
QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class);
CalciteQueryParser calciteSqlParser = new CalciteQueryParser();
calciteSqlParser.parse(queryStatement, AggOption.DEFAULT);
Assert.assertEquals(
queryStatement.getSql().trim().replaceAll("\\s+", ""),
"SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`"
+ "FROM"
+ "`s2_pv_uv_statis`"
Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""),
"SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" + "FROM" + "`s2_pv_uv_statis`"
+ "GROUPBY`imp_date`,`imp_date`");
}
}