topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 02:46:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless] In the translator, adopt Calcite to support multiple WITH structures and optimize the Calcite code. (#1747)

Browse Source
This commit is contained in:
lexluo09
2024-09-30 18:53:27 +08:00
committed by GitHub
parent de2385cd2c
commit 8693e7f8a7
39 changed files with 234 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java → common/src/main/java/com/tencent/supersonic/common/calcite/Configuration.java
View File

@@ -1,10 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite;
package com.tencent.supersonic.common.calcite;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSqlTypeFactoryImpl;
import com.tencent.supersonic.headless.core.translator.calcite.schema.ViewExpanderImpl;
import com.tencent.supersonic.headless.core.utils.SqlDialectFactory;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import org.apache.calcite.avatica.util.Casing;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.calcite.config.CalciteConnectionConfig;
@@ -22,11 +18,13 @@ import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.sql.SqlOperatorTable;
import org.apache.calcite.sql.SqlWriterConfig;
import org.apache.calcite.sql.advise.SqlAdvisor;
import org.apache.calcite.sql.advise.SqlAdvisorValidator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.impl.SqlParserImpl;
import org.apache.calcite.sql.pretty.SqlPrettyWriter;
import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
@@ -158,4 +156,22 @@ public class Configuration {
public static SqlAdvisor getSqlAdvisor(SqlValidatorWithHints validator, EngineType engineType) {
return new SqlAdvisor(validator, getParserConfig(engineType));
}
public static SqlWriterConfig getSqlWriterConfig(EngineType engineType) {
SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType);
SqlWriterConfig config =
SqlPrettyWriter.config()
.withDialect(sqlDialect)
.withKeywordsLowerCase(false)
.withClauseEndsLine(true)
.withAlwaysUseParentheses(false)
.withSelectListItemsOnSeparateLines(false)
.withUpdateSetListNewline(false)
.withIndentation(0);
if (EngineType.MYSQL.equals(engineType)) {
// no backticks around function name
config = config.withQuoteAllIdentifiers(false);
}
return config;
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlConformance.java → common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlConformance.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.calcite.schema;
package com.tencent.supersonic.common.calcite;
import org.apache.calcite.sql.fun.SqlLibrary;
import org.apache.calcite.sql.validate.SqlConformance;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlDialect.java → common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlDialect.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.calcite.schema;
package com.tencent.supersonic.common.calcite;
import com.google.common.base.Preconditions;
import org.apache.calcite.sql.SqlDialect;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SemanticSqlTypeFactoryImpl.java → common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlTypeFactoryImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.calcite.schema;
package com.tencent.supersonic.common.calcite;
import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.calcite.sql.type.SqlTypeFactoryImpl;

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlDialectFactory.java → common/src/main/java/com/tencent/supersonic/common/calcite/SqlDialectFactory.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.core.utils;
package com.tencent.supersonic.common.calcite;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import org.apache.calcite.avatica.util.Casing;
import org.apache.calcite.sql.SqlDialect;
import org.apache.calcite.sql.SqlDialect.Context;

80
common/src/main/java/com/tencent/supersonic/common/calcite/SqlMergeWithUtils.java Normal file
View File

@@ -0,0 +1,80 @@
package com.tencent.supersonic.common.calcite;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlWith;
import org.apache.calcite.sql.SqlWithItem;
import org.apache.calcite.sql.SqlWriterConfig;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.pretty.SqlPrettyWriter;
import java.util.ArrayList;
import java.util.List;
@Slf4j
public class SqlMergeWithUtils {
public static String mergeWith(
EngineType engineType,
String sql,
List<String> parentSqlList,
List<String> parentWithNameList)
throws SqlParseException {
SqlParser.Config parserConfig = Configuration.getParserConfig(engineType);
// Parse the main SQL statement
SqlParser parser = SqlParser.create(sql, parserConfig);
SqlNode sqlNode1 = parser.parseQuery();
// List to hold all WITH items
List<SqlNode> withItemList = new ArrayList<>();
// Iterate over each parentSql and parentWithName pair
for (int i = 0; i < parentSqlList.size(); i++) {
String parentSql = parentSqlList.get(i);
String parentWithName = parentWithNameList.get(i);
// Parse the parent SQL statement
parser = SqlParser.create(parentSql, parserConfig);
SqlNode sqlNode2 = parser.parseQuery();
// Create a new WITH item for parentWithName without quotes
SqlWithItem withItem =
new SqlWithItem(
SqlParserPos.ZERO,
new SqlIdentifier(
parentWithName, SqlParserPos.ZERO), // false to avoid quotes
null,
sqlNode2,
SqlLiteral.createBoolean(false, SqlParserPos.ZERO));
// Add the new WITH item to the list
withItemList.add(withItem);
}
// Extract existing WITH items from sqlNode1 if it is a SqlWith
if (sqlNode1 instanceof SqlWith) {
SqlWith sqlWith = (SqlWith) sqlNode1;
withItemList.addAll(sqlWith.withList.getList());
sqlNode1 = sqlWith.body;
}
// Create a new SqlWith node
SqlWith finalSqlNode =
new SqlWith(
SqlParserPos.ZERO,
new SqlNodeList(withItemList, SqlParserPos.ZERO),
sqlNode1);
// Custom SqlPrettyWriter configuration to avoid quoting identifiers
SqlWriterConfig config = Configuration.getSqlWriterConfig(engineType);
// Pretty print the final SQL
SqlPrettyWriter writer = new SqlPrettyWriter(config);
return writer.format(finalSqlNode);
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/ViewExpanderImpl.java → common/src/main/java/com/tencent/supersonic/common/calcite/ViewExpanderImpl.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.calcite.schema;
package com.tencent.supersonic.common.calcite;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.rel.RelRoot;

1
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAsHelper.java
View File

@@ -14,7 +14,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
/** Sql Parser Select Helper */
@Slf4j
public class SqlAsHelper {

47
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java
View File

@@ -709,51 +709,4 @@ public class SqlReplaceHelper {
}
}
}
public static String dealAliasToOrderBy(String querySql) {
Select selectStatement = SqlSelectHelper.getSelect(querySql);
List<PlainSelect> plainSelectList = new ArrayList<>();
// List<PlainSelect> withPlainSelectList = SqlSelectHelper.getWithItem(selectStatement);
// if (!CollectionUtils.isEmpty(withPlainSelectList)) {
// plainSelectList.addAll(withPlainSelectList);
// }
if (selectStatement instanceof PlainSelect) {
plainSelectList.add((PlainSelect) selectStatement);
} else if (selectStatement instanceof SetOperationList) {
SetOperationList setOperationList = (SetOperationList) selectStatement;
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
setOperationList
.getSelects()
.forEach(
subSelectBody -> {
PlainSelect subPlainSelect = (PlainSelect) subSelectBody;
plainSelectList.add(subPlainSelect);
});
}
}
for (PlainSelect plainSelect : plainSelectList) {
List<SelectItem<?>> selectItemList = plainSelect.getSelectItems();
List<OrderByElement> orderByElementList = plainSelect.getOrderByElements();
if (CollectionUtils.isEmpty(orderByElementList)) {
continue;
}
Map<String, Expression> map = new HashMap<>();
for (int i = 0; i < selectItemList.size(); i++) {
if (!Objects.isNull(selectItemList.get(i).getAlias())) {
map.put(
selectItemList.get(i).getAlias().getName(),
selectItemList.get(i).getExpression());
selectItemList.get(i).setAlias(null);
}
}
for (OrderByElement orderByElement : orderByElementList) {
if (map.containsKey(orderByElement.getExpression().toString())) {
orderByElement.setExpression(
map.get(orderByElement.getExpression().toString()));
}
}
plainSelect.setOrderByElements(orderByElementList);
}
return selectStatement.toString();
}
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/EngineType.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/EngineType.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.api.pojo.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum EngineType {
TDW(0, "tdw"),

64
common/src/test/java/com/tencent/supersonic/common/calcite/SqlWithMergerTest.java Normal file
View File

@@ -0,0 +1,64 @@
package com.tencent.supersonic.common.calcite;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import java.util.Collections;
@Slf4j
class SqlWithMergerTest {
@Test
void testWithMerger() throws SqlParseException {
String sql1 =
"WITH DepartmentVisits AS (\n"
+ " SELECT department, SUM(pv) AS 总访问次数\n"
+ " FROM t_1\n"
+ " WHERE sys_imp_date >= '2024-09-01' AND sys_imp_date <= '2024-09-29'\n"
+ " GROUP BY department\n"
+ ")\n"
+ "SELECT COUNT(*) FROM DepartmentVisits WHERE 总访问次数 > 100";
String sql2 =
"SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv`\n"
+ "FROM (SELECT `user_name`, `department` FROM `s2_user_department`) AS `t2`\n"
+ "LEFT JOIN (SELECT 1 AS `s2_pv_uv_statis_pv`, `imp_date` AS `sys_imp_date`, `user_name`\n"
+ "FROM `s2_pv_uv_statis`) AS `t3` ON `t2`.`user_name` = `t3`.`user_name`";
String mergeSql =
SqlMergeWithUtils.mergeWith(
EngineType.MYSQL,
sql1,
Collections.singletonList(sql2),
Collections.singletonList("t_1"));
System.out.println(mergeSql);
sql1 =
"WITH DepartmentVisits AS (SELECT department, SUM(pv) AS 总访问次数 FROM t_1 WHERE sys_imp_date >= '2024-08-28' "
+ "AND sys_imp_date <= '2024-09-28' GROUP BY department) SELECT COUNT(*) FROM DepartmentVisits WHERE 总访问次数 > 100 LIMIT 1000";
sql2 =
"SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv`\n"
+ "FROM\n"
+ "(SELECT `user_name`, `department`\n"
+ "FROM\n"
+ "`s2_user_department`) AS `t2`\n"
+ "LEFT JOIN (SELECT 1 AS `s2_pv_uv_statis_pv`, `imp_date` AS `sys_imp_date`, `user_name`\n"
+ "FROM\n"
+ "`s2_pv_uv_statis`) AS `t3` ON `t2`.`user_name` = `t3`.`user_name`";
mergeSql =
SqlMergeWithUtils.mergeWith(
EngineType.H2,
sql1,
Collections.singletonList(sql2),
Collections.singletonList("t_1"));
System.out.println(mergeSql);
}
}

2
common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlAsHelperTest.java
View File

@@ -8,7 +8,7 @@ import java.util.List;
class SqlAsHelperTest {
@Test
void testReplaceAggField() {
void getAsFields() {
String sql =
"WITH SalesData AS (\n"
+ " SELECT \n"

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.chat.corrector;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.QueryType;
@@ -41,8 +40,7 @@ public class SelectCorrector extends BaseSemanticCorrector {
return;
}
correctS2SQL = addFieldsToSelect(chatQueryContext, semanticParseInfo, correctS2SQL);
String querySql = SqlReplaceHelper.dealAliasToOrderBy(correctS2SQL);
semanticParseInfo.getSqlInfo().setCorrectedS2SQL(querySql);
semanticParseInfo.getSqlInfo().setCorrectedS2SQL(correctS2SQL);
}
protected String addFieldsToSelect(

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/DbAdaptorFactory.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.adaptor.db;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import java.util.HashMap;
import java.util.Map;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.headless.core.executor;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.core.pojo.Materialization;
import com.tencent.supersonic.headless.core.translator.calcite.Configuration;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.TimeRange;
import com.tencent.supersonic.headless.core.translator.calcite.schema.DataSourceTable;
import com.tencent.supersonic.headless.core.translator.calcite.schema.DataSourceTable.Builder;

20
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java
View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
@@ -8,6 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.translator.converter.QueryConverter;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
@@ -81,6 +84,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
public QueryStatement doParse(
DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) {
log.info("parse dataSetQuery [{}] ", dataSetQueryParam);
SemanticModel semanticModel = queryStatement.getSemanticModel();
EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType());
try {
if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) {
List<String[]> tables = new ArrayList<>();
@@ -99,13 +104,16 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
if (!tables.isEmpty()) {
String sql;
if (dataSetQueryParam.isSupportWith()) {
List<String> parentWithNameList =
tables.stream().map(table -> table[0]).collect(Collectors.toList());
List<String> parentSqlList =
tables.stream().map(table -> table[1]).collect(Collectors.toList());
sql =
"with "
+ tables.stream()
.map(t -> String.format("%s as (%s)", t[0], t[1]))
.collect(Collectors.joining(","))
+ "\n"
+ dataSetQueryParam.getSql();
SqlMergeWithUtils.mergeWith(
engineType,
dataSetQueryParam.getSql(),
parentSqlList,
parentWithNameList);
} else {
sql = dataSetQueryParam.getSql();
for (String[] tb : tables) {

19
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java
View File

@@ -1,7 +1,8 @@
package com.tencent.supersonic.headless.core.translator.calcite;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.QueryParser;
@@ -10,8 +11,10 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticMod
import com.tencent.supersonic.headless.core.translator.calcite.schema.RuntimeOptions;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException;
import org.springframework.stereotype.Component;
import java.util.Collections;
import java.util.Objects;
/** the calcite parse implements */
@@ -31,8 +34,7 @@ public class CalciteQueryParser implements QueryParser {
SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement);
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
aggBuilder.explain(queryStatement, isAgg);
EngineType engineType =
EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType());
EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType());
queryStatement.setSql(aggBuilder.getSql(engineType));
if (Objects.nonNull(queryStatement.getEnableOptimize())
&& queryStatement.getEnableOptimize()
@@ -42,6 +44,7 @@ public class CalciteQueryParser implements QueryParser {
String simplifySql =
aggBuilder.simplify(
getSqlByDataSet(
engineType,
aggBuilder.getSql(engineType),
queryStatement.getDataSetSql(),
queryStatement.getDataSetAlias()),
@@ -70,7 +73,13 @@ public class CalciteQueryParser implements QueryParser {
return semanticSchema;
}
private String getSqlByDataSet(String sql, String dataSetSql, String dataSetAlias) {
return String.format("with %s as (%s) %s", dataSetAlias, sql, dataSetSql);
private String getSqlByDataSet(
EngineType engineType, String parentSql, String dataSetSql, String parentAlias)
throws SqlParseException {
return SqlMergeWithUtils.mergeWith(
engineType,
dataSetSql,
Collections.singletonList(parentSql),
Collections.singletonList(parentAlias));
}
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/AggPlanner.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.headless.core.translator.calcite.planner;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.Configuration;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SchemaBuilder;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/planner/Planner.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.translator.calcite.planner;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
/** parse and generate SQL and other execute information */

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/schema/SchemaBuilder.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.translator.calcite.schema;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.Configuration;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2SQLSqlValidatorImpl;
import org.apache.calcite.jdbc.CalciteSchema;
import org.apache.calcite.prepare.CalciteCatalogReader;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/Renderer.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/AggFunctionNode.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.node;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataSourceNode.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.node;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.Configuration;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DimensionNode.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.node;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import org.apache.calcite.sql.SqlNode;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/IdentifyNode.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.node;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MeasureNode.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.node;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/MetricNode.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.node;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema;
import lombok.Data;

29
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/SemanticNode.java
View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.node;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.Configuration;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.calcite.SemanticSqlDialect;
import com.tencent.supersonic.common.calcite.SqlDialectFactory;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.headless.core.translator.calcite.sql.optimizer.FilterToGroupScanRule;
import com.tencent.supersonic.headless.core.utils.SqlDialectFactory;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.hep.HepPlanner;
@@ -31,7 +31,6 @@ import org.apache.calcite.sql.SqlWriterConfig;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.pretty.SqlPrettyWriter;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.calcite.sql.validate.SqlValidatorWithHints;
@@ -102,7 +101,7 @@ public abstract class SemanticNode {
public static String getSql(SqlNode sqlNode, EngineType engineType) {
UnaryOperator<SqlWriterConfig> sqlWriterConfigUnaryOperator =
(c) -> getSqlWriterConfig(engineType);
(c) -> Configuration.getSqlWriterConfig(engineType);
return sqlNode.toSqlString(sqlWriterConfigUnaryOperator).getSql();
}
@@ -184,24 +183,6 @@ public abstract class SemanticNode {
return sqlNode;
}
private static SqlWriterConfig getSqlWriterConfig(EngineType engineType) {
SemanticSqlDialect sqlDialect = SqlDialectFactory.getSqlDialect(engineType);
SqlWriterConfig config =
SqlPrettyWriter.config()
.withDialect(sqlDialect)
.withKeywordsLowerCase(false)
.withClauseEndsLine(true)
.withAlwaysUseParentheses(false)
.withSelectListItemsOnSeparateLines(false)
.withUpdateSetListNewline(false)
.withIndentation(0);
if (EngineType.MYSQL.equals(engineType)) {
// no backticks around function name
config = config.withQuoteAllIdentifiers(false);
}
return config;
}
private static void sqlVisit(SqlNode sqlNode, Map<String, Object> parseInfo) {
SqlKind kind = sqlNode.getKind();
switch (kind) {

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.translator.calcite.schema.SemanticSchema;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataSource;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java
View File

@@ -3,12 +3,12 @@ package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
@@ -13,7 +14,6 @@ import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.server.pojo;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import java.util.LinkedHashMap;
import java.util.Map;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -4,8 +4,8 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java
View File

@@ -275,6 +275,7 @@ public class ModelConverter {
continue;
}
// Compatible with front-end tmp
String oriFieldName =
measure.getBizName().replaceFirst(modelReq.getBizName() + "_", "");
measure.setExpr(oriFieldName);

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -13,7 +14,6 @@ import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;

2
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.headless.server.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;