topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Add sql field in the OntologyQuery.

Browse Source
This commit is contained in:
jerryjzhang
2025-01-08 19:43:32 +08:00
parent 6e4260f9f1
commit 61e22c2104
10 changed files with 62 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java
View File

@@ -423,8 +423,10 @@ public class SqlReplaceHelper {
painSelect.accept(new SelectVisitorAdapter() {
@Override
public void visit(PlainSelect plainSelect) {
plainSelect.getFromItem().accept(
new TableNameReplaceVisitor(tableName, new HashSet<>(withNameList)));
if (Objects.nonNull(plainSelect.getFromItem())) {
plainSelect.getFromItem().accept(new TableNameReplaceVisitor(tableName,
new HashSet<>(withNameList)));
}
}
});
replaceJoins(painSelect, tableName, withNameList);
@@ -672,11 +674,13 @@ public class SqlReplaceHelper {
List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
for (PlainSelect plainSelect : plainSelects) {
Table table = (Table) plainSelect.getFromItem();
if (table.getName().equals(tableName)) {
replacePlainSelectByExpr(plainSelect, replace);
if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
SqlSelectHelper.addMissingGroupby(plainSelect);
if (Objects.nonNull(plainSelect.getFromItem())) {
Table table = (Table) plainSelect.getFromItem();
if (table.getName().equals(tableName)) {
replacePlainSelectByExpr(plainSelect, replace);
if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
SqlSelectHelper.addMissingGroupby(plainSelect);
}
}
}
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java
View File

@@ -15,7 +15,7 @@ public enum DataType {
SQLSERVER("sqlserver", "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"",
"\"", "\""),
H2("h2", "h2", "org.h2.Driver", "`", "`", "\"", "\""),
H2("H2", "h2", "org.h2.Driver", "`", "`", "\"", "\""),
PHOENIX("phoenix", "hbase phoenix", "org.apache.phoenix.jdbc.PhoenixDriver", "", "", "\"",
"\""),

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java
View File

@@ -29,6 +29,7 @@ public class OntologyQuery {
private List<ColumnOrder> order;
private boolean nativeQuery = true;
private AggOption aggOption = AggOption.NATIVE;
private String sql;
public Set<ModelResp> getModels() {
return modelMap.values().stream().collect(Collectors.toSet());

3
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
@@ -15,7 +16,7 @@ public class QueryStatement {
private StructQuery structQuery;
private SqlQuery sqlQuery;
private OntologyQuery ontologyQuery;
private Integer status = 0;
private QueryState status = QueryState.SUCCESS;
private Boolean isS2SQL = false;
private Boolean enableOptimize = true;
private Triple<String, String, String> minMaxTime;

66
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQuery;
@@ -15,82 +16,77 @@ import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Component
@Slf4j
public class DefaultSemanticTranslator implements SemanticTranslator {
public void translate(QueryStatement queryStatement) {
public void translate(QueryStatement queryStatement) throws Exception {
if (queryStatement.isTranslated()) {
return;
}
try {
for (QueryParser parser : ComponentFactory.getQueryParsers()) {
if (parser.accept(queryStatement)) {
log.debug("QueryConverter accept [{}]", parser.getClass().getName());
parser.parse(queryStatement);
if (queryStatement.getStatus() != 0) {
break;
}
for (QueryParser parser : ComponentFactory.getQueryParsers()) {
if (parser.accept(queryStatement)) {
log.debug("QueryConverter accept [{}]", parser.getClass().getName());
parser.parse(queryStatement);
if (!queryStatement.getStatus().equals(QueryState.SUCCESS)) {
break;
}
}
mergeOntologyQuery(queryStatement);
if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) {
queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql());
}
if (StringUtils.isBlank(queryStatement.getSql())) {
throw new RuntimeException("parse exception: " + queryStatement.getErrMsg());
}
for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) {
if (optimizer.accept(queryStatement)) {
optimizer.rewrite(queryStatement);
}
}
log.info("translated query SQL: [{}]",
StringUtils.normalizeSpace(queryStatement.getSql()));
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("Failed to translate query [{}]", e.getMessage(), e);
}
mergeOntologyQuery(queryStatement);
if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) {
queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql());
}
if (StringUtils.isBlank(queryStatement.getSql())) {
throw new RuntimeException("parse exception: " + queryStatement.getErrMsg());
}
for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) {
if (optimizer.accept(queryStatement)) {
optimizer.rewrite(queryStatement);
}
}
log.info("translated query SQL: [{}]", StringUtils.normalizeSpace(queryStatement.getSql()));
}
private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception {
OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
log.info("parse with ontology: [{}]", ontologyQuery);
if (!queryStatement.isOk()) {
if (Objects.isNull(ontologyQuery) || StringUtils.isBlank(ontologyQuery.getSql())) {
throw new Exception(String.format("parse ontology table [%s] error [%s]",
queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg()));
}
SqlQuery sqlQuery = queryStatement.getSqlQuery();
String ontologyQuerySql = sqlQuery.getSql();
String ontologyOuterSql = sqlQuery.getSql();
String ontologyInnerTable = sqlQuery.getTable();
String ontologyInnerSql = queryStatement.getSql();
String ontologyInnerSql = ontologyQuery.getSql();
List<Pair<String, String>> tables = new ArrayList<>();
tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql));
String finalSql = null;
if (sqlQuery.isSupportWith()) {
EngineType engineType = queryStatement.getOntology().getDatabaseType();
if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) {
if (!SqlMergeWithUtils.hasWith(engineType, ontologyOuterSql)) {
finalSql = "with " + tables.stream()
.map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight()))
.collect(Collectors.joining(",")) + "\n" + ontologyQuerySql;
.collect(Collectors.joining(",")) + "\n" + ontologyOuterSql;
} else {
List<String> withTableList =
tables.stream().map(Pair::getLeft).collect(Collectors.toList());
List<String> withSqlList =
tables.stream().map(Pair::getRight).collect(Collectors.toList());
finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyQuerySql, withSqlList,
finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyOuterSql, withSqlList,
withTableList);
}
} else {
for (Pair<String, String> tb : tables) {
finalSql = StringUtils.replace(ontologyQuerySql, tb.getLeft(),
finalSql = StringUtils.replace(ontologyOuterSql, tb.getLeft(),
"(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()),
-1);
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java
View File

@@ -8,5 +8,5 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
*/
public interface SemanticTranslator {
void translate(QueryStatement queryStatement);
void translate(QueryStatement queryStatement) throws Exception;
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java
View File

@@ -32,7 +32,7 @@ public class OntologyQueryParser implements QueryParser {
.build();
SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema);
String sql = sqlBuilder.buildOntologySql(queryStatement);
queryStatement.setSql(sql);
queryStatement.getOntologyQuery().setSql(sql);
}
}

3
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java
View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.Ontology;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
@@ -50,7 +51,7 @@ public class SqlQueryParser implements QueryParser {
+ ontologyQuery.getDimensions().size()) {
queryStatement
.setErrMsg("There are fields in the SQL not matched with any semantic column.");
queryStatement.setStatus(1);
queryStatement.setStatus(QueryState.INVALID);
return;
}
queryStatement.setOntologyQuery(ontologyQuery);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -337,7 +337,11 @@ public class S2SemanticLayerService implements SemanticLayerService {
List<QueryStatement> queryStatements = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildStructQueryStatement(queryStructReq);
semanticTranslator.translate(queryStatement);
try {
semanticTranslator.translate(queryStatement);
} catch (Exception e) {
log.warn("Failed to translate for semantic query " + queryStructReq);
}
queryStatements.add(queryStatement);
}
log.info("Union multiple query statements:{}", queryStatements);

8
launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java
View File

@@ -121,7 +121,13 @@ public abstract class S2BaseDemo implements CommandLineRunner {
DatabaseReq databaseReq = new DatabaseReq();
databaseReq.setName("S2数据库DEMO");
databaseReq.setDescription("样例数据库实例仅用于体验");
databaseReq.setType(DataType.POSTGRESQL.getFeature());
databaseReq.setType(DataType.H2.getFeature());
String profile = System.getProperty("spring.profiles.active");
if ("postgres".equalsIgnoreCase(profile)) {
databaseReq.setType(DataType.POSTGRESQL.getFeature());
} else if ("mysql".equalsIgnoreCase(profile)) {
databaseReq.setType(DataType.MYSQL.getFeature());
}
databaseReq.setUrl(url);
databaseReq.setUsername(dataSourceProperties.getUsername());
databaseReq