[improvement][headless]Add sql field in the OntologyQuery.

This commit is contained in:
jerryjzhang
2025-01-08 19:43:32 +08:00
parent 6e4260f9f1
commit 61e22c2104
10 changed files with 62 additions and 49 deletions

View File

@@ -423,8 +423,10 @@ public class SqlReplaceHelper {
painSelect.accept(new SelectVisitorAdapter() { painSelect.accept(new SelectVisitorAdapter() {
@Override @Override
public void visit(PlainSelect plainSelect) { public void visit(PlainSelect plainSelect) {
plainSelect.getFromItem().accept( if (Objects.nonNull(plainSelect.getFromItem())) {
new TableNameReplaceVisitor(tableName, new HashSet<>(withNameList))); plainSelect.getFromItem().accept(new TableNameReplaceVisitor(tableName,
new HashSet<>(withNameList)));
}
} }
}); });
replaceJoins(painSelect, tableName, withNameList); replaceJoins(painSelect, tableName, withNameList);
@@ -672,6 +674,7 @@ public class SqlReplaceHelper {
List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
for (PlainSelect plainSelect : plainSelects) { for (PlainSelect plainSelect : plainSelects) {
if (Objects.nonNull(plainSelect.getFromItem())) {
Table table = (Table) plainSelect.getFromItem(); Table table = (Table) plainSelect.getFromItem();
if (table.getName().equals(tableName)) { if (table.getName().equals(tableName)) {
replacePlainSelectByExpr(plainSelect, replace); replacePlainSelectByExpr(plainSelect, replace);
@@ -680,6 +683,7 @@ public class SqlReplaceHelper {
} }
} }
} }
}
return selectStatement.toString(); return selectStatement.toString();
} }

View File

@@ -15,7 +15,7 @@ public enum DataType {
SQLSERVER("sqlserver", "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", SQLSERVER("sqlserver", "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"",
"\"", "\""), "\"", "\""),
H2("h2", "h2", "org.h2.Driver", "`", "`", "\"", "\""), H2("H2", "h2", "org.h2.Driver", "`", "`", "\"", "\""),
PHOENIX("phoenix", "hbase phoenix", "org.apache.phoenix.jdbc.PhoenixDriver", "", "", "\"", PHOENIX("phoenix", "hbase phoenix", "org.apache.phoenix.jdbc.PhoenixDriver", "", "", "\"",
"\""), "\""),

View File

@@ -29,6 +29,7 @@ public class OntologyQuery {
private List<ColumnOrder> order; private List<ColumnOrder> order;
private boolean nativeQuery = true; private boolean nativeQuery = true;
private AggOption aggOption = AggOption.NATIVE; private AggOption aggOption = AggOption.NATIVE;
private String sql;
public Set<ModelResp> getModels() { public Set<ModelResp> getModels() {
return modelMap.values().stream().collect(Collectors.toSet()); return modelMap.values().stream().collect(Collectors.toSet());

View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.core.pojo; package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import lombok.Data; import lombok.Data;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@@ -15,7 +16,7 @@ public class QueryStatement {
private StructQuery structQuery; private StructQuery structQuery;
private SqlQuery sqlQuery; private SqlQuery sqlQuery;
private OntologyQuery ontologyQuery; private OntologyQuery ontologyQuery;
private Integer status = 0; private QueryState status = QueryState.SUCCESS;
private Boolean isS2SQL = false; private Boolean isS2SQL = false;
private Boolean enableOptimize = true; private Boolean enableOptimize = true;
private Triple<String, String, String> minMaxTime; private Triple<String, String, String> minMaxTime;

View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQuery; import com.tencent.supersonic.headless.core.pojo.SqlQuery;
@@ -15,22 +16,22 @@ import org.springframework.stereotype.Component;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Component @Component
@Slf4j @Slf4j
public class DefaultSemanticTranslator implements SemanticTranslator { public class DefaultSemanticTranslator implements SemanticTranslator {
public void translate(QueryStatement queryStatement) { public void translate(QueryStatement queryStatement) throws Exception {
if (queryStatement.isTranslated()) { if (queryStatement.isTranslated()) {
return; return;
} }
try {
for (QueryParser parser : ComponentFactory.getQueryParsers()) { for (QueryParser parser : ComponentFactory.getQueryParsers()) {
if (parser.accept(queryStatement)) { if (parser.accept(queryStatement)) {
log.debug("QueryConverter accept [{}]", parser.getClass().getName()); log.debug("QueryConverter accept [{}]", parser.getClass().getName());
parser.parse(queryStatement); parser.parse(queryStatement);
if (queryStatement.getStatus() != 0) { if (!queryStatement.getStatus().equals(QueryState.SUCCESS)) {
break; break;
} }
} }
@@ -49,48 +50,43 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
optimizer.rewrite(queryStatement); optimizer.rewrite(queryStatement);
} }
} }
log.info("translated query SQL: [{}]", log.info("translated query SQL: [{}]", StringUtils.normalizeSpace(queryStatement.getSql()));
StringUtils.normalizeSpace(queryStatement.getSql()));
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("Failed to translate query [{}]", e.getMessage(), e);
}
} }
private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception { private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception {
OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
log.info("parse with ontology: [{}]", ontologyQuery); log.info("parse with ontology: [{}]", ontologyQuery);
if (!queryStatement.isOk()) { if (Objects.isNull(ontologyQuery) || StringUtils.isBlank(ontologyQuery.getSql())) {
throw new Exception(String.format("parse ontology table [%s] error [%s]", throw new Exception(String.format("parse ontology table [%s] error [%s]",
queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg())); queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg()));
} }
SqlQuery sqlQuery = queryStatement.getSqlQuery(); SqlQuery sqlQuery = queryStatement.getSqlQuery();
String ontologyQuerySql = sqlQuery.getSql(); String ontologyOuterSql = sqlQuery.getSql();
String ontologyInnerTable = sqlQuery.getTable(); String ontologyInnerTable = sqlQuery.getTable();
String ontologyInnerSql = queryStatement.getSql(); String ontologyInnerSql = ontologyQuery.getSql();
List<Pair<String, String>> tables = new ArrayList<>(); List<Pair<String, String>> tables = new ArrayList<>();
tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql)); tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql));
String finalSql = null; String finalSql = null;
if (sqlQuery.isSupportWith()) { if (sqlQuery.isSupportWith()) {
EngineType engineType = queryStatement.getOntology().getDatabaseType(); EngineType engineType = queryStatement.getOntology().getDatabaseType();
if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) { if (!SqlMergeWithUtils.hasWith(engineType, ontologyOuterSql)) {
finalSql = "with " + tables.stream() finalSql = "with " + tables.stream()
.map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight())) .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight()))
.collect(Collectors.joining(",")) + "\n" + ontologyQuerySql; .collect(Collectors.joining(",")) + "\n" + ontologyOuterSql;
} else { } else {
List<String> withTableList = List<String> withTableList =
tables.stream().map(Pair::getLeft).collect(Collectors.toList()); tables.stream().map(Pair::getLeft).collect(Collectors.toList());
List<String> withSqlList = List<String> withSqlList =
tables.stream().map(Pair::getRight).collect(Collectors.toList()); tables.stream().map(Pair::getRight).collect(Collectors.toList());
finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyQuerySql, withSqlList, finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyOuterSql, withSqlList,
withTableList); withTableList);
} }
} else { } else {
for (Pair<String, String> tb : tables) { for (Pair<String, String> tb : tables) {
finalSql = StringUtils.replace(ontologyQuerySql, tb.getLeft(), finalSql = StringUtils.replace(ontologyOuterSql, tb.getLeft(),
"(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()), "(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()),
-1); -1);
} }

View File

@@ -8,5 +8,5 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
*/ */
public interface SemanticTranslator { public interface SemanticTranslator {
void translate(QueryStatement queryStatement); void translate(QueryStatement queryStatement) throws Exception;
} }

View File

@@ -32,7 +32,7 @@ public class OntologyQueryParser implements QueryParser {
.build(); .build();
SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema); SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema);
String sql = sqlBuilder.buildOntologySql(queryStatement); String sql = sqlBuilder.buildOntologySql(queryStatement);
queryStatement.setSql(sql); queryStatement.getOntologyQuery().setSql(sql);
} }
} }

View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.pojo.Ontology;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
@@ -50,7 +51,7 @@ public class SqlQueryParser implements QueryParser {
+ ontologyQuery.getDimensions().size()) { + ontologyQuery.getDimensions().size()) {
queryStatement queryStatement
.setErrMsg("There are fields in the SQL not matched with any semantic column."); .setErrMsg("There are fields in the SQL not matched with any semantic column.");
queryStatement.setStatus(1); queryStatement.setStatus(QueryState.INVALID);
return; return;
} }
queryStatement.setOntologyQuery(ontologyQuery); queryStatement.setOntologyQuery(ontologyQuery);

View File

@@ -337,7 +337,11 @@ public class S2SemanticLayerService implements SemanticLayerService {
List<QueryStatement> queryStatements = new ArrayList<>(); List<QueryStatement> queryStatements = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildStructQueryStatement(queryStructReq); QueryStatement queryStatement = buildStructQueryStatement(queryStructReq);
try {
semanticTranslator.translate(queryStatement); semanticTranslator.translate(queryStatement);
} catch (Exception e) {
log.warn("Failed to translate for semantic query " + queryStructReq);
}
queryStatements.add(queryStatement); queryStatements.add(queryStatement);
} }
log.info("Union multiple query statements:{}", queryStatements); log.info("Union multiple query statements:{}", queryStatements);

View File

@@ -121,7 +121,13 @@ public abstract class S2BaseDemo implements CommandLineRunner {
DatabaseReq databaseReq = new DatabaseReq(); DatabaseReq databaseReq = new DatabaseReq();
databaseReq.setName("S2数据库DEMO"); databaseReq.setName("S2数据库DEMO");
databaseReq.setDescription("样例数据库实例仅用于体验"); databaseReq.setDescription("样例数据库实例仅用于体验");
databaseReq.setType(DataType.H2.getFeature());
String profile = System.getProperty("spring.profiles.active");
if ("postgres".equalsIgnoreCase(profile)) {
databaseReq.setType(DataType.POSTGRESQL.getFeature()); databaseReq.setType(DataType.POSTGRESQL.getFeature());
} else if ("mysql".equalsIgnoreCase(profile)) {
databaseReq.setType(DataType.MYSQL.getFeature());
}
databaseReq.setUrl(url); databaseReq.setUrl(url);
databaseReq.setUsername(dataSourceProperties.getUsername()); databaseReq.setUsername(dataSourceProperties.getUsername());
databaseReq databaseReq