From 61e22c21046f30a0241fd599064e71464f88a607 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Wed, 8 Jan 2025 19:43:32 +0800 Subject: [PATCH] [improvement][headless]Add sql field in the OntologyQuery. --- .../common/jsqlparser/SqlReplaceHelper.java | 18 +++-- .../headless/api/pojo/enums/DataType.java | 2 +- .../headless/core/pojo/OntologyQuery.java | 1 + .../headless/core/pojo/QueryStatement.java | 3 +- .../translator/DefaultSemanticTranslator.java | 66 +++++++++---------- .../core/translator/SemanticTranslator.java | 2 +- .../parser/OntologyQueryParser.java | 2 +- .../translator/parser/SqlQueryParser.java | 3 +- .../service/impl/S2SemanticLayerService.java | 6 +- .../tencent/supersonic/demo/S2BaseDemo.java | 8 ++- 10 files changed, 62 insertions(+), 49 deletions(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java index 1bd114f9d..91fbea3ec 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java @@ -423,8 +423,10 @@ public class SqlReplaceHelper { painSelect.accept(new SelectVisitorAdapter() { @Override public void visit(PlainSelect plainSelect) { - plainSelect.getFromItem().accept( - new TableNameReplaceVisitor(tableName, new HashSet<>(withNameList))); + if (Objects.nonNull(plainSelect.getFromItem())) { + plainSelect.getFromItem().accept(new TableNameReplaceVisitor(tableName, + new HashSet<>(withNameList))); + } } }); replaceJoins(painSelect, tableName, withNameList); @@ -672,11 +674,13 @@ public class SqlReplaceHelper { List plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); for (PlainSelect plainSelect : plainSelects) { - Table table = (Table) plainSelect.getFromItem(); - if (table.getName().equals(tableName)) { - replacePlainSelectByExpr(plainSelect, replace); - if (SqlSelectHelper.hasAggregateFunction(plainSelect)) { - SqlSelectHelper.addMissingGroupby(plainSelect); + if (Objects.nonNull(plainSelect.getFromItem())) { + Table table = (Table) plainSelect.getFromItem(); + if (table.getName().equals(tableName)) { + replacePlainSelectByExpr(plainSelect, replace); + if (SqlSelectHelper.hasAggregateFunction(plainSelect)) { + SqlSelectHelper.addMissingGroupby(plainSelect); + } } } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java index 03578c0f9..2670eec81 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/DataType.java @@ -15,7 +15,7 @@ public enum DataType { SQLSERVER("sqlserver", "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""), - H2("h2", "h2", "org.h2.Driver", "`", "`", "\"", "\""), + H2("H2", "h2", "org.h2.Driver", "`", "`", "\"", "\""), PHOENIX("phoenix", "hbase phoenix", "org.apache.phoenix.jdbc.PhoenixDriver", "", "", "\"", "\""), diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java index bfe70cde0..ba2517be2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/OntologyQuery.java @@ -29,6 +29,7 @@ public class OntologyQuery { private List order; private boolean nativeQuery = true; private AggOption aggOption = AggOption.NATIVE; + private String sql; public Set getModels() { return modelMap.values().stream().collect(Collectors.toSet()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index af5a706dc..83ac4fbf9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.core.pojo; +import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import lombok.Data; import org.apache.commons.lang3.StringUtils; @@ -15,7 +16,7 @@ public class QueryStatement { private StructQuery structQuery; private SqlQuery sqlQuery; private OntologyQuery ontologyQuery; - private Integer status = 0; + private QueryState status = QueryState.SUCCESS; private Boolean isS2SQL = false; private Boolean enableOptimize = true; private Triple minMaxTime; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java index 9beacff58..3a27a88de 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.translator; import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; import com.tencent.supersonic.common.pojo.enums.EngineType; +import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.SqlQuery; @@ -15,82 +16,77 @@ import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; @Component @Slf4j public class DefaultSemanticTranslator implements SemanticTranslator { - public void translate(QueryStatement queryStatement) { + public void translate(QueryStatement queryStatement) throws Exception { if (queryStatement.isTranslated()) { return; } - try { - for (QueryParser parser : ComponentFactory.getQueryParsers()) { - if (parser.accept(queryStatement)) { - log.debug("QueryConverter accept [{}]", parser.getClass().getName()); - parser.parse(queryStatement); - if (queryStatement.getStatus() != 0) { - break; - } + for (QueryParser parser : ComponentFactory.getQueryParsers()) { + if (parser.accept(queryStatement)) { + log.debug("QueryConverter accept [{}]", parser.getClass().getName()); + parser.parse(queryStatement); + if (!queryStatement.getStatus().equals(QueryState.SUCCESS)) { + break; } } - mergeOntologyQuery(queryStatement); - - if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) { - queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql()); - } - if (StringUtils.isBlank(queryStatement.getSql())) { - throw new RuntimeException("parse exception: " + queryStatement.getErrMsg()); - } - - for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) { - if (optimizer.accept(queryStatement)) { - optimizer.rewrite(queryStatement); - } - } - log.info("translated query SQL: [{}]", - StringUtils.normalizeSpace(queryStatement.getSql())); - } catch (Exception e) { - queryStatement.setErrMsg(e.getMessage()); - log.error("Failed to translate query [{}]", e.getMessage(), e); } + mergeOntologyQuery(queryStatement); + + if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) { + queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql()); + } + if (StringUtils.isBlank(queryStatement.getSql())) { + throw new RuntimeException("parse exception: " + queryStatement.getErrMsg()); + } + + for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) { + if (optimizer.accept(queryStatement)) { + optimizer.rewrite(queryStatement); + } + } + log.info("translated query SQL: [{}]", StringUtils.normalizeSpace(queryStatement.getSql())); } private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception { OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); log.info("parse with ontology: [{}]", ontologyQuery); - if (!queryStatement.isOk()) { + if (Objects.isNull(ontologyQuery) || StringUtils.isBlank(ontologyQuery.getSql())) { throw new Exception(String.format("parse ontology table [%s] error [%s]", queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg())); } SqlQuery sqlQuery = queryStatement.getSqlQuery(); - String ontologyQuerySql = sqlQuery.getSql(); + String ontologyOuterSql = sqlQuery.getSql(); String ontologyInnerTable = sqlQuery.getTable(); - String ontologyInnerSql = queryStatement.getSql(); + String ontologyInnerSql = ontologyQuery.getSql(); List> tables = new ArrayList<>(); tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql)); String finalSql = null; if (sqlQuery.isSupportWith()) { EngineType engineType = queryStatement.getOntology().getDatabaseType(); - if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) { + if (!SqlMergeWithUtils.hasWith(engineType, ontologyOuterSql)) { finalSql = "with " + tables.stream() .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight())) - .collect(Collectors.joining(",")) + "\n" + ontologyQuerySql; + .collect(Collectors.joining(",")) + "\n" + ontologyOuterSql; } else { List withTableList = tables.stream().map(Pair::getLeft).collect(Collectors.toList()); List withSqlList = tables.stream().map(Pair::getRight).collect(Collectors.toList()); - finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyQuerySql, withSqlList, + finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyOuterSql, withSqlList, withTableList); } } else { for (Pair tb : tables) { - finalSql = StringUtils.replace(ontologyQuerySql, tb.getLeft(), + finalSql = StringUtils.replace(ontologyOuterSql, tb.getLeft(), "(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()), -1); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java index 601c1626f..a35e88f6d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/SemanticTranslator.java @@ -8,5 +8,5 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement; */ public interface SemanticTranslator { - void translate(QueryStatement queryStatement); + void translate(QueryStatement queryStatement) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java index e4467467a..d58e006d7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/OntologyQueryParser.java @@ -32,7 +32,7 @@ public class OntologyQueryParser implements QueryParser { .build(); SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema); String sql = sqlBuilder.buildOntologySql(queryStatement); - queryStatement.setSql(sql); + queryStatement.getOntologyQuery().setSql(sql); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java index 473ed0fdf..d066043a0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java @@ -10,6 +10,7 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; @@ -50,7 +51,7 @@ public class SqlQueryParser implements QueryParser { + ontologyQuery.getDimensions().size()) { queryStatement .setErrMsg("There are fields in the SQL not matched with any semantic column."); - queryStatement.setStatus(1); + queryStatement.setStatus(QueryState.INVALID); return; } queryStatement.setOntologyQuery(ontologyQuery); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index ed359a7c8..2cfb013ae 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -337,7 +337,11 @@ public class S2SemanticLayerService implements SemanticLayerService { List queryStatements = new ArrayList<>(); for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { QueryStatement queryStatement = buildStructQueryStatement(queryStructReq); - semanticTranslator.translate(queryStatement); + try { + semanticTranslator.translate(queryStatement); + } catch (Exception e) { + log.warn("Failed to translate for semantic query " + queryStructReq); + } queryStatements.add(queryStatement); } log.info("Union multiple query statements:{}", queryStatements); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java index a908cb35b..d2fa45153 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2BaseDemo.java @@ -121,7 +121,13 @@ public abstract class S2BaseDemo implements CommandLineRunner { DatabaseReq databaseReq = new DatabaseReq(); databaseReq.setName("S2数据库DEMO"); databaseReq.setDescription("样例数据库实例仅用于体验"); - databaseReq.setType(DataType.POSTGRESQL.getFeature()); + databaseReq.setType(DataType.H2.getFeature()); + String profile = System.getProperty("spring.profiles.active"); + if ("postgres".equalsIgnoreCase(profile)) { + databaseReq.setType(DataType.POSTGRESQL.getFeature()); + } else if ("mysql".equalsIgnoreCase(profile)) { + databaseReq.setType(DataType.MYSQL.getFeature()); + } databaseReq.setUrl(url); databaseReq.setUsername(dataSourceProperties.getUsername()); databaseReq