[improvement][headless]Merge function of QueryConverter abstraction to QueryParser.

This commit is contained in:
jerryjzhang
2024-12-17 17:38:33 +08:00
parent 84944fa341
commit aa3b8997bd
21 changed files with 126 additions and 189 deletions

View File

@@ -5,8 +5,8 @@ import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQuery; import com.tencent.supersonic.headless.core.pojo.SqlQuery;
import com.tencent.supersonic.headless.core.translator.converter.QueryConverter;
import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.translator.parser.QueryParser;
import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@@ -26,13 +26,13 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
return; return;
} }
try { try {
for (QueryConverter converter : ComponentFactory.getQueryConverters()) { for (QueryParser parser : ComponentFactory.getQueryParsers()) {
if (converter.accept(queryStatement)) { if (parser.accept(queryStatement)) {
log.debug("QueryConverter accept [{}]", converter.getClass().getName()); log.debug("QueryConverter accept [{}]", parser.getClass().getName());
converter.convert(queryStatement); parser.parse(queryStatement);
} }
} }
doOntologyParse(queryStatement); mergeOntologyQuery(queryStatement);
if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) { if (StringUtils.isNotBlank(queryStatement.getSqlQuery().getSimplifiedSql())) {
queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql()); queryStatement.setSql(queryStatement.getSqlQuery().getSimplifiedSql());
@@ -41,8 +41,10 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
throw new RuntimeException("parse exception: " + queryStatement.getErrMsg()); throw new RuntimeException("parse exception: " + queryStatement.getErrMsg());
} }
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { for (QueryOptimizer optimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement); if (optimizer.accept(queryStatement)) {
optimizer.rewrite(queryStatement);
}
} }
log.info("translated query SQL: [{}]", queryStatement.getSql()); log.info("translated query SQL: [{}]", queryStatement.getSql());
} catch (Exception e) { } catch (Exception e) {
@@ -51,10 +53,10 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
} }
} }
private void doOntologyParse(QueryStatement queryStatement) throws Exception { private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception {
OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
log.info("parse with ontology: [{}]", ontologyQuery); log.info("parse with ontology: [{}]", ontologyQuery);
ComponentFactory.getQueryParser().parse(queryStatement);
if (!queryStatement.isOk()) { if (!queryStatement.isOk()) {
throw new Exception(String.format("parse ontology table [%s] error [%s]", throw new Exception(String.format("parse ontology table [%s] error [%s]",
queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg())); queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg()));
@@ -67,31 +69,30 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
List<Pair<String, String>> tables = new ArrayList<>(); List<Pair<String, String>> tables = new ArrayList<>();
tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql)); tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql));
String finalSql = null;
if (sqlQuery.isSupportWith()) { if (sqlQuery.isSupportWith()) {
EngineType engineType = EngineType engineType =
EngineType.fromString(queryStatement.getOntology().getDatabase().getType()); EngineType.fromString(queryStatement.getOntology().getDatabase().getType());
if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) { if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) {
String withSql = "with " + tables.stream() finalSql = "with " + tables.stream()
.map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight())) .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight()))
.collect(Collectors.joining(",")) + "\n" + ontologyQuerySql; .collect(Collectors.joining(",")) + "\n" + ontologyQuerySql;
queryStatement.setSql(withSql);
} else { } else {
List<String> withTableList = List<String> withTableList =
tables.stream().map(Pair::getLeft).collect(Collectors.toList()); tables.stream().map(Pair::getLeft).collect(Collectors.toList());
List<String> withSqlList = List<String> withSqlList =
tables.stream().map(Pair::getRight).collect(Collectors.toList()); tables.stream().map(Pair::getRight).collect(Collectors.toList());
String mergeSql = SqlMergeWithUtils.mergeWith(engineType, ontologyQuerySql, finalSql = SqlMergeWithUtils.mergeWith(engineType, ontologyQuerySql, withSqlList,
withSqlList, withTableList); withTableList);
queryStatement.setSql(mergeSql);
} }
} else { } else {
for (Pair<String, String> tb : tables) { for (Pair<String, String> tb : tables) {
ontologyQuerySql = StringUtils.replace(ontologyQuerySql, tb.getLeft(), finalSql = StringUtils.replace(ontologyQuerySql, tb.getLeft(),
"(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()), "(" + tb.getRight() + ") " + (sqlQuery.isWithAlias() ? "" : tb.getLeft()),
-1); -1);
} }
queryStatement.setSql(ontologyQuerySql);
} }
queryStatement.setSql(finalSql);
} }
} }

View File

@@ -1,13 +0,0 @@
package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
/**
* A query converter performs preprocessing work to the QueryStatement before parsing.
*/
public interface QueryConverter {
boolean accept(QueryStatement queryStatement);
void convert(QueryStatement queryStatement) throws Exception;
}

View File

@@ -14,16 +14,20 @@ import java.util.Objects;
@Component("DbDialectOptimizer") @Component("DbDialectOptimizer")
public class DbDialectOptimizer implements QueryOptimizer { public class DbDialectOptimizer implements QueryOptimizer {
@Override
public boolean accept(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema();
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
return Objects.nonNull(database) && Objects.nonNull(database.getType());
}
@Override @Override
public void rewrite(QueryStatement queryStatement) { public void rewrite(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema();
DatabaseResp database = semanticSchemaResp.getDatabaseResp(); DatabaseResp database = semanticSchemaResp.getDatabaseResp();
String sql = queryStatement.getSql(); String sql = queryStatement.getSql();
if (Objects.isNull(database) || Objects.isNull(database.getType())) { DbAdaptor engineAdaptor =
return; DbAdaptorFactory.getEngineAdaptor(database.getType().toLowerCase());
}
String type = database.getType();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase());
if (Objects.nonNull(engineAdaptor)) { if (Objects.nonNull(engineAdaptor)) {
String adaptedSql = engineAdaptor.rewriteSql(sql); String adaptedSql = engineAdaptor.rewriteSql(sql);
queryStatement.setSql(adaptedSql); queryStatement.setSql(adaptedSql);

View File

@@ -1,39 +0,0 @@
package com.tencent.supersonic.headless.core.translator.optimizer;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.StructQuery;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.Objects;
/** Remove the default metric added by the system when the query only has dimensions */
@Slf4j
@Component("DetailQueryOptimizer")
public class DetailQueryOptimizer implements QueryOptimizer {
@Override
public void rewrite(QueryStatement queryStatement) {
StructQuery structQuery = queryStatement.getStructQuery();
String sqlRaw = queryStatement.getSql().trim();
if (StringUtils.isEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");
}
log.debug("before handleNoMetric, sql:{}", sqlRaw);
// if (isDetailQuery(structQueryParam)) {
// if (!CollectionUtils.isEmpty(structQueryParam.getGroups())) {
// String sqlForm = "select %s from ( %s ) src_no_metric";
// String sql = String.format(sqlForm,
// structQueryParam.getGroups().stream().collect(Collectors.joining(",")),
// sqlRaw);
// queryStatement.setSql(sql);
// }
// }
log.debug("after handleNoMetric, sql:{}", queryStatement.getSql());
}
public boolean isDetailQuery(StructQuery structQuery) {
return Objects.nonNull(structQuery) && structQuery.getQueryType().isNativeAggQuery();
}
}

View File

@@ -7,5 +7,9 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
* derive the most efficient query. * derive the most efficient query.
*/ */
public interface QueryOptimizer { public interface QueryOptimizer {
boolean accept(QueryStatement queryStatement);
void rewrite(QueryStatement queryStatement); void rewrite(QueryStatement queryStatement);
} }

View File

@@ -9,10 +9,13 @@ import org.springframework.stereotype.Component;
@Component("ResultLimitOptimizer") @Component("ResultLimitOptimizer")
public class ResultLimitOptimizer implements QueryOptimizer { public class ResultLimitOptimizer implements QueryOptimizer {
@Override
public boolean accept(QueryStatement queryStatement) {
return !SqlSelectHelper.hasLimit(queryStatement.getSql());
}
@Override @Override
public void rewrite(QueryStatement queryStatement) { public void rewrite(QueryStatement queryStatement) {
if (!SqlSelectHelper.hasLimit(queryStatement.getSql())) { queryStatement.setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit());
queryStatement.setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit());
}
} }
} }

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.converter; package com.tencent.supersonic.headless.core.translator.parser;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
@@ -22,11 +22,11 @@ import java.util.stream.Collectors;
/** /**
* This converter appends default dimension values (if configured) to the where statement. * This parser appends default dimension values (if configured) to the where statement.
*/ */
@Slf4j @Slf4j
@Component("DefaultDimValueConverter") @Component("DefaultDimValueParser")
public class DefaultDimValueConverter implements QueryConverter { public class DefaultDimValueParser implements QueryParser {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
@@ -35,7 +35,7 @@ public class DefaultDimValueConverter implements QueryConverter {
} }
@Override @Override
public void convert(QueryStatement queryStatement) { public void parse(QueryStatement queryStatement) {
List<DimSchemaResp> dimensions = queryStatement.getOntology().getDimensions().stream() List<DimSchemaResp> dimensions = queryStatement.getOntology().getDimensions().stream()
.filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues()))
.collect(Collectors.toList()); .collect(Collectors.toList());

View File

@@ -1,8 +1,7 @@
package com.tencent.supersonic.headless.core.translator.converter; package com.tencent.supersonic.headless.core.translator.parser;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
@@ -13,15 +12,17 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.*; import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/** /**
* This converter replaces dimension bizName in the S2SQL with calculation expression (if * This parser replaces dimension bizName in the S2SQL with calculation expression (if configured).
* configured).
*/ */
@Component("DimExpressionConverter") @Component("DimExpressionParser")
@Slf4j @Slf4j
public class DimExpressionConverter implements QueryConverter { public class DimExpressionParser implements QueryParser {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
return Objects.nonNull(queryStatement.getSqlQuery()) return Objects.nonNull(queryStatement.getSqlQuery())
@@ -29,7 +30,7 @@ public class DimExpressionConverter implements QueryConverter {
} }
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void parse(QueryStatement queryStatement) throws Exception {
SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema(); SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema();
SqlQuery sqlQuery = queryStatement.getSqlQuery(); SqlQuery sqlQuery = queryStatement.getSqlQuery();
@@ -49,22 +50,11 @@ public class DimExpressionConverter implements QueryConverter {
Set<String> queryFields = ontologyQuery.getFields(); Set<String> queryFields = ontologyQuery.getFields();
log.debug("begin to generateDerivedMetric {} [{}]", queryDimensions); log.debug("begin to generateDerivedMetric {} [{}]", queryDimensions);
Set<String> allFields = new HashSet<>();
Map<String, Dimension> dimensionMap = new HashMap<>();
semanticSchema.getModelResps().forEach(modelResp -> {
allFields.addAll(modelResp.getFieldList());
if (modelResp.getModelDetail().getDimensions() != null) {
modelResp.getModelDetail().getDimensions()
.forEach(dimension -> dimensionMap.put(dimension.getBizName(), dimension));
}
});
Map<String, String> dim2Expr = new HashMap<>(); Map<String, String> dim2Expr = new HashMap<>();
for (DimSchemaResp queryDim : queryDimensions) { for (DimSchemaResp queryDim : queryDimensions) {
queryDim.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(queryDim.getExpr())); queryDim.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(queryDim.getExpr()));
dim2Expr.put(queryDim.getBizName(), queryDim.getExpr());
queryFields.addAll(queryDim.getFields()); queryFields.addAll(queryDim.getFields());
dim2Expr.put(queryDim.getBizName(), queryDim.getExpr());
} }
return dim2Expr; return dim2Expr;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.converter; package com.tencent.supersonic.headless.core.translator.parser;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
@@ -17,11 +17,11 @@ import org.springframework.util.CollectionUtils;
import java.util.*; import java.util.*;
/** /**
* This converter replaces metric bizName in the S2SQL with calculation expression (if configured). * This parser replaces metric bizName in the S2SQL with calculation expression (if configured).
*/ */
@Component("MetricExpressionConverter") @Component("MetricExpressionParser")
@Slf4j @Slf4j
public class MetricExpressionConverter implements QueryConverter { public class MetricExpressionParser implements QueryParser {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
return Objects.nonNull(queryStatement.getSqlQuery()) return Objects.nonNull(queryStatement.getSqlQuery())
@@ -29,7 +29,7 @@ public class MetricExpressionConverter implements QueryConverter {
} }
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void parse(QueryStatement queryStatement) throws Exception {
SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema(); SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema();
SqlQuery sqlQuery = queryStatement.getSqlQuery(); SqlQuery sqlQuery = queryStatement.getSqlQuery();
@@ -64,10 +64,10 @@ public class MetricExpressionConverter implements QueryConverter {
Map<String, String> metric2Expr = new HashMap<>(); Map<String, String> metric2Expr = new HashMap<>();
for (MetricSchemaResp queryMetric : queryMetrics) { for (MetricSchemaResp queryMetric : queryMetrics) {
String fieldExpr = buildFieldExpr(allMetrics, allMeasures, queryMetric.getExpr(), String fieldExpr = buildFieldExpr(allMetrics, allMeasures, queryMetric.getExpr(),
queryMetric.getMetricDefineType(), visitedMetrics, queryFields); queryMetric.getMetricDefineType(), visitedMetrics);
// add all fields referenced in the expression // add all fields referenced in the expression
queryMetric.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(fieldExpr)); queryMetric.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(fieldExpr));
log.debug("derived metric {}->{}", queryMetric.getBizName(), fieldExpr); queryFields.addAll(queryMetric.getFields());
if (!queryMetric.getBizName().equals(fieldExpr)) { if (!queryMetric.getBizName().equals(fieldExpr)) {
metric2Expr.put(queryMetric.getBizName(), fieldExpr); metric2Expr.put(queryMetric.getBizName(), fieldExpr);
} }
@@ -78,8 +78,7 @@ public class MetricExpressionConverter implements QueryConverter {
private String buildFieldExpr(final List<MetricSchemaResp> metricResps, private String buildFieldExpr(final List<MetricSchemaResp> metricResps,
final Map<String, Measure> allMeasures, final String metricExpr, final Map<String, Measure> allMeasures, final String metricExpr,
final MetricDefineType metricDefineType, Map<String, String> visitedMetric, final MetricDefineType metricDefineType, Map<String, String> visitedMetric) {
Set<String> queryFields) {
Set<String> fields = SqlSelectHelper.getFieldsFromExpr(metricExpr); Set<String> fields = SqlSelectHelper.getFieldsFromExpr(metricExpr);
if (!CollectionUtils.isEmpty(fields)) { if (!CollectionUtils.isEmpty(fields)) {
Map<String, String> replace = new HashMap<>(); Map<String, String> replace = new HashMap<>();
@@ -97,8 +96,7 @@ public class MetricExpressionConverter implements QueryConverter {
replace.put(field, replace.put(field,
buildFieldExpr(metricResps, allMeasures, buildFieldExpr(metricResps, allMeasures,
metricItem.get().getExpr(), metricItem.get().getExpr(),
metricItem.get().getMetricDefineType(), visitedMetric, metricItem.get().getMetricDefineType(), visitedMetric));
queryFields));
visitedMetric.put(field, replace.get(field)); visitedMetric.put(field, replace.get(field));
} }
break; break;
@@ -111,19 +109,16 @@ public class MetricExpressionConverter implements QueryConverter {
expr = String.format("%s (%s)", measure.getAgg(), metricExpr); expr = String.format("%s (%s)", measure.getAgg(), metricExpr);
} }
replace.put(field, expr); replace.put(field, expr);
queryFields.add(field);
} }
break; break;
case FIELD: case FIELD:
queryFields.add(field);
break;
default: default:
break; break;
} }
} }
if (!CollectionUtils.isEmpty(replace)) { if (!CollectionUtils.isEmpty(replace)) {
String expr = SqlReplaceHelper.replaceExpression(metricExpr, replace); String expr = SqlReplaceHelper.replaceExpression(metricExpr, replace);
log.debug("derived measure {}->{}", metricExpr, expr); log.debug("derived metric {}->{}", metricExpr, expr);
return expr; return expr;
} }
} }

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.converter; package com.tencent.supersonic.headless.core.translator.parser;
import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
@@ -21,9 +21,9 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Component("MetricRatioConverter") @Component("MetricRatioParser")
@Slf4j @Slf4j
public class MetricRatioConverter implements QueryConverter { public class MetricRatioParser implements QueryParser {
public interface EngineSql { public interface EngineSql {
@@ -58,7 +58,7 @@ public class MetricRatioConverter implements QueryConverter {
} }
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void parse(QueryStatement queryStatement) throws Exception {
DatabaseResp database = queryStatement.getOntology().getDatabase(); DatabaseResp database = queryStatement.getOntology().getDatabase();
generateRatioSql(queryStatement, EngineType.fromString(database.getType()), generateRatioSql(queryStatement, EngineType.fromString(database.getType()),
database.getVersion()); database.getVersion());

View File

@@ -1,23 +1,30 @@
package com.tencent.supersonic.headless.core.translator.parser.calcite; package com.tencent.supersonic.headless.core.translator.parser;
import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.pojo.Ontology;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.parser.QueryParser; import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.parser.calcite.SqlBuilder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Component("CalciteQueryParser") import java.util.Objects;
/**
* This parser generates inner sql statement for the ontology query, which would be selected by the
* parsed sql query.
*/
@Component("OntologyQueryParser")
@Slf4j @Slf4j
public class CalciteQueryParser implements QueryParser { public class OntologyQueryParser implements QueryParser {
@Override
public boolean accept(QueryStatement queryStatement) {
return Objects.nonNull(queryStatement.getOntologyQuery());
}
@Override @Override
public void parse(QueryStatement queryStatement) throws Exception { public void parse(QueryStatement queryStatement) throws Exception {
Ontology ontology = queryStatement.getOntology(); Ontology ontology = queryStatement.getOntology();
if (ontology == null) {
queryStatement.setErrMsg("No ontology could be found");
return;
}
S2CalciteSchema semanticSchema = S2CalciteSchema.builder() S2CalciteSchema semanticSchema = S2CalciteSchema.builder()
.schemaKey("DATASET_" + queryStatement.getDataSetId()).ontology(ontology) .schemaKey("DATASET_" + queryStatement.getDataSetId()).ontology(ontology)
.runtimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) .runtimeOptions(RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime())

View File

@@ -6,5 +6,8 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
* A query parser generates physical SQL for the QueryStatement. * A query parser generates physical SQL for the QueryStatement.
*/ */
public interface QueryParser { public interface QueryParser {
boolean accept(QueryStatement queryStatement);
void parse(QueryStatement queryStatement) throws Exception; void parse(QueryStatement queryStatement) throws Exception;
} }

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.parser.calcite; package com.tencent.supersonic.headless.core.translator.parser;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.converter; package com.tencent.supersonic.headless.core.translator.parser;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
@@ -23,12 +23,12 @@ import java.util.Map;
import java.util.Objects; import java.util.Objects;
/** /**
* This converter rewrites S2SQL including conversion from metric/dimension name to bizName and * This parser rewrites S2SQL including conversion from metric/dimension name to bizName and build
* build ontology query in preparation for generation of physical SQL. * ontology query in preparation for generation of physical SQL.
*/ */
@Component("SqlQueryConverter") @Component("SqlQueryParser")
@Slf4j @Slf4j
public class SqlQueryConverter implements QueryConverter { public class SqlQueryParser implements QueryParser {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
@@ -36,7 +36,7 @@ public class SqlQueryConverter implements QueryConverter {
} }
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void parse(QueryStatement queryStatement) throws Exception {
convertNameToBizName(queryStatement); convertNameToBizName(queryStatement);
rewriteOrderBy(queryStatement); rewriteOrderBy(queryStatement);

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.converter; package com.tencent.supersonic.headless.core.translator.parser;
import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
@@ -13,8 +13,8 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
@Slf4j @Slf4j
@Component("SqlVariableConverter") @Component("SqlVariableParser")
public class SqlVariableConverter implements QueryConverter { public class SqlVariableParser implements QueryParser {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
@@ -22,7 +22,7 @@ public class SqlVariableConverter implements QueryConverter {
} }
@Override @Override
public void convert(QueryStatement queryStatement) { public void parse(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema(); SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchema();
List<ModelResp> modelResps = semanticSchemaResp.getModelResps(); List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
if (CollectionUtils.isEmpty(modelResps)) { if (CollectionUtils.isEmpty(modelResps)) {

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.translator.converter; package com.tencent.supersonic.headless.core.translator.parser;
import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
@@ -13,12 +13,12 @@ import org.springframework.stereotype.Component;
import java.util.Objects; import java.util.Objects;
/** /**
* This converter converts struct semantic query into sql query by generating S2SQL based on * This parser converts struct semantic query into sql query by generating S2SQL based on structured
* structured semantic information. * semantic information.
*/ */
@Component("StructQueryConverter") @Component("StructQueryParser")
@Slf4j @Slf4j
public class StructQueryConverter implements QueryConverter { public class StructQueryParser implements QueryParser {
@Override @Override
public boolean accept(QueryStatement queryStatement) { public boolean accept(QueryStatement queryStatement) {
@@ -26,7 +26,7 @@ public class StructQueryConverter implements QueryConverter {
} }
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void parse(QueryStatement queryStatement) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
StructQuery structQuery = queryStatement.getStructQuery(); StructQuery structQuery = queryStatement.getStructQuery();

View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.core.pojo.JoinRelation; import com.tencent.supersonic.headless.core.pojo.JoinRelation;
import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.pojo.Ontology;
import com.tencent.supersonic.headless.core.translator.parser.RuntimeOptions;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.Schema;

View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.cache.QueryCache;
import com.tencent.supersonic.headless.core.executor.QueryAccelerator; import com.tencent.supersonic.headless.core.executor.QueryAccelerator;
import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.translator.converter.QueryConverter;
import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.translator.parser.QueryParser; import com.tencent.supersonic.headless.core.translator.parser.QueryParser;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -20,15 +19,13 @@ import java.util.stream.Collectors;
@Slf4j @Slf4j
public class ComponentFactory { public class ComponentFactory {
private static List<QueryConverter> queryConverters = new ArrayList<>();
private static Map<String, QueryOptimizer> queryOptimizers = new HashMap<>(); private static Map<String, QueryOptimizer> queryOptimizers = new HashMap<>();
private static List<QueryExecutor> queryExecutors = new ArrayList<>(); private static List<QueryExecutor> queryExecutors = new ArrayList<>();
private static List<QueryAccelerator> queryAccelerators = new ArrayList<>(); private static List<QueryAccelerator> queryAccelerators = new ArrayList<>();
private static QueryParser queryParser; private static List<QueryParser> queryParsers = new ArrayList<>();
private static QueryCache queryCache; private static QueryCache queryCache;
static { static {
initQueryConverter();
initQueryOptimizer(); initQueryOptimizer();
initQueryExecutors(); initQueryExecutors();
initQueryAccelerators(); initQueryAccelerators();
@@ -36,13 +33,6 @@ public class ComponentFactory {
initQueryCache(); initQueryCache();
} }
public static List<QueryConverter> getQueryConverters() {
if (queryConverters.isEmpty()) {
initQueryConverter();
}
return queryConverters;
}
public static List<QueryOptimizer> getQueryOptimizers() { public static List<QueryOptimizer> getQueryOptimizers() {
if (queryOptimizers.isEmpty()) { if (queryOptimizers.isEmpty()) {
initQueryOptimizer(); initQueryOptimizer();
@@ -64,11 +54,11 @@ public class ComponentFactory {
return queryAccelerators; return queryAccelerators;
} }
public static QueryParser getQueryParser() { public static List<QueryParser> getQueryParsers() {
if (queryParser == null) { if (queryParsers == null) {
initQueryParser(); initQueryParser();
} }
return queryParser; return queryParsers;
} }
public static QueryCache getQueryCache() { public static QueryCache getQueryCache() {
@@ -103,12 +93,8 @@ public class ComponentFactory {
init(QueryAccelerator.class, queryAccelerators); init(QueryAccelerator.class, queryAccelerators);
} }
private static void initQueryConverter() {
init(QueryConverter.class, queryConverters);
}
private static void initQueryParser() { private static void initQueryParser() {
queryParser = init(QueryParser.class); init(QueryParser.class, queryParsers);
} }
private static void initQueryCache() { private static void initQueryCache() {

View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.parser.aggregate;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser; import com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.testng.Assert; import org.testng.Assert;
@@ -316,7 +316,7 @@ public class CalciteSqlParserTest {
+ " \"createdAt\": 1711367511146,\n" + " \"createdAt\": 1711367511146,\n"
+ " \"updatedAt\": 1711367511146\n" + " }\n" + " }\n" + "}"; + " \"updatedAt\": 1711367511146\n" + " }\n" + " }\n" + "}";
QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class); QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class);
CalciteQueryParser calciteSqlParser = new CalciteQueryParser(); OntologyQueryParser calciteSqlParser = new OntologyQueryParser();
calciteSqlParser.parse(queryStatement); calciteSqlParser.parse(queryStatement);
Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""), Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""),
"SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" + "FROM" + "`s2_pv_uv_statis`" "SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" + "FROM" + "`s2_pv_uv_statis`"

View File

@@ -25,21 +25,19 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\
### headless-core SPIs ### headless-core SPIs
com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\
com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\
com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\
com.tencent.supersonic.headless.core.translator.converter.MetricExpressionConverter,\
com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter
com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\ com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\
com.tencent.supersonic.headless.core.translator.optimizer.DetailQueryOptimizer,\
com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\
com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer
com.tencent.supersonic.headless.core.translator.parser.QueryParser=\ com.tencent.supersonic.headless.core.translator.parser.QueryParser=\
com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser com.tencent.supersonic.headless.core.translator.parser.SqlVariableParser,\
com.tencent.supersonic.headless.core.translator.parser.StructQueryParser,\
com.tencent.supersonic.headless.core.translator.parser.SqlQueryParser,\
com.tencent.supersonic.headless.core.translator.parser.DefaultDimValueParser,\
com.tencent.supersonic.headless.core.translator.parser.DimExpressionParser,\
com.tencent.supersonic.headless.core.translator.parser.MetricExpressionParser,\
com.tencent.supersonic.headless.core.translator.parser.MetricRatioParser,\
com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser
com.tencent.supersonic.headless.core.executor.QueryExecutor=\ com.tencent.supersonic.headless.core.executor.QueryExecutor=\
com.tencent.supersonic.headless.core.executor.JdbcExecutor com.tencent.supersonic.headless.core.executor.JdbcExecutor

View File

@@ -25,22 +25,19 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\
### headless-core SPIs ### headless-core SPIs
com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\
com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\
com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\
com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\
com.tencent.supersonic.headless.core.translator.converter.DimExpressionConverter,\
com.tencent.supersonic.headless.core.translator.converter.MetricExpressionConverter,\
com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter
com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\ com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer=\
com.tencent.supersonic.headless.core.translator.optimizer.DetailQueryOptimizer,\
com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\ com.tencent.supersonic.headless.core.translator.optimizer.DbDialectOptimizer,\
com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer com.tencent.supersonic.headless.core.translator.optimizer.ResultLimitOptimizer
com.tencent.supersonic.headless.core.translator.parser.QueryParser=\ com.tencent.supersonic.headless.core.translator.parser.QueryParser=\
com.tencent.supersonic.headless.core.translator.parser.calcite.CalciteQueryParser com.tencent.supersonic.headless.core.translator.parser.SqlVariableParser,\
com.tencent.supersonic.headless.core.translator.parser.StructQueryParser,\
com.tencent.supersonic.headless.core.translator.parser.SqlQueryParser,\
com.tencent.supersonic.headless.core.translator.parser.DefaultDimValueParser,\
com.tencent.supersonic.headless.core.translator.parser.DimExpressionParser,\
com.tencent.supersonic.headless.core.translator.parser.MetricExpressionParser,\
com.tencent.supersonic.headless.core.translator.parser.MetricRatioParser,\
com.tencent.supersonic.headless.core.translator.parser.OntologyQueryParser
com.tencent.supersonic.headless.core.executor.QueryExecutor=\ com.tencent.supersonic.headless.core.executor.QueryExecutor=\
com.tencent.supersonic.headless.core.executor.JdbcExecutor com.tencent.supersonic.headless.core.executor.JdbcExecutor