topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 12:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Clean code logic of headless translator.

Browse Source
This commit is contained in:
jerryjzhang
2024-11-24 19:07:56 +08:00
parent c22e3ef2e8
commit 860fd5d299
45 changed files with 795 additions and 1058 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java
View File

@@ -528,7 +528,7 @@ public class SqlReplaceHelper {
}
}
private static Select replaceAggAliasOrderItem(Select selectStatement) {
private static Select replaceAggAliasOrderbyField(Select selectStatement) {
if (selectStatement instanceof PlainSelect) {
PlainSelect plainSelect = (PlainSelect) selectStatement;
if (Objects.nonNull(plainSelect.getOrderByElements())) {
@@ -564,15 +564,15 @@ public class SqlReplaceHelper {
if (plainSelect.getFromItem() instanceof ParenthesedSelect) {
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem();
parenthesedSelect
.setSelect(replaceAggAliasOrderItem(parenthesedSelect.getSelect()));
.setSelect(replaceAggAliasOrderbyField(parenthesedSelect.getSelect()));
}
return selectStatement;
}
return selectStatement;
}
public static String replaceAggAliasOrderItem(String sql) {
Select selectStatement = replaceAggAliasOrderItem(SqlSelectHelper.getSelect(sql));
public static String replaceAggAliasOrderbyField(String sql) {
Select selectStatement = replaceAggAliasOrderbyField(SqlSelectHelper.getSelect(sql));
return selectStatement.toString();
}

12
common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java
View File

@@ -4,6 +4,7 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -32,14 +33,9 @@ import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT;
@Data
public class DateModeUtils {
@Value("${s2.query.parameter.sys.date:sys_imp_date}")
private String sysDateCol;
@Value("${s2.query.parameter.sys.month:sys_imp_month}")
private String sysDateMonthCol;
@Value("${s2.query.parameter.sys.month:sys_imp_week}")
private String sysDateWeekCol;
private final String sysDateCol = TimeDimensionEnum.DAY.getName();
private final String sysDateMonthCol = TimeDimensionEnum.MONTH.getName();
private final String sysDateWeekCol = TimeDimensionEnum.WEEK.getName();
@Value("${s2.query.parameter.sys.zipper.begin:start_}")
private String sysZipperDateColBegin;

4
common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java
View File

@@ -325,10 +325,10 @@ class SqlReplaceHelperTest {
}
@Test
void testReplaceAggAliasOrderItem() {
void testReplaceAggAliasOrderbyField() {
String sql = "SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数 "
+ "GROUP BY 部门 ORDER BY SUM(访问次数) DESC LIMIT 10) AS top10";
String replaceSql = SqlReplaceHelper.replaceAggAliasOrderItem(sql);
String replaceSql = SqlReplaceHelper.replaceAggAliasOrderbyField(sql);
Assert.assertEquals(
"SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数 "
+ "GROUP BY 部门 ORDER BY 2 DESC LIMIT 10) AS top10",

24
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java
View File

@@ -1,24 +0,0 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import lombok.Data;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Data
public class ParseSqlReq {
private Map<String, String> variables;
private String sql = "";
private List<MetricTable> tables;
private boolean supportWith = true;
private boolean withAlias = true;
public Map<String, String> getVariables() {
if (variables == null) {
variables = new HashMap<>();
}
return variables;
}
}

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java
View File

@@ -34,12 +34,11 @@ public class QueryFilter implements Serializable {
QueryFilter that = (QueryFilter) o;
return Objects.equal(bizName, that.bizName) && Objects.equal(name, that.name)
&& operator == that.operator && Objects.equal(value, that.value)
&& Objects.equal(elementID, that.elementID)
&& Objects.equal(function, that.function);
}
@Override
public int hashCode() {
return Objects.hashCode(bizName, name, operator, value, elementID, function);
return Objects.hashCode(bizName, name, operator, value, function);
}
}

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java
View File

@@ -1,14 +0,0 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import lombok.Data;
import java.util.List;
@Data
public class DataSetQueryParam {
private String sql = "";
private List<MetricTable> tables;
private boolean supportWith = true;
private boolean withAlias = true;
}

17
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java
View File

@@ -1,17 +0,0 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import lombok.Data;
import java.util.List;
@Data
public class MetricQueryParam {
private List<String> metrics;
private List<String> dimensions;
private String where;
private Long limit;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
}

22
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -1,34 +1,25 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import java.util.List;
@Data
public class QueryStatement {
private Long dataSetId;
private List<Long> modelIds;
private String sql;
private String errMsg;
private QueryParam queryParam;
private MetricQueryParam metricQueryParam;
private DataSetQueryParam dataSetQueryParam;
private StructQueryParam structQueryParam;
private SqlQueryParam sqlQueryParam;
private OntologyQueryParam ontologyQueryParam;
private Integer status = 0;
private Boolean isS2SQL = false;
private List<ImmutablePair<String, String>> timeRanges;
private Boolean enableOptimize = true;
private Triple<String, String, String> minMaxTime;
private String dataSetSql;
private String dataSetAlias;
private String dataSetSimplifySql;
private Boolean enableLimitWrapper = false;
private Ontology ontology;
private SemanticSchemaResp semanticSchemaResp;
private Integer limit = 1000;
@@ -41,9 +32,4 @@ public class QueryStatement {
public boolean isTranslated() {
return isTranslated != null && isTranslated && isOk();
}
public QueryStatement error(String msg) {
this.setErrMsg(msg);
return this;
}
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQueryParam.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.headless.core.pojo;
import lombok.Data;
@Data
public class SqlQueryParam {
private String sql;
private String table;
private boolean supportWith = true;
private boolean withAlias = true;
private String simplifiedSql;
}

21
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQueryParam.java
View File

@@ -1,21 +1,18 @@
package com.tencent.supersonic.headless.api.pojo;
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Param;
import lombok.Data;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@Data
public class QueryParam {
// struct
public class StructQueryParam {
private List<String> groups = new ArrayList();
private List<Aggregator> aggregators = new ArrayList();
private List<Order> orders = new ArrayList();
@@ -24,17 +21,5 @@ public class QueryParam {
private DateConf dateInfo;
private Long limit = 2000L;
private QueryType queryType;
private String s2SQL;
private String correctS2SQL;
private Long dataSetId;
private String dataSetName;
private Set<Long> modelIds = new HashSet<>();
private List<Param> params = new ArrayList<>();
// metric
private List<String> metrics = new ArrayList();
private List<String> dimensions;
private String where;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
}

524
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java
View File

@@ -1,507 +1,97 @@
package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.converter.QueryConverter;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
@Slf4j
public class DefaultSemanticTranslator implements SemanticTranslator {
@Autowired
private SqlGenerateUtils sqlGenerateUtils;
public void translate(QueryStatement queryStatement) {
if (queryStatement.isTranslated()) {
return;
}
try {
preprocess(queryStatement);
parse(queryStatement);
optimize(queryStatement);
for (QueryConverter converter : ComponentFactory.getQueryConverters()) {
if (converter.accept(queryStatement)) {
log.debug("QueryConverter accept [{}]", converter.getClass().getName());
converter.convert(queryStatement);
}
}
doOntologyParse(queryStatement);
if (StringUtils.isNotBlank(queryStatement.getSqlQueryParam().getSimplifiedSql())) {
queryStatement.setSql(queryStatement.getSqlQueryParam().getSimplifiedSql());
}
if (StringUtils.isBlank(queryStatement.getSql())) {
throw new RuntimeException("parse exception: " + queryStatement.getErrMsg());
}
if (!SqlSelectHelper.hasLimit(queryStatement.getSql())) {
queryStatement
.setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit());
}
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement);
}
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("Failed to translate semantic query [{}]", e);
}
}
private void parse(QueryStatement queryStatement) throws Exception {
QueryParam queryParam = queryStatement.getQueryParam();
if (Objects.isNull(queryStatement.getDataSetQueryParam())) {
queryStatement.setDataSetQueryParam(new DataSetQueryParam());
}
if (Objects.isNull(queryStatement.getMetricQueryParam())) {
queryStatement.setMetricQueryParam(new MetricQueryParam());
private void doOntologyParse(QueryStatement queryStatement) throws Exception {
OntologyQueryParam ontologyQueryParam = queryStatement.getOntologyQueryParam();
SqlQueryParam sqlQueryParam = queryStatement.getSqlQueryParam();
log.info("parse with ontology: [{}]", ontologyQueryParam);
ComponentFactory.getQueryParser().parse(queryStatement);
if (!queryStatement.isOk()) {
throw new Exception(String.format("parse table [%s] error [%s]",
sqlQueryParam.getTable(), queryStatement.getErrMsg()));
}
log.debug("SemanticConverter before [{}]", queryParam);
for (QueryConverter headlessConverter : ComponentFactory.getQueryConverters()) {
if (headlessConverter.accept(queryStatement)) {
log.debug("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.convert(queryStatement);
List<Pair<String, String>> tables = new ArrayList<>();
tables.add(Pair.of(sqlQueryParam.getTable(), queryStatement.getSql()));
if (sqlQueryParam.isSupportWith()) {
EngineType engineType =
EngineType.fromString(queryStatement.getOntology().getDatabase().getType());
if (!SqlMergeWithUtils.hasWith(engineType, sqlQueryParam.getSql())) {
String withSql = "with " + tables.stream()
.map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight()))
.collect(Collectors.joining(",")) + "\n" + sqlQueryParam.getSql();
queryStatement.setSql(withSql);
} else {
List<String> parentTableList =
tables.stream().map(Pair::getLeft).collect(Collectors.toList());
List<String> parentSqlList =
tables.stream().map(Pair::getRight).collect(Collectors.toList());
String mergeSql = SqlMergeWithUtils.mergeWith(engineType, sqlQueryParam.getSql(),
parentSqlList, parentTableList);
queryStatement.setSql(mergeSql);
}
}
log.debug("SemanticConverter after {} {} {}", queryParam,
queryStatement.getDataSetQueryParam(), queryStatement.getMetricQueryParam());
if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) {
doParse(queryStatement.getDataSetQueryParam(), queryStatement);
} else {
queryStatement.getMetricQueryParam()
.setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
doParse(queryStatement,
AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery()));
}
if (StringUtils.isEmpty(queryStatement.getSql())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
}
if (StringUtils.isNotBlank(queryStatement.getSql())
&& !SqlSelectHelper.hasLimit(queryStatement.getSql())) {
String querySql =
queryStatement.getSql() + " limit " + queryStatement.getLimit().toString();
queryStatement.setSql(querySql);
}
}
private QueryStatement doParse(DataSetQueryParam dataSetQueryParam,
QueryStatement queryStatement) {
log.info("parse dataSetQuery [{}] ", dataSetQueryParam);
Ontology ontology = queryStatement.getOntology();
EngineType engineType = EngineType.fromString(ontology.getDatabase().getType());
try {
if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) {
List<String[]> tables = new ArrayList<>();
boolean isSingleTable = dataSetQueryParam.getTables().size() == 1;
for (MetricTable metricTable : dataSetQueryParam.getTables()) {
QueryStatement tableSql = parserSql(metricTable, isSingleTable,
dataSetQueryParam, queryStatement);
if (isSingleTable && StringUtils.isNotBlank(tableSql.getDataSetSimplifySql())) {
queryStatement.setSql(tableSql.getDataSetSimplifySql());
queryStatement.setDataSetQueryParam(dataSetQueryParam);
return queryStatement;
}
tables.add(new String[] {metricTable.getAlias(), tableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql;
if (dataSetQueryParam.isSupportWith()) {
if (!SqlMergeWithUtils.hasWith(engineType, dataSetQueryParam.getSql())) {
sql = "with "
+ tables.stream()
.map(t -> String.format("%s as (%s)", t[0], t[1]))
.collect(Collectors.joining(","))
+ "\n" + dataSetQueryParam.getSql();
} else {
List<String> parentWithNameList = tables.stream().map(table -> table[0])
.collect(Collectors.toList());
List<String> parentSqlList = tables.stream().map(table -> table[1])
.collect(Collectors.toList());
sql = SqlMergeWithUtils.mergeWith(engineType,
dataSetQueryParam.getSql(), parentSqlList, parentWithNameList);
}
} else {
sql = dataSetQueryParam.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0], "(" + tb[1] + ") "
+ (dataSetQueryParam.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setDataSetQueryParam(dataSetQueryParam);
return queryStatement;
}
String dsSql = sqlQueryParam.getSql();
for (Pair<String, String> tb : tables) {
dsSql = StringUtils.replace(dsSql, tb.getLeft(), "(" + tb.getRight() + ") "
+ (sqlQueryParam.isWithAlias() ? "" : tb.getLeft()), -1);
}
} catch (Exception e) {
log.error("physicalSql error {}", e);
queryStatement.setErrMsg(e.getMessage());
}
return queryStatement;
}
private QueryStatement doParse(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
log.info("parse metricQuery [{}] isAgg [{}]", metricQueryParam, isAgg);
try {
ComponentFactory.getQueryParser().parse(queryStatement, isAgg);
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e);
}
return queryStatement;
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable,
DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) throws Exception {
MetricQueryParam metricQueryParam = new MetricQueryParam();
metricQueryParam.setMetrics(metricTable.getMetrics());
metricQueryParam.setDimensions(metricTable.getDimensions());
metricQueryParam.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricQueryParam.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false);
tableSql.setMetricQueryParam(metricQueryParam);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setDataSetId(queryStatement.getDataSetId());
tableSql.setOntology(queryStatement.getOntology());
if (isSingleMetricTable) {
tableSql.setDataSetSql(dataSetQueryParam.getSql());
tableSql.setDataSetAlias(metricTable.getAlias());
}
tableSql = doParse(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {
throw new Exception(String.format("parser table [%s] error [%s]",
metricTable.getAlias(), tableSql.getErrMsg()));
}
return tableSql;
}
private void optimize(QueryStatement queryStatement) {
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement);
queryStatement.setSql(dsSql);
}
}
private void preprocess(QueryStatement queryStatement) {
if (StringUtils.isBlank(queryStatement.getSql())) {
return;
}
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
convertNameToBizName(queryStatement);
rewriteFunction(queryStatement);
queryStatement.setSql(SqlRemoveHelper.removeUnderscores(queryStatement.getSql()));
String tableName = SqlSelectHelper.getTableName(queryStatement.getSql());
if (StringUtils.isEmpty(tableName)) {
return;
}
// correct order item is same as agg alias
String reqSql = queryStatement.getSql();
queryStatement.setSql(SqlReplaceHelper.replaceAggAliasOrderItem(queryStatement.getSql()));
log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, queryStatement.getSql());
// 5.build MetricTables
List<String> allFields = SqlSelectHelper.getAllSelectFields(queryStatement.getSql());
List<MetricSchemaResp> metricSchemas = getMetrics(semanticSchemaResp, allFields);
List<String> metrics =
metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList());
Set<String> dimensions = getDimensions(semanticSchemaResp, allFields);
QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable();
metricTable.getMetrics().addAll(metrics);
metricTable.getDimensions().addAll(dimensions);
metricTable.setAlias(tableName.toLowerCase());
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
getDefaultModel(semanticSchemaResp, metricTable.getDimensions())));
} else {
queryStructReq.getAggregators()
.addAll(metricTable.getMetrics().stream()
.map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN))
.collect(Collectors.toList()));
}
AggOption aggOption = getAggOption(queryStatement, metricSchemas);
metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
// 6.build ParseSqlReq
DataSetQueryParam datasetQueryParam = new DataSetQueryParam();
datasetQueryParam.setTables(tables);
datasetQueryParam.setSql(queryStatement.getSql());
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
database.getVersion())) {
datasetQueryParam.setSupportWith(false);
datasetQueryParam.setWithAlias(false);
}
// 7. do deriveMetric
generateDerivedMetric(semanticSchemaResp, aggOption, datasetQueryParam);
// 8.physicalSql by ParseSqlReq
// queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryStatement.getSql()));
queryStructReq.setDataSetId(queryStatement.getDataSetId());
queryStructReq.setQueryType(getQueryType(aggOption));
log.debug("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryParam queryParam = new QueryParam();
BeanUtils.copyProperties(queryStructReq, queryParam);
queryStatement.setQueryParam(queryParam);
queryStatement.setDataSetQueryParam(datasetQueryParam);
// queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
}
private AggOption getAggOption(QueryStatement queryStatement,
List<MetricSchemaResp> metricSchemas) {
String sql = queryStatement.getSql();
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql)
&& !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) {
log.debug("getAggOption simple sql set to DEFAULT");
return AggOption.DEFAULT;
}
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql)
|| SqlSelectFunctionHelper.hasFunction(sql, "count")
|| SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
return AggOption.OUTER;
}
// if (queryStatement.isInnerLayerNative()) {
// return AggOption.NATIVE;
// }
if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql)
|| SqlSelectHelper.hasGroupBy(sql)) {
return AggOption.OUTER;
}
long defaultAggNullCnt = metricSchemas.stream().filter(
m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg()))
.count();
if (defaultAggNullCnt > 0) {
log.debug("getAggOption find null defaultAgg metric set to NATIVE");
return AggOption.OUTER;
}
return AggOption.DEFAULT;
}
private void convertNameToBizName(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp);
String sql = queryStatement.getSql();
log.debug("dataSetId:{},convert name to bizName before:{}", queryStatement.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceSqlByPositions(sql);
log.debug("replaceSqlByPositions:{}", sql);
sql = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
log.debug("dataSetId:{},convert name to bizName after:{}", queryStatement.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceTable(sql,
Constants.TABLE_PREFIX + queryStatement.getDataSetId());
log.debug("replaceTableName after:{}", sql);
queryStatement.setSql(sql);
}
private Set<String> getDimensions(SemanticSchemaResp semanticSchemaResp,
List<String> allFields) {
Map<String, String> dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(),
SchemaItem::getBizName, (k1, k2) -> k1));
dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.DAY.getName());
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase()))
.collect(Collectors.toSet());
}
private List<MetricSchemaResp> getMetrics(SemanticSchemaResp semanticSchemaResp,
List<String> allFields) {
Map<String, MetricSchemaResp> metricLowerToNameMap =
semanticSchemaResp.getMetrics().stream().collect(Collectors
.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry));
return allFields.stream()
.filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase()))
.collect(Collectors.toList());
}
private void rewriteFunction(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
}
String type = database.getType();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase());
if (Objects.nonNull(engineAdaptor)) {
String functionNameCorrector =
engineAdaptor.functionNameCorrector(queryStatement.getSql());
queryStatement.setSql(functionNameCorrector);
}
}
protected Map<String, String> getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) {
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = semanticSchemaResp.getDimensions().stream().flatMap(
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
Map<String, String> metricResults = semanticSchemaResp.getMetrics().stream().flatMap(
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
dimensionResults.putAll(metricResults);
return dimensionResults;
}
private Stream<Pair<String, String>> getPairStream(String aliasStr, String name,
String bizName) {
Set<Pair<String, String>> elements = new HashSet<>();
elements.add(Pair.of(name, bizName));
if (StringUtils.isNotBlank(aliasStr)) {
List<String> aliasList = SchemaItem.getAliasList(aliasStr);
for (String alias : aliasList) {
elements.add(Pair.of(alias, bizName));
}
}
return elements.stream();
}
private QueryType getQueryType(AggOption aggOption) {
boolean isAgg = AggOption.isAgg(aggOption);
QueryType queryType = QueryType.DETAIL;
if (isAgg) {
queryType = QueryType.AGGREGATE;
}
return queryType;
}
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
DataSetQueryParam viewQueryParam) {
String sql = viewQueryParam.getSql();
for (MetricTable metricTable : viewQueryParam.getTables()) {
Set<String> measures = new HashSet<>();
Map<String, String> replaces = generateDerivedMetric(semanticSchemaResp, aggOption,
metricTable.getMetrics(), metricTable.getDimensions(), measures);
if (!CollectionUtils.isEmpty(replaces)) {
// metricTable sql use measures replace metric
sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces);
metricTable.setAggOption(AggOption.NATIVE);
// metricTable use measures replace metric
if (!CollectionUtils.isEmpty(measures)) {
metricTable.setMetrics(new ArrayList<>(measures));
} else {
// empty measure , fill default
metricTable.setMetrics(new ArrayList<>());
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
getDefaultModel(semanticSchemaResp, metricTable.getDimensions())));
}
}
}
viewQueryParam.setSql(sql);
}
private Map<String, String> generateDerivedMetric(SemanticSchemaResp semanticSchemaResp,
AggOption aggOption, List<String> metrics, List<String> dimensions,
Set<String> measures) {
Map<String, String> result = new HashMap<>();
List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
// Check if any metric is derived
boolean hasDerivedMetrics =
metricResps.stream().anyMatch(m -> metrics.contains(m.getBizName()) && MetricType
.isDerived(m.getMetricDefineType(), m.getMetricDefineByMeasureParams()));
if (!hasDerivedMetrics) {
return result;
}
log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics);
Set<String> allFields = new HashSet<>();
Map<String, Measure> allMeasures = new HashMap<>();
semanticSchemaResp.getModelResps().forEach(modelResp -> {
allFields.addAll(modelResp.getFieldList());
if (modelResp.getModelDetail().getMeasures() != null) {
modelResp.getModelDetail().getMeasures()
.forEach(measure -> allMeasures.put(measure.getBizName(), measure));
}
});
Set<String> derivedDimensions = new HashSet<>();
Set<String> derivedMetrics = new HashSet<>();
Map<String, String> visitedMetrics = new HashMap<>();
for (MetricResp metricResp : metricResps) {
if (metrics.contains(metricResp.getBizName())) {
boolean isDerived = MetricType.isDerived(metricResp.getMetricDefineType(),
metricResp.getMetricDefineByMeasureParams());
if (isDerived) {
String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields,
allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp),
metricResp.getMetricDefineType(), aggOption, visitedMetrics,
derivedMetrics, derivedDimensions);
result.put(metricResp.getBizName(), expr);
log.debug("derived metric {}->{}", metricResp.getBizName(), expr);
} else {
measures.add(metricResp.getBizName());
}
}
}
measures.addAll(derivedMetrics);
derivedDimensions.stream().filter(dimension -> !dimensions.contains(dimension))
.forEach(dimensions::add);
return result;
}
private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List<String> dimensions) {
if (!CollectionUtils.isEmpty(dimensions)) {
Map<String, Long> modelMatchCnt = new HashMap<>();
for (ModelResp modelResp : semanticSchemaResp.getModelResps()) {
modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions()
.stream().filter(d -> dimensions.contains(d.getBizName())).count());
}
return modelMatchCnt.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
.map(m -> m.getKey()).findFirst().orElse("");
}
return semanticSchemaResp.getModelResps().get(0).getBizName();
}
}

30
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java
View File

@@ -1,14 +1,12 @@
package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.stream.Collectors;
/** Remove the default metric added by the system when the query only has dimensions */
@Slf4j
@@ -17,26 +15,26 @@ public class DetailQueryOptimizer implements QueryOptimizer {
@Override
public void rewrite(QueryStatement queryStatement) {
QueryParam queryParam = queryStatement.getQueryParam();
StructQueryParam structQueryParam = queryStatement.getStructQueryParam();
String sqlRaw = queryStatement.getSql().trim();
if (StringUtils.isEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");
}
log.debug("before handleNoMetric, sql:{}", sqlRaw);
if (isDetailQuery(queryParam)) {
if (queryParam.getMetrics().size() == 0
&& !CollectionUtils.isEmpty(queryParam.getGroups())) {
String sqlForm = "select %s from ( %s ) src_no_metric";
String sql = String.format(sqlForm,
queryParam.getGroups().stream().collect(Collectors.joining(",")), sqlRaw);
queryStatement.setSql(sql);
}
}
// if (isDetailQuery(structQueryParam)) {
// if (!CollectionUtils.isEmpty(structQueryParam.getGroups())) {
// String sqlForm = "select %s from ( %s ) src_no_metric";
// String sql = String.format(sqlForm,
// structQueryParam.getGroups().stream().collect(Collectors.joining(",")),
// sqlRaw);
// queryStatement.setSql(sql);
// }
// }
log.debug("after handleNoMetric, sql:{}", queryStatement.getSql());
}
public boolean isDetailQuery(QueryParam queryParam) {
return Objects.nonNull(queryParam) && queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(queryParam.getMetrics());
public boolean isDetailQuery(StructQueryParam structQueryParam) {
return Objects.nonNull(structQueryParam)
&& structQueryParam.getQueryType().isNativeAggQuery();
}
}

3
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java
View File

@@ -1,9 +1,8 @@
package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
/** A query parser generates physical SQL for the QueryStatement. */
public interface QueryParser {
void parse(QueryStatement queryStatement, AggOption aggOption) throws Exception;
void parse(QueryStatement queryStatement) throws Exception;
}

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.core.translator.calcite;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.QueryParser;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
@@ -16,7 +15,7 @@ import org.springframework.stereotype.Component;
public class CalciteQueryParser implements QueryParser {
@Override
public void parse(QueryStatement queryStatement, AggOption isAgg) throws Exception {
public void parse(QueryStatement queryStatement) throws Exception {
Ontology ontology = queryStatement.getOntology();
if (ontology == null) {
queryStatement.setErrMsg("No ontology could be found");
@@ -29,7 +28,7 @@ public class CalciteQueryParser implements QueryParser {
.enableOptimize(queryStatement.getEnableOptimize()).build())
.build();
SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema);
sqlBuilder.build(queryStatement, isAgg);
sqlBuilder.build(queryStatement);
}
}

10
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java
View File

@@ -1,17 +1,19 @@
package com.tencent.supersonic.headless.api.pojo;
package com.tencent.supersonic.headless.core.translator.calcite.s2sql;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import lombok.Data;
import java.util.List;
@Data
public class MetricTable {
private String alias;
public class OntologyQueryParam {
private List<String> metrics = Lists.newArrayList();
private List<String> dimensions = Lists.newArrayList();
private String where;
private Long limit;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
private AggOption aggOption = AggOption.DEFAULT;
}

62
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java
View File

@@ -1,14 +1,13 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.headless.core.translator.calcite.sql.render.FilterRender;
@@ -17,23 +16,16 @@ import com.tencent.supersonic.headless.core.translator.calcite.sql.render.Render
import com.tencent.supersonic.headless.core.translator.calcite.sql.render.SourceRender;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Objects;
import java.util.*;
/** parsing from query dimensions and metrics */
@Slf4j
public class SqlBuilder {
private final S2CalciteSchema schema;
private MetricQueryParam metricQueryParam;
private OntologyQueryParam ontologyQueryParam;
private SqlValidatorScope scope;
private SqlNode parserNode;
private boolean isAgg = false;
@@ -43,45 +35,32 @@ public class SqlBuilder {
this.schema = schema;
}
public void build(QueryStatement queryStatement, AggOption aggOption) throws Exception {
this.metricQueryParam = queryStatement.getMetricQueryParam();
if (metricQueryParam.getMetrics() == null) {
metricQueryParam.setMetrics(new ArrayList<>());
public void build(QueryStatement queryStatement) throws Exception {
this.ontologyQueryParam = queryStatement.getOntologyQueryParam();
if (ontologyQueryParam.getMetrics() == null) {
ontologyQueryParam.setMetrics(new ArrayList<>());
}
if (metricQueryParam.getDimensions() == null) {
metricQueryParam.setDimensions(new ArrayList<>());
if (ontologyQueryParam.getDimensions() == null) {
ontologyQueryParam.setDimensions(new ArrayList<>());
}
if (metricQueryParam.getLimit() == null) {
metricQueryParam.setLimit(0L);
if (ontologyQueryParam.getLimit() == null) {
ontologyQueryParam.setLimit(0L);
}
this.aggOption = aggOption;
this.aggOption = ontologyQueryParam.getAggOption();
buildParseNode();
Database database = queryStatement.getOntology().getDatabase();
EngineType engineType = EngineType.fromString(database.getType());
optimizeParseNode(engineType);
String sql = getSql(engineType);
queryStatement.setSql(sql);
if (Objects.nonNull(queryStatement.getEnableOptimize())
&& queryStatement.getEnableOptimize()
&& Objects.nonNull(queryStatement.getDataSetAlias())
&& !queryStatement.getDataSetAlias().isEmpty()) {
// simplify model sql with query sql
String simplifySql = rewrite(getSqlByDataSet(engineType, sql,
queryStatement.getDataSetSql(), queryStatement.getDataSetAlias()), engineType);
if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) {
log.debug("simplifySql [{}]", simplifySql);
queryStatement.setDataSetSimplifySql(simplifySql);
}
}
}
private void buildParseNode() throws Exception {
// find the match Datasource
scope = SchemaBuilder.getScope(schema);
List<DataModel> dataModels =
DataModelNode.getRelatedDataModels(scope, schema, metricQueryParam);
DataModelNode.getRelatedDataModels(scope, schema, ontologyQueryParam);
if (dataModels == null || dataModels.isEmpty()) {
throw new Exception("data model not found");
}
@@ -98,14 +77,14 @@ public class SqlBuilder {
while (it.hasNext()) {
Renderer renderer = it.next();
if (previous != null) {
previous.render(metricQueryParam, dataModels, scope, schema, !isAgg);
previous.render(ontologyQueryParam, dataModels, scope, schema, !isAgg);
renderer.setTable(previous
.builderAs(DataModelNode.getNames(dataModels) + "_" + String.valueOf(i)));
i++;
}
previous = renderer;
}
builders.getLast().render(metricQueryParam, dataModels, scope, schema, !isAgg);
builders.getLast().render(ontologyQueryParam, dataModels, scope, schema, !isAgg);
parserNode = builders.getLast().builder();
}
@@ -116,7 +95,7 @@ public class SqlBuilder {
// default by dataModel time aggregation
if (Objects.nonNull(dataModel.getAggTime()) && !dataModel.getAggTime()
.equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) {
if (!metricQueryParam.isNativeQuery()) {
if (!ontologyQueryParam.isNativeQuery()) {
return true;
}
}
@@ -164,13 +143,4 @@ public class SqlBuilder {
}
}
private String getSqlByDataSet(EngineType engineType, String parentSql, String dataSetSql,
String parentAlias) throws SqlParseException {
if (!SqlMergeWithUtils.hasWith(engineType, dataSetSql)) {
return String.format("with %s as (%s) %s", parentAlias, parentSql, dataSetSql);
}
return SqlMergeWithUtils.mergeWith(engineType, dataSetSql,
Collections.singletonList(parentSql), Collections.singletonList(parentAlias));
}
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java
View File

@@ -4,13 +4,13 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.common.calcite.Configuration;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.SchemaBuilder;
import lombok.extern.slf4j.Slf4j;
@@ -150,7 +150,7 @@ public class DataModelNode extends SemanticNode {
}
public static void getQueryDimensionMeasure(S2CalciteSchema schema,
MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures) {
OntologyQueryParam metricCommand, Set<String> queryDimension, List<String> measures) {
queryDimension.addAll(metricCommand.getDimensions().stream()
.map(d -> d.contains(Constants.DIMENSION_IDENTIFY)
? d.split(Constants.DIMENSION_IDENTIFY)[1]
@@ -166,7 +166,7 @@ public class DataModelNode extends SemanticNode {
}
public static void mergeQueryFilterDimensionMeasure(S2CalciteSchema schema,
MetricQueryParam metricCommand, Set<String> queryDimension, List<String> measures,
OntologyQueryParam metricCommand, Set<String> queryDimension, List<String> measures,
SqlValidatorScope scope) throws Exception {
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());
if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
@@ -192,7 +192,7 @@ public class DataModelNode extends SemanticNode {
}
public static List<DataModel> getRelatedDataModels(SqlValidatorScope scope,
S2CalciteSchema schema, MetricQueryParam metricCommand) throws Exception {
S2CalciteSchema schema, OntologyQueryParam metricCommand) throws Exception {
List<DataModel> dataModels = new ArrayList<>();
// check by metric
@@ -208,7 +208,7 @@ public class DataModelNode extends SemanticNode {
sourceMeasure.retainAll(measures);
dataSourceMeasures.put(entry.getKey(), sourceMeasure.size());
}
log.info("dataSourceMeasures [{}]", dataSourceMeasures);
log.info("metrics: [{}]", dataSourceMeasures);
Optional<Map.Entry<String, Integer>> base = dataSourceMeasures.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst();
if (base.isPresent()) {
@@ -267,7 +267,7 @@ public class DataModelNode extends SemanticNode {
}
private static boolean checkMatch(Set<String> sourceMeasure, Set<String> queryDimension,
List<String> measures, Set<String> dimension, MetricQueryParam metricCommand,
List<String> measures, Set<String> dimension, OntologyQueryParam metricCommand,
SqlValidatorScope scope, EngineType engineType) throws Exception {
boolean isAllMatch = true;
sourceMeasure.retainAll(measures);

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode;
@@ -26,7 +26,7 @@ import java.util.stream.Collectors;
public class FilterRender extends Renderer {
@Override
public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
public void render(OntologyQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
TableView tableView = super.tableView;
SqlNode filterNode = null;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
@@ -9,6 +8,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.AggFunctionNode;
@@ -47,7 +47,7 @@ import java.util.stream.Collectors;
public class JoinRender extends Renderer {
@Override
public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
public void render(OntologyQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere();
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java
View File

@@ -2,8 +2,8 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode;
@@ -22,7 +22,7 @@ import java.util.List;
public class OutputRender extends Renderer {
@Override
public void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
public void render(OntologyQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
TableView selectDataSet = super.tableView;
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java
View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MeasureNode;
@@ -114,6 +114,6 @@ public abstract class Renderer {
return SemanticNode.buildAs(alias, tableView.build());
}
public abstract void render(MetricQueryParam metricCommand, List<DataModel> dataModels,
public abstract void render(OntologyQueryParam metricCommand, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception;
}

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.core.translator.calcite.sql.render;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
@@ -9,6 +8,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView;
import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode;
@@ -336,9 +336,9 @@ public class SourceRender extends Renderer {
}
}
public void render(MetricQueryParam metricQueryParam, List<DataModel> dataModels,
public void render(OntologyQueryParam ontologyQueryParam, List<DataModel> dataModels,
SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricQueryParam.getWhere();
String queryWhere = ontologyQueryParam.getWhere();
Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>();
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());
@@ -349,13 +349,13 @@ public class SourceRender extends Renderer {
}
if (dataModels.size() == 1) {
DataModel dataModel = dataModels.get(0);
super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(),
metricQueryParam.getDimensions(), metricQueryParam.getWhere(), dataModel, scope,
schema, nonAgg);
super.tableView = renderOne("", fieldWhere, ontologyQueryParam.getMetrics(),
ontologyQueryParam.getDimensions(), ontologyQueryParam.getWhere(), dataModel,
scope, schema, nonAgg);
return;
}
JoinRender joinRender = new JoinRender();
joinRender.render(metricQueryParam, dataModels, scope, schema, nonAgg);
joinRender.render(ontologyQueryParam, dataModels, scope, schema, nonAgg);
super.tableView = joinRender.getTableView();
}
}

15
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java
View File

@@ -4,7 +4,6 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import lombok.extern.slf4j.Slf4j;
@@ -28,8 +27,8 @@ public class DefaultDimValueConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
return !Objects.isNull(queryStatement.getDataSetQueryParam())
&& !StringUtils.isBlank(queryStatement.getDataSetQueryParam().getSql());
return Objects.nonNull(queryStatement.getSqlQueryParam())
&& StringUtils.isNotBlank(queryStatement.getSqlQueryParam().getSql());
}
@Override
@@ -40,15 +39,13 @@ public class DefaultDimValueConverter implements QueryConverter {
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
String sql = queryStatement.getDataSetQueryParam().getSql();
String sql = queryStatement.getSqlQueryParam().getSql();
List<String> whereFields = SqlSelectHelper.getWhereFields(sql).stream()
.filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) {
return;
}
MetricTable metricTable =
queryStatement.getDataSetQueryParam().getTables().stream().findFirst().orElse(null);
List<Expression> expressions = Lists.newArrayList();
for (Dimension dimension : dimensions) {
ExpressionList expressionList = new ExpressionList();
@@ -59,11 +56,11 @@ public class DefaultDimValueConverter implements QueryConverter {
inExpression.setLeftExpression(new Column(dimension.getBizName()));
inExpression.setRightExpression(expressionList);
expressions.add(inExpression);
if (metricTable != null) {
metricTable.getDimensions().add(dimension.getBizName());
if (Objects.nonNull(queryStatement.getSqlQueryParam().getTable())) {
queryStatement.getOntologyQueryParam().getDimensions().add(dimension.getBizName());
}
}
sql = SqlAddHelper.addWhere(sql, expressions);
queryStatement.getDataSetQueryParam().setSql(sql);
queryStatement.getSqlQueryParam().setSql(sql);
}
}

291
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java
View File

@@ -6,82 +6,46 @@ import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/** supplement the QueryStatement when query with custom aggregation method */
@Component("CalculateAggConverter")
@Slf4j
public class CalculateAggConverter implements QueryConverter {
public class MetricRatioConverter implements QueryConverter {
public interface EngineSql {
String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql);
}
public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement,
EngineType engineTypeEnum, String version) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
QueryParam queryParam = queryStatement.getQueryParam();
// 同环比
if (isRatioAccept(queryParam)) {
return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
}
DataSetQueryParam sqlCommand = new DataSetQueryParam();
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryParam.getMetrics());
metricTable.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommand, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
String sql = String.format("select %s from %s %s %s %s",
sqlGenerateUtils.getSelect(queryParam), metricTableName,
sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam),
sqlGenerateUtils.getLimit(queryParam));
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false);
sql = String.format("select %s from %s t0 %s %s %s",
sqlGenerateUtils.getSelect(queryParam), metricTableName,
sqlGenerateUtils.getGroupBy(queryParam),
sqlGenerateUtils.getOrderBy(queryParam), sqlGenerateUtils.getLimit(queryParam));
}
sqlCommand.setSql(sql);
return sqlCommand;
String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith,
String metricSql);
}
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
if (Objects.isNull(queryStatement.getStructQueryParam()) || queryStatement.getIsS2SQL()
|| !isRatioAccept(queryStatement.getStructQueryParam())) {
return false;
}
QueryParam queryParam = queryStatement.getQueryParam();
if (queryParam.getQueryType().isNativeAggQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryParam.getAggregators())) {
StructQueryParam structQueryParam = queryStatement.getStructQueryParam();
if (structQueryParam.getQueryType().isNativeAggQuery()
|| CollectionUtils.isEmpty(structQueryParam.getAggregators())) {
return false;
}
int nonSumFunction = 0;
for (Aggregator agg : queryParam.getAggregators()) {
for (Aggregator agg : structQueryParam.getAggregators()) {
if (agg.getFunc() == null || "".equals(agg.getFunc())) {
return false;
}
@@ -98,14 +62,13 @@ public class CalculateAggConverter implements QueryConverter {
@Override
public void convert(QueryStatement queryStatement) throws Exception {
Database database = queryStatement.getOntology().getDatabase();
DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement,
EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
queryStatement.setDataSetQueryParam(dataSetQueryParam);
generateRatioSql(queryStatement, EngineType.fromString(database.getType().toUpperCase()),
database.getVersion());
}
/** Ratio */
public boolean isRatioAccept(QueryParam queryParam) {
Long ratioFuncNum = queryParam.getAggregators().stream()
public boolean isRatioAccept(StructQueryParam structQueryParam) {
Long ratioFuncNum = structQueryParam.getAggregators().stream()
.filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)
|| f.getFunc().equals(AggOperatorEnum.RATIO_OVER)))
.count();
@@ -115,53 +78,47 @@ public class CalculateAggConverter implements QueryConverter {
return false;
}
public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement,
EngineType engineTypeEnum, String version) throws Exception {
public void generateRatioSql(QueryStatement queryStatement, EngineType engineTypeEnum,
String version) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
QueryParam queryParam = queryStatement.getQueryParam();
check(queryParam);
StructQueryParam structQueryParam = queryStatement.getStructQueryParam();
check(structQueryParam);
queryStatement.setEnableOptimize(false);
DataSetQueryParam sqlCommand = new DataSetQueryParam();
OntologyQueryParam ontologyQueryParam = queryStatement.getOntologyQueryParam();
ontologyQueryParam.setAggOption(AggOption.AGGREGATION);
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
metricTable.setMetrics(queryParam.getMetrics());
metricTable.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommend, complete where:{}", where);
metricTable.setWhere(where);
metricTable.setAggOption(AggOption.AGGREGATION);
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
boolean isOver = isOverRatio(queryParam);
boolean isOver = isOverRatio(structQueryParam);
String sql = "";
SqlQueryParam dsParam = queryStatement.getSqlQueryParam();
dsParam.setTable(metricTableName);
switch (engineTypeEnum) {
case H2:
sql = new H2EngineSql().sql(queryParam, isOver, true, metricTableName);
sql = new H2EngineSql().sql(structQueryParam, isOver, true, metricTableName);
break;
case MYSQL:
case DORIS:
case CLICKHOUSE:
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
sqlCommand.setSupportWith(false);
dsParam.setSupportWith(false);
}
if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) {
sql = new MysqlEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(),
metricTableName);
sql = new MysqlEngineSql().sql(structQueryParam, isOver,
dsParam.isSupportWith(), metricTableName);
} else {
sql = new CkEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(),
sql = new CkEngineSql().sql(structQueryParam, isOver, dsParam.isSupportWith(),
metricTableName);
}
break;
default:
}
sqlCommand.setSql(sql);
return sqlCommand;
dsParam.setSql(sql);
}
public class H2EngineSql implements EngineSql {
public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr = queryParam.getAggregators().stream().map(f -> {
public String getOverSelect(StructQueryParam structQueryParam, boolean isOver) {
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s",
@@ -171,43 +128,44 @@ public class CalculateAggConverter implements QueryConverter {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: String.join(",", structQueryParam.getGroups()) + "," + aggStr;
}
public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryParam.getDateInfo())) {
public String getTimeSpan(StructQueryParam structQueryParam, boolean isOver,
boolean isAdd) {
if (Objects.nonNull(structQueryParam.getDateInfo())) {
String addStr = isAdd ? "" : "-";
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) {
return "day," + (isOver ? addStr + "7" : addStr + "1");
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return isOver ? "month," + addStr + "1" : "day," + addStr + "7";
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH.MONTH)) {
return isOver ? "year," + addStr + "1" : "month," + addStr + "1";
}
}
return "";
}
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
String timeDim = getTimeDim(structQueryParam);
String timeSpan = getTimeSpan(structQueryParam, isOver, true);
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ",
aliasRight + timeDim, aliasLeft + timeDim, timeSpan,
aliasRight + timeDim);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ",
getTimeSpan(queryParam, isOver, false), aliasLeft + timeDim,
getTimeSpan(structQueryParam, isOver, false), aliasLeft + timeDim,
aliasRight + timeDim);
}
return String.format("%s = TIMESTAMPADD(%s,%s) ", aliasLeft + timeDim, timeSpan,
@@ -217,7 +175,7 @@ public class CalculateAggConverter implements QueryConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
for (String group : structQueryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -228,35 +186,36 @@ public class CalculateAggConverter implements QueryConverter {
}
@Override
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith,
String metricSql) {
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
getOverSelect(structQueryParam, isOver), getAllSelect(structQueryParam, "t0."),
getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql,
getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam),
getLimit(structQueryParam));
return sql;
}
}
public class CkEngineSql extends MysqlEngineSql {
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
String timeDim = getTimeDim(structQueryParam);
String timeSpan = "INTERVAL " + getTimeSpan(structQueryParam, isOver, true);
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false),
aliasLeft + timeDim, getTimeSpan(structQueryParam, isOver, false),
aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim,
@@ -266,7 +225,7 @@ public class CalculateAggConverter implements QueryConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
for (String group : structQueryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -277,45 +236,49 @@ public class CalculateAggConverter implements QueryConverter {
}
@Override
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith,
String metricSql) {
if (!asWith) {
return String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
getOverSelect(structQueryParam, isOver),
getAllSelect(structQueryParam, "t0."),
getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql,
getJoinOn(structQueryParam, isOver, "t0.", "t1."),
getOrderBy(structQueryParam), getLimit(structQueryParam));
}
return String.format(
",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s "
+ "from t0 left join t1 on %s ) metric_tb_src %s %s ",
metricSql, metricSql, getOverSelect(queryParam, isOver),
getAllSelect(queryParam, "t0."), getAllJoinSelect(queryParam, "t1."),
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
metricSql, metricSql, getOverSelect(structQueryParam, isOver),
getAllSelect(structQueryParam, "t0."),
getAllJoinSelect(structQueryParam, "t1."),
getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam),
getLimit(structQueryParam));
}
}
public class MysqlEngineSql implements EngineSql {
public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) {
if (Objects.nonNull(queryParam.getDateInfo())) {
public String getTimeSpan(StructQueryParam structQueryParam, boolean isOver,
boolean isAdd) {
if (Objects.nonNull(structQueryParam.getDateInfo())) {
String addStr = isAdd ? "" : "-";
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) {
return isOver ? addStr + "7 day" : addStr + "1 day";
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)) {
return isOver ? addStr + "1 month" : addStr + "7 day";
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return isOver ? addStr + "1 year" : addStr + "1 month";
}
}
return "";
}
public String getOverSelect(QueryParam queryParam, boolean isOver) {
String aggStr = queryParam.getAggregators().stream().map(f -> {
public String getOverSelect(StructQueryParam structQueryParam, boolean isOver) {
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s",
@@ -325,26 +288,26 @@ public class CalculateAggConverter implements QueryConverter {
return f.getColumn();
}
}).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: String.join(",", structQueryParam.getGroups()) + "," + aggStr;
}
public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft,
public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft,
String aliasRight) {
String timeDim = getTimeDim(queryParam);
String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true);
String aggStr = queryParam.getAggregators().stream().map(f -> {
String timeDim = getTimeDim(structQueryParam);
String timeSpan = "INTERVAL " + getTimeSpan(structQueryParam, isOver, true);
String aggStr = structQueryParam.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER)
|| f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) {
return String.format(
"%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ",
aliasLeft + timeDim, aliasRight + timeDim, timeSpan);
}
if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)
&& isOver) {
return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s",
aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false),
aliasLeft + timeDim, getTimeSpan(structQueryParam, isOver, false),
aliasRight + timeDim);
}
return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim,
@@ -354,7 +317,7 @@ public class CalculateAggConverter implements QueryConverter {
}
}).collect(Collectors.joining(" and "));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
for (String group : structQueryParam.getGroups()) {
if (group.equalsIgnoreCase(timeDim)) {
continue;
}
@@ -365,51 +328,53 @@ public class CalculateAggConverter implements QueryConverter {
}
@Override
public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) {
public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith,
String metricSql) {
String sql = String.format(
"select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ",
getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."),
getAllJoinSelect(queryParam, "t1."), metricSql, metricSql,
getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam),
getLimit(queryParam));
getOverSelect(structQueryParam, isOver), getAllSelect(structQueryParam, "t0."),
getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql,
getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam),
getLimit(structQueryParam));
return sql;
}
}
private String getAllJoinSelect(QueryParam queryParam, String alias) {
String aggStr = queryParam.getAggregators().stream()
private String getAllJoinSelect(StructQueryParam structQueryParam, String alias) {
String aggStr = structQueryParam.getAggregators().stream()
.map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "") + "_roll")
.collect(Collectors.joining(","));
List<String> groups = new ArrayList<>();
for (String group : queryParam.getGroups()) {
for (String group : structQueryParam.getGroups()) {
groups.add(alias + group + " as " + group + "_roll");
}
return CollectionUtils.isEmpty(groups) ? aggStr : String.join(",", groups) + "," + aggStr;
}
private String getGroupDimWithOutTime(QueryParam queryParam) {
String timeDim = getTimeDim(queryParam);
return queryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
private String getGroupDimWithOutTime(StructQueryParam structQueryParam) {
String timeDim = getTimeDim(structQueryParam);
return structQueryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(","));
}
private static String getTimeDim(QueryParam queryParam) {
private static String getTimeDim(StructQueryParam structQueryParam) {
DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class);
return dateModeUtils.getSysDateCol(queryParam.getDateInfo());
return dateModeUtils.getSysDateCol(structQueryParam.getDateInfo());
}
private static String getLimit(QueryParam queryParam) {
if (queryParam != null && queryParam.getLimit() != null && queryParam.getLimit() > 0) {
return " limit " + String.valueOf(queryParam.getLimit());
private static String getLimit(StructQueryParam structQueryParam) {
if (structQueryParam != null && structQueryParam.getLimit() != null
&& structQueryParam.getLimit() > 0) {
return " limit " + String.valueOf(structQueryParam.getLimit());
}
return "";
}
private String getAllSelect(QueryParam queryParam, String alias) {
String aggStr = queryParam.getAggregators().stream().map(f -> getSelectField(f, alias))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: alias + String.join("," + alias, queryParam.getGroups()) + "," + aggStr;
private String getAllSelect(StructQueryParam structQueryParam, String alias) {
String aggStr = structQueryParam.getAggregators().stream()
.map(f -> getSelectField(f, alias)).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: alias + String.join("," + alias, structQueryParam.getGroups()) + "," + aggStr;
}
private String getSelectField(final Aggregator agg, String alias) {
@@ -421,32 +386,32 @@ public class CalculateAggConverter implements QueryConverter {
return sqlGenerateUtils.getSelectField(agg);
}
private String getGroupBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getGroups())) {
private String getGroupBy(StructQueryParam structQueryParam) {
if (CollectionUtils.isEmpty(structQueryParam.getGroups())) {
return "";
}
return "group by " + String.join(",", queryParam.getGroups());
return "group by " + String.join(",", structQueryParam.getGroups());
}
private static String getOrderBy(QueryParam queryParam) {
return "order by " + getTimeDim(queryParam) + " desc";
private static String getOrderBy(StructQueryParam structQueryParam) {
return "order by " + getTimeDim(structQueryParam) + " desc";
}
private boolean isOverRatio(QueryParam queryParam) {
Long overCt = queryParam.getAggregators().stream()
private boolean isOverRatio(StructQueryParam structQueryParam) {
Long overCt = structQueryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
return overCt > 0;
}
private void check(QueryParam queryParam) throws Exception {
Long ratioOverNum = queryParam.getAggregators().stream()
private void check(StructQueryParam structQueryParam) throws Exception {
Long ratioOverNum = structQueryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count();
Long ratioRollNum = queryParam.getAggregators().stream()
Long ratioRollNum = structQueryParam.getAggregators().stream()
.filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count();
if (ratioOverNum > 0 && ratioRollNum > 0) {
throw new Exception("not support over ratio and roll ratio together ");
}
if (getTimeDim(queryParam).isEmpty()) {
if (getTimeDim(structQueryParam).isEmpty()) {
throw new Exception("miss time filter");
}
}

74
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java
View File

@@ -1,74 +0,0 @@
package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/** QueryConverter default implement */
@Component("ParserDefaultConverter")
@Slf4j
public class ParserDefaultConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) {
return false;
}
CalculateAggConverter calculateConverterAgg =
ContextUtils.getBean(CalculateAggConverter.class);
return !calculateConverterAgg.accept(queryStatement);
}
@Override
public void convert(QueryStatement queryStatement) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
QueryParam queryParam = queryStatement.getQueryParam();
MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam();
MetricQueryParam metricReq =
generateSqlCommand(queryStatement.getQueryParam(), queryStatement);
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryParam, null));
BeanUtils.copyProperties(metricReq, metricQueryParam);
}
public MetricQueryParam generateSqlCommand(QueryParam queryParam,
QueryStatement queryStatement) {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
MetricQueryParam metricQueryParam = new MetricQueryParam();
metricQueryParam.setMetrics(queryParam.getMetrics());
metricQueryParam.setDimensions(queryParam.getGroups());
String where = sqlGenerateUtils.generateWhere(queryParam, null);
log.info("in generateSqlCommend, complete where:{}", where);
metricQueryParam.setWhere(where);
metricQueryParam.setOrder(queryParam.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection()))
.collect(Collectors.toList()));
metricQueryParam.setLimit(queryParam.getLimit());
// support detail query
if (queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(metricQueryParam.getMetrics())) {
Map<Long, DataModel> modelMap = queryStatement.getOntology().getModelMap();
for (Long modelId : modelMap.keySet()) {
String modelBizName = modelMap.get(modelId).getName();
String internalMetricName =
sqlGenerateUtils.generateInternalMetricName(modelBizName);
metricQueryParam.getMetrics().add(internalMetricName);
}
}
return metricQueryParam;
}
}

308
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java Normal file
View File

@@ -0,0 +1,308 @@
package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component("SqlQueryConverter")
@Slf4j
public class SqlQueryConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.nonNull(queryStatement.getSqlQueryParam()) && queryStatement.getIsS2SQL()) {
return true;
}
return false;
}
@Override
public void convert(QueryStatement queryStatement) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
convertNameToBizName(queryStatement);
rewriteFunction(queryStatement);
String reqSql = queryStatement.getSqlQueryParam().getSql();
String tableName = SqlSelectHelper.getTableName(reqSql);
if (StringUtils.isEmpty(tableName)) {
return;
}
// replace order by field with the select sequence number
queryStatement.setSql(SqlReplaceHelper.replaceAggAliasOrderbyField(reqSql));
log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, queryStatement.getSql());
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
// fill dataSetQuery
SqlQueryParam sqlQueryParam = queryStatement.getSqlQueryParam();
sqlQueryParam.setTable(tableName.toLowerCase());
if (!sqlGenerateUtils.isSupportWith(
EngineType.fromString(semanticSchemaResp.getDatabaseResp().getType().toUpperCase()),
semanticSchemaResp.getDatabaseResp().getVersion())) {
sqlQueryParam.setSupportWith(false);
sqlQueryParam.setWithAlias(false);
}
// build ontologyQuery
List<String> allFields = SqlSelectHelper.getAllSelectFields(queryStatement.getSql());
List<MetricSchemaResp> metricSchemas = getMetrics(semanticSchemaResp, allFields);
List<String> metrics =
metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList());
AggOption aggOption = getAggOption(queryStatement, metricSchemas);
Set<String> dimensions = getDimensions(semanticSchemaResp, allFields);
OntologyQueryParam ontologyQueryParam = new OntologyQueryParam();
ontologyQueryParam.getMetrics().addAll(metrics);
ontologyQueryParam.getDimensions().addAll(dimensions);
ontologyQueryParam.setAggOption(aggOption);
ontologyQueryParam.setNativeQuery(!AggOption.isAgg(aggOption));
log.info("parse sqlQuery [{}] ", sqlQueryParam);
queryStatement.setOntologyQueryParam(ontologyQueryParam);
queryStatement.setSql(sqlQueryParam.getSql());
generateDerivedMetric(sqlGenerateUtils, queryStatement);
}
private AggOption getAggOption(QueryStatement queryStatement,
List<MetricSchemaResp> metricSchemas) {
String sql = queryStatement.getSql();
if (SqlSelectFunctionHelper.hasAggregateFunction(sql)) {
return AggOption.AGGREGATION;
}
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql)
&& !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) {
log.debug("getAggOption simple sql set to DEFAULT");
return AggOption.DEFAULT;
}
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql)
|| SqlSelectFunctionHelper.hasFunction(sql, "count")
|| SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
return AggOption.OUTER;
}
if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql)
|| SqlSelectHelper.hasGroupBy(sql)) {
return AggOption.OUTER;
}
long defaultAggNullCnt = metricSchemas.stream().filter(
m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg()))
.count();
if (defaultAggNullCnt > 0) {
log.debug("getAggOption find null defaultAgg metric set to NATIVE");
return AggOption.OUTER;
}
return AggOption.DEFAULT;
}
private Set<String> getDimensions(SemanticSchemaResp semanticSchemaResp,
List<String> allFields) {
Map<String, String> dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(),
SchemaItem::getBizName, (k1, k2) -> k1));
dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.DAY.getName());
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase()))
.collect(Collectors.toSet());
}
private List<MetricSchemaResp> getMetrics(SemanticSchemaResp semanticSchemaResp,
List<String> allFields) {
Map<String, MetricSchemaResp> metricLowerToNameMap =
semanticSchemaResp.getMetrics().stream().collect(Collectors
.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry));
return allFields.stream()
.filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase()))
.collect(Collectors.toList());
}
private void generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils,
QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
SqlQueryParam dsParam = queryStatement.getSqlQueryParam();
OntologyQueryParam ontology = queryStatement.getOntologyQueryParam();
String sql = dsParam.getSql();
Set<String> measures = new HashSet<>();
Map<String, String> replaces = generateDerivedMetric(sqlGenerateUtils, semanticSchemaResp,
ontology.getAggOption(), ontology.getMetrics(), ontology.getDimensions(), measures);
if (!CollectionUtils.isEmpty(replaces)) {
// metricTable sql use measures replace metric
sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces);
ontology.setAggOption(AggOption.NATIVE);
// metricTable use measures replace metric
if (!CollectionUtils.isEmpty(measures)) {
ontology.getMetrics().addAll(measures);
} else {
// empty measure , fill default
ontology.setMetrics(new ArrayList<>());
ontology.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
getDefaultModel(semanticSchemaResp, ontology.getDimensions())));
}
}
dsParam.setSql(sql);
}
private Map<String, String> generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils,
SemanticSchemaResp semanticSchemaResp, AggOption aggOption, List<String> metrics,
List<String> dimensions, Set<String> measures) {
Map<String, String> result = new HashMap<>();
List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
// Check if any metric is derived
boolean hasDerivedMetrics =
metricResps.stream().anyMatch(m -> metrics.contains(m.getBizName()) && MetricType
.isDerived(m.getMetricDefineType(), m.getMetricDefineByMeasureParams()));
if (!hasDerivedMetrics) {
return result;
}
log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics);
Set<String> allFields = new HashSet<>();
Map<String, Measure> allMeasures = new HashMap<>();
semanticSchemaResp.getModelResps().forEach(modelResp -> {
allFields.addAll(modelResp.getFieldList());
if (modelResp.getModelDetail().getMeasures() != null) {
modelResp.getModelDetail().getMeasures()
.forEach(measure -> allMeasures.put(measure.getBizName(), measure));
}
});
Set<String> derivedDimensions = new HashSet<>();
Set<String> derivedMetrics = new HashSet<>();
Map<String, String> visitedMetrics = new HashMap<>();
for (MetricResp metricResp : metricResps) {
if (metrics.contains(metricResp.getBizName())) {
boolean isDerived = MetricType.isDerived(metricResp.getMetricDefineType(),
metricResp.getMetricDefineByMeasureParams());
if (isDerived) {
String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields,
allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp),
metricResp.getMetricDefineType(), aggOption, visitedMetrics,
derivedMetrics, derivedDimensions);
result.put(metricResp.getBizName(), expr);
log.debug("derived metric {}->{}", metricResp.getBizName(), expr);
} else {
measures.add(metricResp.getBizName());
}
}
}
measures.addAll(derivedMetrics);
derivedDimensions.stream().filter(dimension -> !dimensions.contains(dimension))
.forEach(dimensions::add);
return result;
}
private void convertNameToBizName(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp);
String sql = queryStatement.getSqlQueryParam().getSql();
log.debug("dataSetId:{},convert name to bizName before:{}", queryStatement.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
log.debug("dataSetId:{},convert name to bizName after:{}", queryStatement.getDataSetId(),
sql);
sql = SqlReplaceHelper.replaceTable(sql,
Constants.TABLE_PREFIX + queryStatement.getDataSetId());
log.debug("replaceTableName after:{}", sql);
queryStatement.getSqlQueryParam().setSql(sql);
}
private void rewriteFunction(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
String sql = queryStatement.getSqlQueryParam().getSql();
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
}
String type = database.getType();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase());
if (Objects.nonNull(engineAdaptor)) {
String functionNameCorrector = engineAdaptor.functionNameCorrector(sql);
queryStatement.getSqlQueryParam().setSql(functionNameCorrector);
}
}
protected Map<String, String> getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) {
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = semanticSchemaResp.getDimensions().stream().flatMap(
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
Map<String, String> metricResults = semanticSchemaResp.getMetrics().stream().flatMap(
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
dimensionResults.putAll(metricResults);
return dimensionResults;
}
private Stream<Pair<String, String>> getPairStream(String aliasStr, String name,
String bizName) {
Set<Pair<String, String>> elements = new HashSet<>();
elements.add(Pair.of(name, bizName));
if (StringUtils.isNotBlank(aliasStr)) {
List<String> aliasList = SchemaItem.getAliasList(aliasStr);
for (String alias : aliasList) {
elements.add(Pair.of(alias, bizName));
}
}
return elements.stream();
}
private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List<String> dimensions) {
if (!CollectionUtils.isEmpty(dimensions)) {
Map<String, Long> modelMatchCnt = new HashMap<>();
for (ModelResp modelResp : semanticSchemaResp.getModelResps()) {
modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions()
.stream().filter(d -> dimensions.contains(d.getBizName())).count());
}
return modelMatchCnt.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
.map(m -> m.getKey()).findFirst().orElse("");
}
return semanticSchemaResp.getModelResps().get(0).getBizName();
}
}

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java
View File

@@ -14,12 +14,12 @@ import java.util.List;
import java.util.Objects;
@Slf4j
@Component("SqlVariableParseConverter")
public class SqlVariableParseConverter implements QueryConverter {
@Component("SqlVariableConverter")
public class SqlVariableConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.isNull(queryStatement.getQueryParam())) {
if (Objects.isNull(queryStatement.getStructQueryParam()) && queryStatement.getIsS2SQL()) {
return false;
}
return true;
@@ -38,7 +38,7 @@ public class SqlVariableParseConverter implements QueryConverter {
String sqlParsed =
SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(),
modelResp.getModelDetail().getSqlVariables(),
queryStatement.getQueryParam().getParams());
queryStatement.getStructQueryParam().getParams());
DataModel dataModel =
queryStatement.getOntology().getDataModelMap().get(modelResp.getBizName());
dataModel.setSqlQuery(sqlParsed);

74
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java Normal file
View File

@@ -0,0 +1,74 @@
package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.Objects;
import java.util.stream.Collectors;
@Component("ParserDefaultConverter")
@Slf4j
public class StructQueryConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
if (Objects.nonNull(queryStatement.getStructQueryParam()) && !queryStatement.getIsS2SQL()) {
return true;
}
return false;
}
@Override
public void convert(QueryStatement queryStatement) throws Exception {
SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class);
StructQueryParam structQueryParam = queryStatement.getStructQueryParam();
String dsTable = "t_1";
SqlQueryParam sqlParam = new SqlQueryParam();
sqlParam.setTable(dsTable);
String sql = String.format("select %s from %s %s %s %s",
sqlGenerateUtils.getSelect(structQueryParam), dsTable,
sqlGenerateUtils.getGroupBy(structQueryParam),
sqlGenerateUtils.getOrderBy(structQueryParam),
sqlGenerateUtils.getLimit(structQueryParam));
Database database = queryStatement.getOntology().getDatabase();
EngineType engineType = EngineType.fromString(database.getType().toUpperCase());
if (!sqlGenerateUtils.isSupportWith(engineType, database.getVersion())) {
sqlParam.setSupportWith(false);
sql = String.format("select %s from %s t0 %s %s %s",
sqlGenerateUtils.getSelect(structQueryParam), dsTable,
sqlGenerateUtils.getGroupBy(structQueryParam),
sqlGenerateUtils.getOrderBy(structQueryParam),
sqlGenerateUtils.getLimit(structQueryParam));
}
sqlParam.setSql(sql);
queryStatement.setSqlQueryParam(sqlParam);
OntologyQueryParam ontologyQueryParam = new OntologyQueryParam();
ontologyQueryParam.getDimensions().addAll(structQueryParam.getGroups());
ontologyQueryParam.getMetrics().addAll(structQueryParam.getAggregators().stream()
.map(a -> a.getColumn()).collect(Collectors.toList()));
String where = sqlGenerateUtils.generateWhere(structQueryParam, null);
ontologyQueryParam.setWhere(where);
ontologyQueryParam.setAggOption(AggOption.AGGREGATION);
ontologyQueryParam.setNativeQuery(structQueryParam.getQueryType().isNativeAggQuery());
ontologyQueryParam.setOrder(structQueryParam.getOrders().stream()
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection()))
.collect(Collectors.toList()));
ontologyQueryParam.setLimit(structQueryParam.getLimit());
queryStatement.setOntologyQueryParam(ontologyQueryParam);
log.info("parse structQuery [{}] ", queryStatement.getSqlQueryParam());
}
}

67
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -12,7 +12,6 @@ import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
@@ -20,6 +19,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.core.config.ExecutorConfig;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -85,25 +85,26 @@ public class SqlGenerateUtils {
return selectSql;
}
public String getLimit(QueryParam queryParam) {
if (queryParam != null && queryParam.getLimit() != null && queryParam.getLimit() > 0) {
return " limit " + queryParam.getLimit();
public String getLimit(StructQueryParam structQueryParam) {
if (structQueryParam != null && structQueryParam.getLimit() != null
&& structQueryParam.getLimit() > 0) {
return " limit " + structQueryParam.getLimit();
}
return "";
}
public String getSelect(QueryParam queryParam) {
String aggStr = queryParam.getAggregators().stream().map(this::getSelectField)
public String getSelect(StructQueryParam structQueryParam) {
String aggStr = structQueryParam.getAggregators().stream().map(this::getSelectField)
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: String.join(",", structQueryParam.getGroups()) + "," + aggStr;
}
public String getSelect(QueryParam queryParam, Map<String, String> deriveMetrics) {
String aggStr = queryParam.getAggregators().stream()
public String getSelect(StructQueryParam structQueryParam, Map<String, String> deriveMetrics) {
String aggStr = structQueryParam.getAggregators().stream()
.map(a -> getSelectField(a, deriveMetrics)).collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr
: String.join(",", queryParam.getGroups()) + "," + aggStr;
return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr
: String.join(",", structQueryParam.getGroups()) + "," + aggStr;
}
public String getSelectField(final Aggregator agg) {
@@ -128,46 +129,46 @@ public class SqlGenerateUtils {
return deriveMetrics.get(agg.getColumn());
}
public String getGroupBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getGroups())) {
public String getGroupBy(StructQueryParam structQueryParam) {
if (CollectionUtils.isEmpty(structQueryParam.getGroups())) {
return "";
}
return "group by " + String.join(",", queryParam.getGroups());
return "group by " + String.join(",", structQueryParam.getGroups());
}
public String getOrderBy(QueryParam queryParam) {
if (CollectionUtils.isEmpty(queryParam.getOrders())) {
public String getOrderBy(StructQueryParam structQueryParam) {
if (CollectionUtils.isEmpty(structQueryParam.getOrders())) {
return "";
}
return "order by " + queryParam.getOrders().stream()
return "order by " + structQueryParam.getOrders().stream()
.map(order -> " " + order.getColumn() + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String getOrderBy(QueryParam queryParam, Map<String, String> deriveMetrics) {
if (CollectionUtils.isEmpty(queryParam.getOrders())) {
public String getOrderBy(StructQueryParam structQueryParam, Map<String, String> deriveMetrics) {
if (CollectionUtils.isEmpty(structQueryParam.getOrders())) {
return "";
}
if (!queryParam.getOrders().stream()
if (!structQueryParam.getOrders().stream()
.anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
return getOrderBy(queryParam);
return getOrderBy(structQueryParam);
}
return "order by " + queryParam.getOrders().stream()
return "order by " + structQueryParam.getOrders().stream()
.map(order -> " " + (deriveMetrics.containsKey(order.getColumn())
? deriveMetrics.get(order.getColumn())
: order.getColumn()) + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String generateWhere(QueryParam queryParam, ItemDateResp itemDateResp) {
public String generateWhere(StructQueryParam structQueryParam, ItemDateResp itemDateResp) {
String whereClauseFromFilter =
sqlFilterUtils.getWhereClause(queryParam.getDimensionFilters());
String whereFromDate = getDateWhereClause(queryParam.getDateInfo(), itemDateResp);
return mergeDateWhereClause(queryParam, whereClauseFromFilter, whereFromDate);
sqlFilterUtils.getWhereClause(structQueryParam.getDimensionFilters());
String whereFromDate = getDateWhereClause(structQueryParam.getDateInfo(), itemDateResp);
return mergeDateWhereClause(structQueryParam, whereClauseFromFilter, whereFromDate);
}
private String mergeDateWhereClause(QueryParam queryParam, String whereClauseFromFilter,
String whereFromDate) {
private String mergeDateWhereClause(StructQueryParam structQueryParam,
String whereClauseFromFilter, String whereFromDate) {
if (StringUtils.isNotEmpty(whereFromDate)
&& StringUtils.isNotEmpty(whereClauseFromFilter)) {
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
@@ -179,7 +180,7 @@ public class SqlGenerateUtils {
return whereFromDate;
} else if (Objects.isNull(whereFromDate) && StringUtils.isEmpty(whereClauseFromFilter)) {
log.debug("the current date information is empty, enter the date initialization logic");
return dateModeUtils.defaultRecentDateInfo(queryParam.getDateInfo());
return dateModeUtils.defaultRecentDateInfo(structQueryParam.getDateInfo());
}
return whereClauseFromFilter;
}
@@ -203,12 +204,12 @@ public class SqlGenerateUtils {
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
}
public Triple<String, String, String> getBeginEndTime(QueryParam queryParam,
public Triple<String, String, String> getBeginEndTime(StructQueryParam structQueryParam,
ItemDateResp dataDate) {
if (Objects.isNull(queryParam.getDateInfo())) {
if (Objects.isNull(structQueryParam.getDateInfo())) {
return Triple.of("", "", "");
}
DateConf dateConf = queryParam.getDateInfo();
DateConf dateConf = structQueryParam.getDateInfo();
String dateInfo = dateModeUtils.getSysDateCol(dateConf);
if (dateInfo.isEmpty()) {
return Triple.of("", "", "");

3
headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.chat.core.parser.aggregate;
import com.alibaba.fastjson.JSON;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.CalciteQueryParser;
import org.junit.jupiter.api.Test;
@@ -318,7 +317,7 @@ public class CalciteSqlParserTest {
+ " \"updatedAt\": 1711367511146\n" + " }\n" + " }\n" + "}";
QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class);
CalciteQueryParser calciteSqlParser = new CalciteQueryParser();
calciteSqlParser.parse(queryStatement, AggOption.DEFAULT);
calciteSqlParser.parse(queryStatement);
Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""),
"SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" + "FROM" + "`s2_pv_uv_statis`"
+ "GROUPBY`imp_date`,`imp_date`");

73
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -6,24 +6,10 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp;
import com.tencent.supersonic.headless.api.pojo.request.*;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.chat.knowledge.MapResult;
@@ -33,6 +19,8 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.core.cache.QueryCache;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQueryParam;
import com.tencent.supersonic.headless.core.pojo.StructQueryParam;
import com.tencent.supersonic.headless.core.translator.SemanticTranslator;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
@@ -52,12 +40,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
@Service
@@ -307,30 +290,13 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryStatement;
}
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) {
// If dataSetId or DataSetName is empty, parse dataSetId from the SQL
if (querySqlReq.needGetDataSetId()) {
Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user);
querySqlReq.setDataSetId(dataSetId);
}
QueryStatement queryStatement = buildStructQueryStatement(querySqlReq);
queryStatement.setIsS2SQL(true);
queryStatement.setSql(querySqlReq.getSql());
return queryStatement;
}
private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) {
private QueryStatement buildQueryStatement(SemanticQueryReq queryReq) {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
schemaFilterReq.setDataSetId(queryReq.getDataSetId());
schemaFilterReq.setModelIds(queryReq.getModelIds());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(schemaFilterReq);
QueryStatement queryStatement = new QueryStatement();
QueryParam queryParam = new QueryParam();
BeanUtils.copyProperties(queryReq, queryParam);
queryStatement.setQueryParam(queryParam);
queryStatement.setModelIds(queryReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setDataSetId(queryReq.getDataSetId());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
@@ -338,6 +304,31 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryStatement;
}
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) {
QueryStatement queryStatement = buildQueryStatement(querySqlReq);
queryStatement.setIsS2SQL(true);
SqlQueryParam sqlQueryParam = new SqlQueryParam();
sqlQueryParam.setSql(querySqlReq.getSql());
queryStatement.setSqlQueryParam(sqlQueryParam);
// If dataSetId or DataSetName is empty, parse dataSetId from the SQL
if (querySqlReq.needGetDataSetId()) {
Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user);
querySqlReq.setDataSetId(dataSetId);
}
return queryStatement;
}
private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) {
QueryStatement queryStatement = buildQueryStatement(queryReq);
StructQueryParam structQueryParam = new StructQueryParam();
BeanUtils.copyProperties(queryReq, structQueryParam);
queryStatement.setStructQueryParam(structQueryParam);
queryStatement.setIsS2SQL(false);
return queryStatement;
}
private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq) {
List<QueryStatement> queryStatements = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java
View File

@@ -67,7 +67,8 @@ public class MetricDrillDownChecker {
List<MetricResp> metricResps = getMetrics(metricFields, semanticSchemaResp);
if (!checkDrillDownDimension(dimensionBizName, metricResps, semanticSchemaResp)) {
DimSchemaResp dimSchemaResp = semanticSchemaResp.getDimension(dimensionBizName);
if (Objects.nonNull(dimSchemaResp) && dimSchemaResp.isPartitionTime()) {
if (Objects.isNull(dimSchemaResp)
|| (Objects.nonNull(dimSchemaResp) && dimSchemaResp.isPartitionTime())) {
continue;
}
String errMsg =

23
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -2,20 +2,13 @@ package com.tencent.supersonic.headless.server.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam;
import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema;
import com.tencent.supersonic.headless.core.translator.calcite.sql.SqlBuilder;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.*;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
@@ -26,7 +19,7 @@ import java.util.List;
class HeadlessParserServiceTest {
public static SqlParserResp parser(S2CalciteSchema semanticSchema,
MetricQueryParam metricQueryParam, boolean isAgg) {
OntologyQueryParam ontologyQueryParam, boolean isAgg) {
SqlParserResp sqlParser = new SqlParserResp();
try {
if (semanticSchema == null) {
@@ -35,14 +28,14 @@ class HeadlessParserServiceTest {
}
SqlBuilder aggBuilder = new SqlBuilder(semanticSchema);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setMetricQueryParam(metricQueryParam);
aggBuilder.build(queryStatement, AggOption.getAggregation(!isAgg));
queryStatement.setOntologyQueryParam(ontologyQueryParam);
aggBuilder.build(queryStatement);
EngineType engineType =
EngineType.fromString(semanticSchema.getOntology().getDatabase().getType());
sqlParser.setSql(aggBuilder.getSql(engineType));
} catch (Exception e) {
sqlParser.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e);
log.error("parser error metricQueryReq[{}] error [{}]", ontologyQueryParam, e);
}
return sqlParser;
}
@@ -161,7 +154,7 @@ class HeadlessParserServiceTest {
// HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
MetricQueryParam metricCommand = new MetricQueryParam();
OntologyQueryParam metricCommand = new OntologyQueryParam();
metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date")));
metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv")));
metricCommand.setWhere(
@@ -174,7 +167,7 @@ class HeadlessParserServiceTest {
addDepartment(semanticSchema);
MetricQueryParam metricCommand2 = new MetricQueryParam();
OntologyQueryParam metricCommand2 = new OntologyQueryParam();
metricCommand2.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date",
"user_name__department", "user_name", "user_name__page")));
metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv")));

3
headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java
View File

@@ -68,8 +68,5 @@ class QueryNLReqBuilderTest {
DateModeUtils dateModeUtils = new DateModeUtils();
mockContextUtils.when(() -> ContextUtils.getBean(DateModeUtils.class))
.thenReturn(dateModeUtils);
dateModeUtils.setSysDateCol("sys_imp_date");
dateModeUtils.setSysDateWeekCol("sys_imp_week");
dateModeUtils.setSysDateMonthCol("sys_imp_month");
}
}

10
launchers/headless/src/main/resources/META-INF/spring.factories
View File

@@ -26,9 +26,10 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\
com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\
com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlVariableParseConverter,\
com.tencent.supersonic.headless.core.translator.converter.CalculateAggConverter,\
com.tencent.supersonic.headless.core.translator.converter.ParserDefaultConverter
com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\
com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\
com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter
com.tencent.supersonic.headless.core.translator.QueryOptimizer=\
com.tencent.supersonic.headless.core.translator.DetailQueryOptimizer
@@ -46,4 +47,5 @@ com.tencent.supersonic.headless.core.cache.QueryCache=\
### headless-server SPIs
com.tencent.supersonic.headless.server.modeller.SemanticModeller=\
com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller
com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller, \
com.tencent.supersonic.headless.server.modeller.LLMSemanticModeller

25
launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2CompanyDemo.java
View File

@@ -7,26 +7,12 @@ import com.tencent.supersonic.chat.server.agent.Agent;
import com.tencent.supersonic.chat.server.agent.AgentToolType;
import com.tencent.supersonic.chat.server.agent.DatasetTool;
import com.tencent.supersonic.chat.server.agent.ToolConfig;
import com.tencent.supersonic.chat.server.processor.execute.DataInterpretProcessor;
import com.tencent.supersonic.common.pojo.ChatApp;
import com.tencent.supersonic.common.pojo.JoinCondition;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AppModule;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.enums.*;
import com.tencent.supersonic.common.util.ChatAppManager;
import com.tencent.supersonic.headless.api.pojo.AggregateTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
@@ -40,11 +26,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.*;
@Component
@Slf4j
@@ -272,7 +254,6 @@ public class S2CompanyDemo extends S2BaseDemo {
Map<String, ChatApp> chatAppConfig =
Maps.newHashMap(ChatAppManager.getAllApps(AppModule.CHAT));
chatAppConfig.values().forEach(app -> app.setChatModelId(demoChatModel.getId()));
chatAppConfig.get(DataInterpretProcessor.APP_KEY).setEnable(true);
agent.setChatAppConfig(chatAppConfig);
agentService.createAgent(agent, defaultUser);

3
launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java
View File

@@ -146,7 +146,8 @@ public class S2VisitsDemo extends S2BaseDemo {
agent.setStatus(1);
agent.setEnableSearch(1);
agent.setExamples(Lists.newArrayList("近15天超音数访问次数汇总", "按部门统计超音数的访问人数", "对比alice和lucy的停留时长",
"过去30天访问次数最高的部门top3", "近1个月总访问次数超过100次的部门有几个", "过去半个月每个核心用户的总停留时长"));
"过去30天访问次数最高的部门top3", "近1个月总访问次数超过100次的部门有几个", "过去半个月每个核心用户的总停留时长",
"今年以来访问次数最高的一天是哪一天"));
// configure tools
ToolConfig toolConfig = new ToolConfig();

7
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -26,9 +26,10 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\
com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\
com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlVariableParseConverter,\
com.tencent.supersonic.headless.core.translator.converter.CalculateAggConverter,\
com.tencent.supersonic.headless.core.translator.converter.ParserDefaultConverter
com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\
com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\
com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter
com.tencent.supersonic.headless.core.translator.QueryOptimizer=\
com.tencent.supersonic.headless.core.translator.DetailQueryOptimizer

1
launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java
View File

@@ -12,7 +12,6 @@ import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.service.ChatModelService;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.util.DataUtils;
import org.springframework.beans.factory.annotation.Autowired;

6
launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java
View File

@@ -20,7 +20,7 @@ public class DetailTest extends BaseTest {
@Test
public void test_detail_dimension() throws Exception {
QueryResult actualResult = submitNewChat("周杰伦流派和代表作", DataUtils.tagAgentId);
QueryResult actualResult = submitNewChat("周杰伦流派和代表作", DataUtils.singerAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
@@ -31,7 +31,7 @@ public class DetailTest extends BaseTest {
expectedParseInfo.setAggType(AggregateTypeEnum.NONE);
QueryFilter dimensionFilter =
DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 8L);
DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 17L);
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
expectedParseInfo.getDimensions()
@@ -43,7 +43,7 @@ public class DetailTest extends BaseTest {
@Test
public void test_detail_filter() throws Exception {
QueryResult actualResult = submitNewChat("国风歌手", DataUtils.tagAgentId);
QueryResult actualResult = submitNewChat("国风歌手", DataUtils.singerAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();

33
launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java
View File

@@ -9,6 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.chat.query.rule.metric.MetricFilterQuery;
import com.tencent.supersonic.headless.chat.query.rule.metric.MetricGroupByQuery;
import com.tencent.supersonic.headless.chat.query.rule.metric.MetricModelQuery;
import com.tencent.supersonic.headless.chat.query.rule.metric.MetricTopNQuery;
import com.tencent.supersonic.util.DataUtils;
import org.junit.jupiter.api.Order;
@@ -28,13 +29,28 @@ import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
public class MetricTest extends BaseTest {
@Test
public void testMetric() throws Exception {
QueryResult actualResult = submitNewChat("超音数 访问次数", DataUtils.metricAgentId);
public void testMetricModel() throws Exception {
QueryResult actualResult = submitNewChat("超音数 访问次数", DataUtils.productAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
expectedResult.setChatContext(expectedParseInfo);
expectedResult.setQueryMode(MetricModelQuery.QUERY_MODE);
expectedParseInfo.setAggType(NONE);
expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数"));
expectedParseInfo.setDateInfo(
DataUtils.getDateConf(DateConf.DateMode.BETWEEN, unit, period, startDay, endDay));
expectedParseInfo.setQueryType(QueryType.AGGREGATE);
assertQueryResult(expectedResult, actualResult);
assert actualResult.getQueryResults().size() == 1;
}
@Test
public void testMetricFilter() throws Exception {
QueryResult actualResult = submitNewChat("alice的访问次数", DataUtils.metricAgentId);
QueryResult actualResult = submitNewChat("alice的访问次数", DataUtils.productAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
@@ -57,7 +73,8 @@ public class MetricTest extends BaseTest {
@Test
public void testMetricGroupBy() throws Exception {
QueryResult actualResult = submitNewChat("近7天超音数各部门的访问次数", DataUtils.metricAgentId);
System.setProperty("s2.test", "true");
QueryResult actualResult = submitNewChat("近7天超音数各部门的访问次数", DataUtils.productAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
@@ -79,7 +96,7 @@ public class MetricTest extends BaseTest {
@Test
public void testMetricFilterCompare() throws Exception {
QueryResult actualResult = submitNewChat("对比alice和lucy的访问次数", DataUtils.metricAgentId);
QueryResult actualResult = submitNewChat("对比alice和lucy的访问次数", DataUtils.productAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
@@ -107,7 +124,7 @@ public class MetricTest extends BaseTest {
@Test
@Order(3)
public void testMetricTopN() throws Exception {
QueryResult actualResult = submitNewChat("近3天访问次数最多的用户", DataUtils.metricAgentId);
QueryResult actualResult = submitNewChat("近3天访问次数最多的用户", DataUtils.productAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
@@ -128,7 +145,7 @@ public class MetricTest extends BaseTest {
@Test
public void testMetricGroupBySum() throws Exception {
QueryResult actualResult = submitNewChat("近7天超音数各部门的访问次数总和", DataUtils.metricAgentId);
QueryResult actualResult = submitNewChat("近7天超音数各部门的访问次数总和", DataUtils.productAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
expectedResult.setChatContext(expectedParseInfo);
@@ -154,7 +171,7 @@ public class MetricTest extends BaseTest {
String dateStr = textFormat.format(format.parse(startDay));
QueryResult actualResult =
submitNewChat(String.format("alice在%s的访问次数", dateStr), DataUtils.metricAgentId);
submitNewChat(String.format("alice在%s的访问次数", dateStr), DataUtils.productAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();

1
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java
View File

@@ -20,6 +20,7 @@ public class QueryByMetricTest extends BaseTest {
@Test
public void testWithMetricAndDimensionBizNames() throws Exception {
System.setProperty("s2.test", "true");
QueryMetricReq queryMetricReq = new QueryMetricReq();
queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));

2
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java
View File

@@ -46,6 +46,7 @@ public class QueryByStructTest extends BaseTest {
@Test
public void testDetailQuery() throws Exception {
System.setProperty("s2.test", "true");
QueryStructReq queryStructReq =
buildQueryStructReq(Arrays.asList("user_name", "department"), QueryType.DETAIL);
SemanticQueryResp semanticQueryResp =
@@ -86,6 +87,7 @@ public class QueryByStructTest extends BaseTest {
@Test
public void testFilterQuery() throws Exception {
System.setProperty("s2.test", "true");
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"));
List<Filter> dimensionFilters = new ArrayList<>();
Filter filter = new Filter();

8
launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -15,10 +15,10 @@ import static java.time.LocalDate.now;
public class DataUtils {
public static final Integer metricAgentId = 1;
public static final Integer tagAgentId = 2;
public static final Integer productAgentId = 1;
public static final Integer companyAgentId = 2;
public static final Integer singerAgentId = 3;
public static final Integer ONE_TURNS_CHAT_ID = 10;
public static final Integer MULTI_TURNS_CHAT_ID = 11;
private static final User user_test = User.getDefaultUser();
public static User getUser() {
@@ -40,7 +40,7 @@ public class DataUtils {
public static ChatParseReq getChatParseReq(Integer id, String query, boolean enableLLM) {
ChatParseReq chatParseReq = new ChatParseReq();
chatParseReq.setQueryText(query);
chatParseReq.setAgentId(metricAgentId);
chatParseReq.setAgentId(productAgentId);
chatParseReq.setChatId(id);
chatParseReq.setUser(user_test);
chatParseReq.setDisableLLM(!enableLLM);

2
launchers/standalone/src/test/resources/s2-config.yaml
View File

@@ -21,7 +21,7 @@ s2:
date: true
demo:
names: S2VisitsDemo,S2SingerDemo
names: S2VisitsDemo,S2SingerDemo,S2CompanyDemo
enableLLM: false
authentication: