topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 04:57:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless)(Chat) Change View to DataSet (#782)

Browse Source 
* (improvement)(Headless)(Chat) Change view to dataSet



---------

Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
LXW
2024-03-04 11:48:41 +08:00
committed by GitHub
parent b29e429271
commit a41da3f5fe
184 changed files with 1628 additions and 1532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DataSetInfoStat.java Normal file
View File

@@ -0,0 +1,22 @@
package com.tencent.supersonic.headless.core.knowledge;
import lombok.Builder;
import lombok.Data;
import lombok.ToString;
import java.io.Serializable;
@Data
@ToString
@Builder
public class DataSetInfoStat implements Serializable {
private long dataSetCount;
private long metricDataSetCount;
private long dimensionDataSetCount;
private long dimensionValueDataSetCount;
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java
View File

@@ -58,7 +58,7 @@ public class SearchService {
.collect(Collectors.toList());
for (HanlpMapResult hanlpMapResult : hanlpMapResults) {
List<String> natures = hanlpMapResult.getNatures().stream()
.map(nature -> NatureHelper.changeModel2View(nature, modelIdToViewIds))
.map(nature -> NatureHelper.changeModel2DataSet(nature, modelIdToViewIds))
.flatMap(Collection::stream).collect(Collectors.toList());
hanlpMapResult.setNatures(natures);
}

22
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/ViewInfoStat.java
View File

@@ -1,22 +0,0 @@
package com.tencent.supersonic.headless.core.knowledge;
import lombok.Builder;
import lombok.Data;
import lombok.ToString;
import java.io.Serializable;
@Data
@ToString
@Builder
public class ViewInfoStat implements Serializable {
private long viewCount;
private long metricViewCount;
private long dimensionViewCount;
private long dimensionValueViewCount;
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java
View File

@@ -21,13 +21,13 @@ public class ModelWordBuilder extends BaseWordBuilder {
public List<DictWord> doGet(String word, SchemaElement schemaElement) {
List<DictWord> result = Lists.newArrayList();
//modelName
DictWord dictWord = buildDictWord(word, schemaElement.getView());
DictWord dictWord = buildDictWord(word, schemaElement.getDataSet());
result.add(dictWord);
//alias
List<String> aliasList = schemaElement.getAlias();
if (CollectionUtils.isNotEmpty(aliasList)) {
for (String alias : aliasList) {
result.add(buildDictWord(alias, schemaElement.getView()));
result.add(buildDictWord(alias, schemaElement.getDataSet()));
}
}
return result;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java
View File

@@ -224,7 +224,7 @@ public class HanlpHelper {
public static List<S2Term> transform2ApiTerm(Term term, Map<Long, List<Long>> modelIdToViewIds) {
List<S2Term> s2Terms = Lists.newArrayList();
List<String> natures = NatureHelper.changeModel2View(String.valueOf(term.getNature()), modelIdToViewIds);
List<String> natures = NatureHelper.changeModel2DataSet(String.valueOf(term.getNature()), modelIdToViewIds);
for (String nature : natures) {
S2Term s2Term = new S2Term();
BeanUtils.copyProperties(term, s2Term);

56
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java
View File

@@ -5,7 +5,7 @@ import com.hankcs.hanlp.corpus.tag.Nature;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat;
import com.tencent.supersonic.headless.core.knowledge.DataSetInfoStat;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@@ -41,7 +41,7 @@ public class NatureHelper {
result = SchemaElementType.ENTITY;
break;
case VIEW:
result = SchemaElementType.VIEW;
result = SchemaElementType.DATASET;
break;
case VALUE:
result = SchemaElementType.VALUE;
@@ -55,12 +55,12 @@ public class NatureHelper {
return result;
}
private static boolean isViewOrEntity(S2Term term, Integer model) {
private static boolean isDataSetOrEntity(S2Term term, Integer model) {
return (DictWordType.NATURE_SPILT + model).equals(term.nature.toString()) || term.nature.toString()
.endsWith(DictWordType.ENTITY.getType());
}
public static Integer getViewByNature(Nature nature) {
public static Integer getDataSetByNature(Nature nature) {
if (nature.startsWith(DictWordType.NATURE_SPILT)) {
String[] dimensionValues = nature.toString().split(DictWordType.NATURE_SPILT);
if (StringUtils.isNumeric(dimensionValues[1])) {
@@ -70,7 +70,7 @@ public class NatureHelper {
return 0;
}
public static Long getViewId(String nature) {
public static Long getDataSetId(String nature) {
try {
String[] split = nature.split(DictWordType.NATURE_SPILT);
if (split.length <= 1) {
@@ -96,13 +96,13 @@ public class NatureHelper {
return null;
}
private static Nature changeModel2View(String nature, Long viewId) {
private static Nature changeModel2DataSet(String nature, Long dataSetId) {
try {
String[] split = nature.split(DictWordType.NATURE_SPILT);
if (split.length <= 1) {
return null;
}
split[1] = String.valueOf(viewId);
split[1] = String.valueOf(dataSetId);
return Nature.create(StringUtils.join(split, DictWordType.NATURE_SPILT));
} catch (NumberFormatException e) {
log.error("", e);
@@ -110,17 +110,17 @@ public class NatureHelper {
return null;
}
public static List<String> changeModel2View(String nature, Map<Long, List<Long>> modelIdToViewIds) {
public static List<String> changeModel2DataSet(String nature, Map<Long, List<Long>> modelIdToDataSetIds) {
Long modelId = getModelId(nature);
List<Long> viewIds = modelIdToViewIds.get(modelId);
if (CollectionUtils.isEmpty(viewIds)) {
List<Long> dataSetIds = modelIdToDataSetIds.get(modelId);
if (CollectionUtils.isEmpty(dataSetIds)) {
return Lists.newArrayList();
}
return viewIds.stream().map(viewId -> String.valueOf(changeModel2View(nature, viewId)))
return dataSetIds.stream().map(dataSetId -> String.valueOf(changeModel2DataSet(nature, dataSetId)))
.collect(Collectors.toList());
}
public static boolean isDimensionValueViewId(String nature) {
public static boolean isDimensionValueDataSetId(String nature) {
if (StringUtils.isEmpty(nature)) {
return false;
}
@@ -135,21 +135,21 @@ public class NatureHelper {
&& StringUtils.isNumeric(split[1]);
}
public static ViewInfoStat getViewStat(List<S2Term> terms) {
return ViewInfoStat.builder()
.viewCount(getViewCount(terms))
.dimensionViewCount(getDimensionCount(terms))
.metricViewCount(getMetricCount(terms))
.dimensionValueViewCount(getDimensionValueCount(terms))
public static DataSetInfoStat getDataSetStat(List<S2Term> terms) {
return DataSetInfoStat.builder()
.dataSetCount(getDataSetCount(terms))
.dimensionDataSetCount(getDimensionCount(terms))
.metricDataSetCount(getMetricCount(terms))
.dimensionValueDataSetCount(getDimensionValueCount(terms))
.build();
}
private static long getViewCount(List<S2Term> terms) {
return terms.stream().filter(term -> isViewOrEntity(term, getViewByNature(term.nature))).count();
private static long getDataSetCount(List<S2Term> terms) {
return terms.stream().filter(term -> isDataSetOrEntity(term, getDataSetByNature(term.nature))).count();
}
private static long getDimensionValueCount(List<S2Term> terms) {
return terms.stream().filter(term -> isDimensionValueViewId(term.nature.toString())).count();
return terms.stream().filter(term -> isDimensionValueDataSetId(term.nature.toString())).count();
}
private static long getDimensionCount(List<S2Term> terms) {
@@ -169,13 +169,13 @@ public class NatureHelper {
* @param terms
* @return
*/
public static Map<Long, Map<DictWordType, Integer>> getViewToNatureStat(List<S2Term> terms) {
public static Map<Long, Map<DictWordType, Integer>> getDataSetToNatureStat(List<S2Term> terms) {
Map<Long, Map<DictWordType, Integer>> modelToNature = new HashMap<>();
terms.stream().filter(
term -> term.nature.startsWith(DictWordType.NATURE_SPILT)
).forEach(term -> {
DictWordType dictWordType = DictWordType.getNatureType(String.valueOf(term.nature));
Long model = getViewId(String.valueOf(term.nature));
Long model = getDataSetId(String.valueOf(term.nature));
Map<DictWordType, Integer> natureTypeMap = new HashMap<>();
natureTypeMap.put(dictWordType, 1);
@@ -196,15 +196,15 @@ public class NatureHelper {
return modelToNature;
}
public static List<Long> selectPossibleViews(List<S2Term> terms) {
Map<Long, Map<DictWordType, Integer>> modelToNatureStat = getViewToNatureStat(terms);
Integer maxViewTypeSize = modelToNatureStat.entrySet().stream()
public static List<Long> selectPossibleDataSets(List<S2Term> terms) {
Map<Long, Map<DictWordType, Integer>> modelToNatureStat = getDataSetToNatureStat(terms);
Integer maxDataSetTypeSize = modelToNatureStat.entrySet().stream()
.max(Comparator.comparingInt(o -> o.getValue().size())).map(entry -> entry.getValue().size())
.orElse(null);
if (Objects.isNull(maxViewTypeSize) || maxViewTypeSize == 0) {
if (Objects.isNull(maxDataSetTypeSize) || maxDataSetTypeSize == 0) {
return new ArrayList<>();
}
return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxViewTypeSize)
return modelToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxDataSetTypeSize)
.map(entry -> entry.getKey()).collect(Collectors.toList());
}

59
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java
View File

@@ -9,18 +9,19 @@ import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Component
@Slf4j
@Primary
@@ -28,8 +29,8 @@ public class DefaultQueryParser implements QueryParser {
public void parse(QueryStatement queryStatement) throws Exception {
QueryParam queryParam = queryStatement.getQueryParam();
if (Objects.isNull(queryStatement.getViewQueryParam())) {
queryStatement.setViewQueryParam(new ViewQueryParam());
if (Objects.isNull(queryStatement.getDataSetQueryParam())) {
queryStatement.setDataSetQueryParam(new DataSetQueryParam());
}
if (Objects.isNull(queryStatement.getMetricQueryParam())) {
queryStatement.setMetricQueryParam(new MetricQueryParam());
@@ -41,10 +42,10 @@ public class DefaultQueryParser implements QueryParser {
headlessConverter.convert(queryStatement);
}
}
log.info("SemanticConverter after {} {} {}", queryParam, queryStatement.getViewQueryParam(),
log.info("SemanticConverter after {} {} {}", queryParam, queryStatement.getDataSetQueryParam(),
queryStatement.getMetricQueryParam());
if (!queryStatement.getViewQueryParam().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getViewQueryParam(), queryStatement);
if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getDataSetQueryParam(), queryStatement);
} else {
queryStatement.getMetricQueryParam().setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement);
@@ -61,37 +62,37 @@ public class DefaultQueryParser implements QueryParser {
queryStatement.setSql(querySql);
}
public QueryStatement parser(ViewQueryParam viewQueryParam, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", viewQueryParam);
public QueryStatement parser(DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", dataSetQueryParam);
try {
if (!CollectionUtils.isEmpty(viewQueryParam.getTables())) {
if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) {
List<String[]> tables = new ArrayList<>();
Boolean isSingleTable = viewQueryParam.getTables().size() == 1;
for (MetricTable metricTable : viewQueryParam.getTables()) {
QueryStatement tableSql = parserSql(metricTable, isSingleTable, viewQueryParam, queryStatement);
if (isSingleTable && Objects.nonNull(tableSql.getViewSimplifySql())
&& !tableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(tableSql.getViewSimplifySql());
queryStatement.setViewQueryParam(viewQueryParam);
Boolean isSingleTable = dataSetQueryParam.getTables().size() == 1;
for (MetricTable metricTable : dataSetQueryParam.getTables()) {
QueryStatement tableSql = parserSql(metricTable, isSingleTable, dataSetQueryParam, queryStatement);
if (isSingleTable && Objects.nonNull(tableSql.getDataSetQueryParam())
&& !tableSql.getDataSetSimplifySql().isEmpty()) {
queryStatement.setSql(tableSql.getDataSetSimplifySql());
queryStatement.setDataSetQueryParam(dataSetQueryParam);
return queryStatement;
}
tables.add(new String[]{metricTable.getAlias(), tableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql = "";
if (viewQueryParam.isSupportWith()) {
if (dataSetQueryParam.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + viewQueryParam.getSql();
Collectors.toList())) + "\n" + dataSetQueryParam.getSql();
} else {
sql = viewQueryParam.getSql();
sql = dataSetQueryParam.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (viewQueryParam.isWithAlias() ? "" : tb[0]), -1);
"(" + tb[1] + ") " + (dataSetQueryParam.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setViewQueryParam(viewQueryParam);
queryStatement.setDataSetQueryParam(dataSetQueryParam);
return queryStatement;
}
}
@@ -119,7 +120,7 @@ public class DefaultQueryParser implements QueryParser {
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable,
ViewQueryParam viewQueryParam,
DataSetQueryParam dataSetQueryParam,
QueryStatement queryStatement) throws Exception {
MetricQueryParam metricReq = new MetricQueryParam();
metricReq.setMetrics(metricTable.getMetrics());
@@ -131,11 +132,11 @@ public class DefaultQueryParser implements QueryParser {
tableSql.setMetricQueryParam(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setViewId(queryStatement.getViewId());
tableSql.setDataSetId(queryStatement.getDataSetId());
tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) {
tableSql.setViewSql(viewQueryParam.getSql());
tableSql.setViewAlias(metricTable.getAlias());
tableSql.setDataSetSql(dataSetQueryParam.getSql());
tableSql.setDataSetAlias(metricTable.getAlias());
}
tableSql = parser(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java
View File

@@ -37,14 +37,14 @@ public class CalciteSqlParser implements SqlParser {
queryStatement.setSql(aggBuilder.getSql(engineType));
queryStatement.setSourceId(aggBuilder.getSourceId());
if (Objects.nonNull(queryStatement.getEnableOptimize()) && queryStatement.getEnableOptimize()
&& Objects.nonNull(queryStatement.getViewAlias()) && !queryStatement.getViewAlias().isEmpty()) {
&& Objects.nonNull(queryStatement.getDataSetAlias()) && !queryStatement.getDataSetAlias().isEmpty()) {
// simplify model sql with query sql
String simplifySql = aggBuilder.simplify(
getSqlByView(aggBuilder.getSql(engineType), queryStatement.getViewSql(),
queryStatement.getViewAlias()), engineType);
getSqlByDataSet(aggBuilder.getSql(engineType), queryStatement.getDataSetSql(),
queryStatement.getDataSetAlias()), engineType);
if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) {
log.debug("simplifySql [{}]", simplifySql);
queryStatement.setViewSimplifySql(simplifySql);
queryStatement.setDataSetSimplifySql(simplifySql);
}
}
return queryStatement;
@@ -62,7 +62,7 @@ public class CalciteSqlParser implements SqlParser {
return semanticSchema;
}
private String getSqlByView(String sql, String viewSql, String viewAlias) {
return String.format("with %s as (%s) %s", viewAlias, sql, viewSql);
private String getSqlByDataSet(String sql, String dataSetSql, String dataSetAlias) {
return String.format("with %s as (%s) %s", dataSetAlias, sql, dataSetSql);
}
}

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java
View File

@@ -53,16 +53,16 @@ public class SchemaBuilder {
.withRowCount(1)
.build();
schema.add(MATERIALIZATION_SYS_SOURCE, srcTable);
DataSourceTable viewTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW)
DataSourceTable dataSetTable = DataSourceTable.newBuilder(MATERIALIZATION_SYS_VIEW)
.addField(MATERIALIZATION_SYS_FIELD_DATE, SqlTypeName.DATE)
.addField(MATERIALIZATION_SYS_FIELD_DATA, SqlTypeName.BIGINT)
.withRowCount(1)
.build();
schema.add(MATERIALIZATION_SYS_VIEW, viewTable);
schema.add(MATERIALIZATION_SYS_VIEW, dataSetTable);
return rootSchema;
}
public static void addSourceView(CalciteSchema viewSchema, String dbSrc, String tbSrc, Set<String> dates,
public static void addSourceView(CalciteSchema dataSetSchema, String dbSrc, String tbSrc, Set<String> dates,
Set<String> dimensions, Set<String> metrics) {
String tb = tbSrc.toLowerCase();
String db = dbSrc.toLowerCase();
@@ -80,14 +80,14 @@ public class SchemaBuilder {
.withRowCount(1)
.build();
if (Objects.nonNull(db) && !db.isEmpty()) {
SchemaPlus schemaPlus = viewSchema.plus().getSubSchema(db);
SchemaPlus schemaPlus = dataSetSchema.plus().getSubSchema(db);
if (Objects.isNull(schemaPlus)) {
viewSchema.plus().add(db, new AbstractSchema());
schemaPlus = viewSchema.plus().getSubSchema(db);
dataSetSchema.plus().add(db, new AbstractSchema());
schemaPlus = dataSetSchema.plus().getSubSchema(db);
}
schemaPlus.add(tb, srcTable);
} else {
viewSchema.add(tb, srcTable);
dataSetSchema.add(tb, srcTable);
}
}
}

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/ViewExpanderImpl.java
View File

@@ -1,10 +1,11 @@
package com.tencent.supersonic.headless.core.parser.calcite.schema;
import java.util.List;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.type.RelDataType;
import java.util.List;
/**
* customize the ViewExpander
*/
@@ -14,7 +15,7 @@ public class ViewExpanderImpl implements RelOptTable.ViewExpander {
@Override
public RelRoot expandView(RelDataType rowType, String queryString, List<String> schemaPath,
List<String> viewPath) {
List<String> dataSetPath) {
return null;
}
}

27
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
@@ -13,6 +12,18 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -24,16 +35,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.springframework.util.CollectionUtils;
@Slf4j
public class DataSourceNode extends SemanticNode {
@@ -126,9 +127,9 @@ public class DataSourceNode extends SemanticNode {
return build(datasource, scope);
}
EngineType engineType = EngineType.fromString(datasource.getType());
SqlNode view = new SqlBasicCall(new LateralViewExplodeNode(), Arrays.asList(build(datasource, scope),
SqlNode dataSet = new SqlBasicCall(new LateralViewExplodeNode(), Arrays.asList(build(datasource, scope),
new SqlNodeList(getExtendField(exprList, scope, engineType), SqlParserPos.ZERO)), SqlParserPos.ZERO);
return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, view);
return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, dataSet);
}
public static List<SqlNode> getExtendField(Set<String> exprList, SqlValidatorScope scope, EngineType engineType)

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java
View File

@@ -12,11 +12,11 @@ import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlWriter;
/**
* extend node to handle lateral explode view
* extend node to handle lateral explode dataSet
*/
public class LateralViewExplodeNode extends ExtendNode {
public final String sqlNameView = "view";
public final String sqlNameView = "dataSet";
public final String sqlNameExplode = "explode";
public LateralViewExplodeNode() {

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java
View File

@@ -34,7 +34,7 @@ import java.util.UUID;
import java.util.stream.Collectors;
/**
* process the table view from the defined data model schema
* process the table dataSet from the defined data model schema
*/
@Slf4j
public class SourceRender extends Renderer {

23
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -42,14 +42,15 @@ public class CalculateAggConverter implements HeadlessConverter {
String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql);
}
public ViewQueryParam generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version)
public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement,
EngineType engineTypeEnum, String version)
throws Exception {
QueryParam queryParam = queryStatement.getQueryParam();
// 同环比
if (isRatioAccept(queryParam)) {
return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
}
ViewQueryParam sqlCommand = new ViewQueryParam();
DataSetQueryParam sqlCommand = new DataSetQueryParam();
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);
@@ -105,13 +106,13 @@ public class CalculateAggConverter implements HeadlessConverter {
@Override
public void convert(QueryStatement queryStatement) throws Exception {
ViewQueryParam sqlCommend = queryStatement.getViewQueryParam();
DataSetQueryParam sqlCommend = queryStatement.getDataSetQueryParam();
Database database = queryStatement.getSemanticModel().getDatabase();
ViewQueryParam viewQueryParam = generateSqlCommend(queryStatement,
DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement,
EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
sqlCommend.setSql(viewQueryParam.getSql());
sqlCommend.setTables(viewQueryParam.getTables());
sqlCommend.setSupportWith(viewQueryParam.isSupportWith());
sqlCommend.setSql(dataSetQueryParam.getSql());
sqlCommend.setTables(dataSetQueryParam.getTables());
sqlCommend.setSupportWith(dataSetQueryParam.isSupportWith());
}
/**
@@ -128,13 +129,13 @@ public class CalculateAggConverter implements HeadlessConverter {
return false;
}
public ViewQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
String version)
public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
String version)
throws Exception {
QueryParam queryParam = queryStatement.getQueryParam();
check(queryParam);
queryStatement.setEnableOptimize(false);
ViewQueryParam sqlCommand = new ViewQueryParam();
DataSetQueryParam sqlCommand = new DataSetQueryParam();
String metricTableName = "v_metric_tb_tmp";
MetricTable metricTable = new MetricTable();
metricTable.setAlias(metricTableName);

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java
View File

@@ -27,8 +27,8 @@ public class DefaultDimValueConverter implements HeadlessConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
return !Objects.isNull(queryStatement.getViewQueryParam())
&& !StringUtils.isBlank(queryStatement.getViewQueryParam().getSql());
return !Objects.isNull(queryStatement.getDataSetQueryParam())
&& !StringUtils.isBlank(queryStatement.getDataSetQueryParam().getSql());
}
@Override
@@ -39,14 +39,14 @@ public class DefaultDimValueConverter implements HeadlessConverter {
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
String sql = queryStatement.getViewQueryParam().getSql();
String sql = queryStatement.getDataSetQueryParam().getSql();
List<String> whereFields = SqlSelectHelper.getWhereFields(sql)
.stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) {
return;
}
MetricTable metricTable = queryStatement.getViewQueryParam()
MetricTable metricTable = queryStatement.getDataSetQueryParam()
.getTables().stream().findFirst().orElse(null);
List<Expression> expressions = Lists.newArrayList();
for (Dimension dimension : dimensions) {
@@ -63,7 +63,7 @@ public class DefaultDimValueConverter implements HeadlessConverter {
}
}
sql = SqlAddHelper.addWhere(sql, expressions);
queryStatement.getViewQueryParam().setSql(sql);
queryStatement.getDataSetQueryParam().setSql(sql);
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java
View File

@@ -6,7 +6,7 @@ import lombok.Data;
import java.util.List;
@Data
public class ViewQueryParam {
public class DataSetQueryParam {
private String sql = "";
private List<MetricTable> tables;
private boolean supportWith = true;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java
View File

@@ -6,9 +6,9 @@ import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -11,7 +11,7 @@ import java.util.List;
@Data
public class QueryStatement {
private Long viewId;
private Long dataSetId;
private List<Long> modelIds;
private String sql = "";
private String sourceId = "";
@@ -19,15 +19,15 @@ public class QueryStatement {
private Boolean ok;
private QueryParam queryParam;
private MetricQueryParam metricQueryParam;
private ViewQueryParam viewQueryParam;
private DataSetQueryParam dataSetQueryParam;
private Integer status = 0;
private Boolean isS2SQL = false;
private List<ImmutablePair<String, String>> timeRanges;
private Boolean enableOptimize = true;
private Triple<String, String, String> minMaxTime;
private String viewSql = "";
private String viewAlias = "";
private String viewSimplifySql = "";
private String dataSetSql = "";
private String dataSetAlias = "";
private String dataSetSimplifySql = "";
private Boolean enableLimitWrapper = false;
private SemanticModel semanticModel;