topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 20:51:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Clean code logic of headless translator.

Browse Source
This commit is contained in:
jerryjzhang
2024-11-21 09:09:24 +08:00
parent d7586a5d3b
commit be05f977d5
9 changed files with 52 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Ontology.java
View File

@@ -14,7 +14,7 @@ import java.util.stream.Collectors;
public class Ontology { public class Ontology {
private List<Metric> metrics = new ArrayList<>(); private List<Metric> metrics = new ArrayList<>();
private Map<String, DataModel> datasourceMap = new HashMap<>(); private Map<String, DataModel> dataModelMap = new HashMap<>();
private Map<String, List<Dimension>> dimensionMap = new HashMap<>(); private Map<String, List<Dimension>> dimensionMap = new HashMap<>();
private List<Materialization> materializationList = new ArrayList<>(); private List<Materialization> materializationList = new ArrayList<>();
private List<JoinRelation> joinRelations; private List<JoinRelation> joinRelations;
@@ -26,7 +26,7 @@ public class Ontology {
} }
public Map<Long, DataModel> getModelMap() { public Map<Long, DataModel> getModelMap() {
return datasourceMap.values().stream() return dataModelMap.values().stream()
.collect(Collectors.toMap(DataModel::getId, dataSource -> dataSource)); .collect(Collectors.toMap(DataModel::getId, dataSource -> dataSource));
} }
} }

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2CalciteSchema.java
View File

@@ -29,11 +29,15 @@ public class S2CalciteSchema extends AbstractSchema {
return this; return this;
} }
public Map<String, DataModel> getDatasource() { public Map<String, DataModel> getDataModels() {
return ontology.getDatasourceMap(); return ontology.getDataModelMap();
} }
public Map<String, List<Dimension>> getDimension() { public List<Metric> getMetrics() {
return ontology.getMetrics();
}
public Map<String, List<Dimension>> getDimensions() {
return ontology.getDimensionMap(); return ontology.getDimensionMap();
} }
@@ -41,8 +45,4 @@ public class S2CalciteSchema extends AbstractSchema {
return ontology.getJoinRelations(); return ontology.getJoinRelations();
} }
public List<Metric> getMetrics() {
return ontology.getMetrics();
}
} }

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java
View File

@@ -32,8 +32,8 @@ import java.util.Objects;
@Slf4j @Slf4j
public class SqlBuilder { public class SqlBuilder {
private MetricQueryParam metricQueryParam;
private final S2CalciteSchema schema; private final S2CalciteSchema schema;
private MetricQueryParam metricQueryParam;
private SqlValidatorScope scope; private SqlValidatorScope scope;
private SqlNode parserNode; private SqlNode parserNode;
private boolean isAgg = false; private boolean isAgg = false;

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java
View File

@@ -202,7 +202,7 @@ public class DataModelNode extends SemanticNode {
DataModel baseDataModel = null; DataModel baseDataModel = null;
// one , match measure count // one , match measure count
Map<String, Integer> dataSourceMeasures = new HashMap<>(); Map<String, Integer> dataSourceMeasures = new HashMap<>();
for (Map.Entry<String, DataModel> entry : schema.getDatasource().entrySet()) { for (Map.Entry<String, DataModel> entry : schema.getDataModels().entrySet()) {
Set<String> sourceMeasure = entry.getValue().getMeasures().stream() Set<String> sourceMeasure = entry.getValue().getMeasures().stream()
.map(mm -> mm.getName()).collect(Collectors.toSet()); .map(mm -> mm.getName()).collect(Collectors.toSet());
sourceMeasure.retainAll(measures); sourceMeasure.retainAll(measures);
@@ -212,7 +212,7 @@ public class DataModelNode extends SemanticNode {
Optional<Map.Entry<String, Integer>> base = dataSourceMeasures.entrySet().stream() Optional<Map.Entry<String, Integer>> base = dataSourceMeasures.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst(); .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst();
if (base.isPresent()) { if (base.isPresent()) {
baseDataModel = schema.getDatasource().get(base.get().getKey()); baseDataModel = schema.getDataModels().get(base.get().getKey());
dataModels.add(baseDataModel); dataModels.add(baseDataModel);
} }
// second , check match all dimension and metric // second , check match all dimension and metric
@@ -223,8 +223,8 @@ public class DataModelNode extends SemanticNode {
Set<String> dimension = baseDataModel.getDimensions().stream().map(dd -> dd.getName()) Set<String> dimension = baseDataModel.getDimensions().stream().map(dd -> dd.getName())
.collect(Collectors.toSet()); .collect(Collectors.toSet());
baseDataModel.getIdentifiers().stream().forEach(i -> dimension.add(i.getName())); baseDataModel.getIdentifiers().stream().forEach(i -> dimension.add(i.getName()));
if (schema.getDimension().containsKey(baseDataModel.getName())) { if (schema.getDimensions().containsKey(baseDataModel.getName())) {
schema.getDimension().get(baseDataModel.getName()).stream() schema.getDimensions().get(baseDataModel.getName()).stream()
.forEach(d -> dimension.add(d.getName())); .forEach(d -> dimension.add(d.getName()));
} }
filterMeasure.addAll(sourceMeasure); filterMeasure.addAll(sourceMeasure);
@@ -319,8 +319,8 @@ public class DataModelNode extends SemanticNode {
} }
boolean isMatch = false; boolean isMatch = false;
boolean isRight = before.contains(joinRelation.getLeft()); boolean isRight = before.contains(joinRelation.getLeft());
DataModel other = isRight ? schema.getDatasource().get(joinRelation.getRight()) DataModel other = isRight ? schema.getDataModels().get(joinRelation.getRight())
: schema.getDatasource().get(joinRelation.getLeft()); : schema.getDataModels().get(joinRelation.getLeft());
if (!queryDimension.isEmpty()) { if (!queryDimension.isEmpty()) {
Set<String> linkDimension = other.getDimensions().stream() Set<String> linkDimension = other.getDimensions().stream()
.map(dd -> dd.getName()).collect(Collectors.toSet()); .map(dd -> dd.getName()).collect(Collectors.toSet());
@@ -336,8 +336,8 @@ public class DataModelNode extends SemanticNode {
if (!linkMeasure.isEmpty()) { if (!linkMeasure.isEmpty()) {
isMatch = true; isMatch = true;
} }
if (!isMatch && schema.getDimension().containsKey(other.getName())) { if (!isMatch && schema.getDimensions().containsKey(other.getName())) {
Set<String> linkDimension = schema.getDimension().get(other.getName()).stream() Set<String> linkDimension = schema.getDimensions().get(other.getName()).stream()
.map(dd -> dd.getName()).collect(Collectors.toSet()); .map(dd -> dd.getName()).collect(Collectors.toSet());
linkDimension.retainAll(queryDimension); linkDimension.retainAll(queryDimension);
if (!linkDimension.isEmpty()) { if (!linkDimension.isEmpty()) {
@@ -362,7 +362,7 @@ public class DataModelNode extends SemanticNode {
} }
} }
orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> { orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> {
linkDataModels.add(schema.getDatasource().get(d.getKey())); linkDataModels.add(schema.getDataModels().get(d.getKey()));
}); });
} }
return linkDataModels; return linkDataModels;
@@ -388,7 +388,7 @@ public class DataModelNode extends SemanticNode {
S2CalciteSchema schema) { S2CalciteSchema schema) {
Set<String> linkDataSourceName = new HashSet<>(); Set<String> linkDataSourceName = new HashSet<>();
List<DataModel> linkDataModels = new ArrayList<>(); List<DataModel> linkDataModels = new ArrayList<>();
for (Map.Entry<String, DataModel> entry : schema.getDatasource().entrySet()) { for (Map.Entry<String, DataModel> entry : schema.getDataModels().entrySet()) {
if (entry.getKey().equalsIgnoreCase(baseDataModel.getName())) { if (entry.getKey().equalsIgnoreCase(baseDataModel.getName())) {
continue; continue;
} }
@@ -419,7 +419,7 @@ public class DataModelNode extends SemanticNode {
} }
} }
} }
for (Map.Entry<String, List<Dimension>> entry : schema.getDimension().entrySet()) { for (Map.Entry<String, List<Dimension>> entry : schema.getDimensions().entrySet()) {
if (!queryDimension.isEmpty()) { if (!queryDimension.isEmpty()) {
Set<String> linkDimension = entry.getValue().stream().map(dd -> dd.getName()) Set<String> linkDimension = entry.getValue().stream().map(dd -> dd.getName())
.collect(Collectors.toSet()); .collect(Collectors.toSet());
@@ -430,7 +430,7 @@ public class DataModelNode extends SemanticNode {
} }
} }
for (String linkName : linkDataSourceName) { for (String linkName : linkDataSourceName) {
linkDataModels.add(schema.getDatasource().get(linkName)); linkDataModels.add(schema.getDataModels().get(linkName));
} }
if (!CollectionUtils.isEmpty(linkDataModels)) { if (!CollectionUtils.isEmpty(linkDataModels)) {
List<DataModel> all = new ArrayList<>(); List<DataModel> all = new ArrayList<>();

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -241,8 +241,8 @@ public class JoinRender extends Renderer {
break; break;
} }
} }
if (schema.getDimension().containsKey(dataModel.getName())) { if (schema.getDimensions().containsKey(dataModel.getName())) {
for (Dimension dim : schema.getDimension().get(dataModel.getName())) { for (Dimension dim : schema.getDimensions().get(dataModel.getName())) {
if (dim.getName().equalsIgnoreCase(oriDimension)) { if (dim.getName().equalsIgnoreCase(oriDimension)) {
isAdd = true; isAdd = true;
} }

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java
View File

@@ -108,7 +108,7 @@ public class SourceRender extends Renderer {
private static void buildDimension(String alias, String dimension, DataModel datasource, private static void buildDimension(String alias, String dimension, DataModel datasource,
S2CalciteSchema schema, boolean nonAgg, Map<String, String> extendFields, S2CalciteSchema schema, boolean nonAgg, Map<String, String> extendFields,
TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception { TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception {
List<Dimension> dimensionList = schema.getDimension().get(datasource.getName()); List<Dimension> dimensionList = schema.getDimensions().get(datasource.getName());
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());
boolean isAdd = false; boolean isAdd = false;
if (!CollectionUtils.isEmpty(dimensionList)) { if (!CollectionUtils.isEmpty(dimensionList)) {
@@ -195,7 +195,7 @@ public class SourceRender extends Renderer {
} }
} }
for (String where : fields) { for (String where : fields) {
List<Dimension> dimensionList = schema.getDimension().get(datasource.getName()); List<Dimension> dimensionList = schema.getDimensions().get(datasource.getName());
boolean isAdd = false; boolean isAdd = false;
if (!CollectionUtils.isEmpty(dimensionList)) { if (!CollectionUtils.isEmpty(dimensionList)) {
for (Dimension dim : dimensionList) { for (Dimension dim : dimensionList) {
@@ -262,8 +262,8 @@ public class SourceRender extends Renderer {
dimensions.add(oriField); dimensions.add(oriField);
return; return;
} }
if (schema.getDimension().containsKey(datasource.getName())) { if (schema.getDimensions().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim = schema.getDimension().get(datasource.getName()) Optional<Dimension> dataSourceDim = schema.getDimensions().get(datasource.getName())
.stream().filter(d -> d.getName().equalsIgnoreCase(field)).findFirst(); .stream().filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
if (dataSourceDim.isPresent()) { if (dataSourceDim.isPresent()) {
dimensions.add(oriField); dimensions.add(oriField);
@@ -300,8 +300,8 @@ public class SourceRender extends Renderer {
if (identify.isPresent()) { if (identify.isPresent()) {
return true; return true;
} }
if (schema.getDimension().containsKey(datasource.getName())) { if (schema.getDimensions().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim = schema.getDimension().get(datasource.getName()) Optional<Dimension> dataSourceDim = schema.getDimensions().get(datasource.getName())
.stream().filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); .stream().filter(d -> d.getName().equalsIgnoreCase(name)).findFirst();
if (dataSourceDim.isPresent()) { if (dataSourceDim.isPresent()) {
return true; return true;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java
View File

@@ -40,7 +40,7 @@ public class SqlVariableParseConverter implements QueryConverter {
modelResp.getModelDetail().getSqlVariables(), modelResp.getModelDetail().getSqlVariables(),
queryStatement.getQueryParam().getParams()); queryStatement.getQueryParam().getParams());
DataModel dataModel = DataModel dataModel =
queryStatement.getOntology().getDatasourceMap().get(modelResp.getBizName()); queryStatement.getOntology().getDataModelMap().get(modelResp.getBizName());
dataModel.setSqlQuery(sqlParsed); dataModel.setSqlQuery(sqlParsed);
} }
} }

35
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -72,10 +72,10 @@ public class SemanticSchemaManager {
getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName)); getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
} }
if (!dataModelYamlTpls.isEmpty()) { if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataModel> dataSourceMap = Map<String, DataModel> dataModelMap =
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect( dataModelYamlTpls.stream().map(SemanticSchemaManager::getDataModel).collect(
Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1)); Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1));
ontology.setDatasourceMap(dataSourceMap); ontology.setDataModelMap(dataModelMap);
} }
if (!dimensionYamlTpls.isEmpty()) { if (!dimensionYamlTpls.isEmpty()) {
Map<String, List<Dimension>> dimensionMap = new HashMap<>(); Map<String, List<Dimension>> dimensionMap = new HashMap<>();
@@ -103,9 +103,8 @@ public class SemanticSchemaManager {
} }
tagMap.get(tagResp.getModelId()).add(tagResp); tagMap.get(tagResp.getModelId()).add(tagResp);
} }
if (Objects.nonNull(ontology.getDatasourceMap()) if (Objects.nonNull(ontology.getDataModelMap()) && !ontology.getDataModelMap().isEmpty()) {
&& !ontology.getDatasourceMap().isEmpty()) { for (Map.Entry<String, DataModel> entry : ontology.getDataModelMap().entrySet()) {
for (Map.Entry<String, DataModel> entry : ontology.getDatasourceMap().entrySet()) {
List<Dimension> modelDimensions = new ArrayList<>(); List<Dimension> modelDimensions = new ArrayList<>();
if (!ontology.getDimensionMap().containsKey(entry.getKey())) { if (!ontology.getDimensionMap().containsKey(entry.getKey())) {
ontology.getDimensionMap().put(entry.getKey(), modelDimensions); ontology.getDimensionMap().put(entry.getKey(), modelDimensions);
@@ -175,30 +174,30 @@ public class SemanticSchemaManager {
return getDimension(t); return getDimension(t);
} }
public static DataModel getDatasource(final DataModelYamlTpl d) { public static DataModel getDataModel(final DataModelYamlTpl d) {
DataModel datasource = DataModel.builder().id(d.getId()).modelId(d.getSourceId()) DataModel dataModel = DataModel.builder().id(d.getId()).modelId(d.getSourceId())
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()) .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
.tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers())) .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasureParams(d.getMeasures())) .measures(getMeasureParams(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions())).build(); .dimensions(getDimensions(d.getDimensions())).build();
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions())); dataModel.setAggTime(getDataModelAggTime(dataModel.getDimensions()));
if (Objects.nonNull(d.getModelSourceTypeEnum())) { if (Objects.nonNull(d.getModelSourceTypeEnum())) {
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
} }
if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) { if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
Set<String> measures = datasource.getMeasures().stream().map(mm -> mm.getName()) Set<String> measures = dataModel.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet()); .collect(Collectors.toSet());
for (Field f : d.getFields()) { for (Field f : d.getFields()) {
if (!measures.contains(f.getFieldName())) { if (!measures.contains(f.getFieldName())) {
datasource.getMeasures().add(Measure.builder().expr(f.getFieldName()) dataModel.getMeasures().add(Measure.builder().expr(f.getFieldName())
.name(f.getFieldName()).agg("").build()); .name(f.getFieldName()).agg("").build());
} }
} }
} }
return datasource; return dataModel;
} }
private static String getDataSourceAggTime(List<Dimension> dimensions) { private static String getDataModelAggTime(List<Dimension> dimensions) {
Optional<Dimension> timeDimension = dimensions.stream() Optional<Dimension> timeDimension = dimensions.stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())) .filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst(); .findFirst();
@@ -364,12 +363,12 @@ public class SemanticSchemaManager {
if (schema != null) { if (schema != null) {
String dataSourceName = datasourceYamlTpl.getName(); String dataSourceName = datasourceYamlTpl.getName();
Optional<Entry<String, DataModel>> datasourceYamlTplMap = Optional<Entry<String, DataModel>> datasourceYamlTplMap =
schema.getDatasource().entrySet().stream() schema.getDataModels().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst(); .filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
if (datasourceYamlTplMap.isPresent()) { if (datasourceYamlTplMap.isPresent()) {
datasourceYamlTplMap.get().setValue(datasourceYamlTpl); datasourceYamlTplMap.get().setValue(datasourceYamlTpl);
} else { } else {
schema.getDatasource().put(dataSourceName, datasourceYamlTpl); schema.getDataModels().put(dataSourceName, datasourceYamlTpl);
} }
} }
} }
@@ -378,14 +377,14 @@ public class SemanticSchemaManager {
List<Dimension> dimensionYamlTpls) throws Exception { List<Dimension> dimensionYamlTpls) throws Exception {
if (schema != null) { if (schema != null) {
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema
.getDimension().entrySet().stream() .getDimensions().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst(); .filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
if (datasourceYamlTplMap.isPresent()) { if (datasourceYamlTplMap.isPresent()) {
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue()); updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
} else { } else {
List<Dimension> dimensions = new ArrayList<>(); List<Dimension> dimensions = new ArrayList<>();
updateDimension(dimensionYamlTpls, dimensions); updateDimension(dimensionYamlTpls, dimensions);
schema.getDimension().put(datasourceBizName, dimensions); schema.getDimensions().put(datasourceBizName, dimensions);
} }
} }
} }

8
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -120,7 +120,7 @@ class HeadlessParserServiceTest {
S2CalciteSchema semanticSchema = S2CalciteSchema.builder().build(); S2CalciteSchema semanticSchema = S2CalciteSchema.builder().build();
SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.update(semanticSchema,
SemanticSchemaManager.getDatasource(datasource)); SemanticSchemaManager.getDataModel(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl(); DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("page"); dimension1.setExpr("page");
@@ -233,8 +233,8 @@ class HeadlessParserServiceTest {
identifies.add(identify); identifies.add(identify);
datasource.setIdentifiers(identifies); datasource.setIdentifiers(identifies);
semanticSchema.getDatasource().put("user_department", semanticSchema.getDataModels().put("user_department",
SemanticSchemaManager.getDatasource(datasource)); SemanticSchemaManager.getDataModel(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl(); DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("department"); dimension1.setExpr("department");
@@ -243,7 +243,7 @@ class HeadlessParserServiceTest {
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>(); List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1); dimensionYamlTpls.add(dimension1);
semanticSchema.getDimension().put("user_department", semanticSchema.getDimensions().put("user_department",
SemanticSchemaManager.getDimensions(dimensionYamlTpls)); SemanticSchemaManager.getDimensions(dimensionYamlTpls));
} }
} }