topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Clean code logic of headless translator.

Browse Source
This commit is contained in:
jerryjzhang
2024-11-21 09:09:24 +08:00
parent d7586a5d3b
commit be05f977d5
9 changed files with 52 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/Ontology.java
View File

@@ -14,7 +14,7 @@ import java.util.stream.Collectors;
public class Ontology {
private List<Metric> metrics = new ArrayList<>();
private Map<String, DataModel> datasourceMap = new HashMap<>();
private Map<String, DataModel> dataModelMap = new HashMap<>();
private Map<String, List<Dimension>> dimensionMap = new HashMap<>();
private List<Materialization> materializationList = new ArrayList<>();
private List<JoinRelation> joinRelations;
@@ -26,7 +26,7 @@ public class Ontology {
}
public Map<Long, DataModel> getModelMap() {
return datasourceMap.values().stream()
return dataModelMap.values().stream()
.collect(Collectors.toMap(DataModel::getId, dataSource -> dataSource));
}
}

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/S2CalciteSchema.java
View File

@@ -29,11 +29,15 @@ public class S2CalciteSchema extends AbstractSchema {
return this;
}
public Map<String, DataModel> getDatasource() {
return ontology.getDatasourceMap();
public Map<String, DataModel> getDataModels() {
return ontology.getDataModelMap();
}
public Map<String, List<Dimension>> getDimension() {
public List<Metric> getMetrics() {
return ontology.getMetrics();
}
public Map<String, List<Dimension>> getDimensions() {
return ontology.getDimensionMap();
}
@@ -41,8 +45,4 @@ public class S2CalciteSchema extends AbstractSchema {
return ontology.getJoinRelations();
}
public List<Metric> getMetrics() {
return ontology.getMetrics();
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java
View File

@@ -32,8 +32,8 @@ import java.util.Objects;
@Slf4j
public class SqlBuilder {
private MetricQueryParam metricQueryParam;
private final S2CalciteSchema schema;
private MetricQueryParam metricQueryParam;
private SqlValidatorScope scope;
private SqlNode parserNode;
private boolean isAgg = false;

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java
View File

@@ -202,7 +202,7 @@ public class DataModelNode extends SemanticNode {
DataModel baseDataModel = null;
// one , match measure count
Map<String, Integer> dataSourceMeasures = new HashMap<>();
for (Map.Entry<String, DataModel> entry : schema.getDatasource().entrySet()) {
for (Map.Entry<String, DataModel> entry : schema.getDataModels().entrySet()) {
Set<String> sourceMeasure = entry.getValue().getMeasures().stream()
.map(mm -> mm.getName()).collect(Collectors.toSet());
sourceMeasure.retainAll(measures);
@@ -212,7 +212,7 @@ public class DataModelNode extends SemanticNode {
Optional<Map.Entry<String, Integer>> base = dataSourceMeasures.entrySet().stream()
.sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst();
if (base.isPresent()) {
baseDataModel = schema.getDatasource().get(base.get().getKey());
baseDataModel = schema.getDataModels().get(base.get().getKey());
dataModels.add(baseDataModel);
}
// second , check match all dimension and metric
@@ -223,8 +223,8 @@ public class DataModelNode extends SemanticNode {
Set<String> dimension = baseDataModel.getDimensions().stream().map(dd -> dd.getName())
.collect(Collectors.toSet());
baseDataModel.getIdentifiers().stream().forEach(i -> dimension.add(i.getName()));
if (schema.getDimension().containsKey(baseDataModel.getName())) {
schema.getDimension().get(baseDataModel.getName()).stream()
if (schema.getDimensions().containsKey(baseDataModel.getName())) {
schema.getDimensions().get(baseDataModel.getName()).stream()
.forEach(d -> dimension.add(d.getName()));
}
filterMeasure.addAll(sourceMeasure);
@@ -319,8 +319,8 @@ public class DataModelNode extends SemanticNode {
}
boolean isMatch = false;
boolean isRight = before.contains(joinRelation.getLeft());
DataModel other = isRight ? schema.getDatasource().get(joinRelation.getRight())
: schema.getDatasource().get(joinRelation.getLeft());
DataModel other = isRight ? schema.getDataModels().get(joinRelation.getRight())
: schema.getDataModels().get(joinRelation.getLeft());
if (!queryDimension.isEmpty()) {
Set<String> linkDimension = other.getDimensions().stream()
.map(dd -> dd.getName()).collect(Collectors.toSet());
@@ -336,8 +336,8 @@ public class DataModelNode extends SemanticNode {
if (!linkMeasure.isEmpty()) {
isMatch = true;
}
if (!isMatch && schema.getDimension().containsKey(other.getName())) {
Set<String> linkDimension = schema.getDimension().get(other.getName()).stream()
if (!isMatch && schema.getDimensions().containsKey(other.getName())) {
Set<String> linkDimension = schema.getDimensions().get(other.getName()).stream()
.map(dd -> dd.getName()).collect(Collectors.toSet());
linkDimension.retainAll(queryDimension);
if (!linkDimension.isEmpty()) {
@@ -362,7 +362,7 @@ public class DataModelNode extends SemanticNode {
}
}
orders.entrySet().stream().sorted(Map.Entry.comparingByValue()).forEach(d -> {
linkDataModels.add(schema.getDatasource().get(d.getKey()));
linkDataModels.add(schema.getDataModels().get(d.getKey()));
});
}
return linkDataModels;
@@ -388,7 +388,7 @@ public class DataModelNode extends SemanticNode {
S2CalciteSchema schema) {
Set<String> linkDataSourceName = new HashSet<>();
List<DataModel> linkDataModels = new ArrayList<>();
for (Map.Entry<String, DataModel> entry : schema.getDatasource().entrySet()) {
for (Map.Entry<String, DataModel> entry : schema.getDataModels().entrySet()) {
if (entry.getKey().equalsIgnoreCase(baseDataModel.getName())) {
continue;
}
@@ -419,7 +419,7 @@ public class DataModelNode extends SemanticNode {
}
}
}
for (Map.Entry<String, List<Dimension>> entry : schema.getDimension().entrySet()) {
for (Map.Entry<String, List<Dimension>> entry : schema.getDimensions().entrySet()) {
if (!queryDimension.isEmpty()) {
Set<String> linkDimension = entry.getValue().stream().map(dd -> dd.getName())
.collect(Collectors.toSet());
@@ -430,7 +430,7 @@ public class DataModelNode extends SemanticNode {
}
}
for (String linkName : linkDataSourceName) {
linkDataModels.add(schema.getDatasource().get(linkName));
linkDataModels.add(schema.getDataModels().get(linkName));
}
if (!CollectionUtils.isEmpty(linkDataModels)) {
List<DataModel> all = new ArrayList<>();

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -241,8 +241,8 @@ public class JoinRender extends Renderer {
break;
}
}
if (schema.getDimension().containsKey(dataModel.getName())) {
for (Dimension dim : schema.getDimension().get(dataModel.getName())) {
if (schema.getDimensions().containsKey(dataModel.getName())) {
for (Dimension dim : schema.getDimensions().get(dataModel.getName())) {
if (dim.getName().equalsIgnoreCase(oriDimension)) {
isAdd = true;
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java
View File

@@ -108,7 +108,7 @@ public class SourceRender extends Renderer {
private static void buildDimension(String alias, String dimension, DataModel datasource,
S2CalciteSchema schema, boolean nonAgg, Map<String, String> extendFields,
TableView dataSet, TableView output, SqlValidatorScope scope) throws Exception {
List<Dimension> dimensionList = schema.getDimension().get(datasource.getName());
List<Dimension> dimensionList = schema.getDimensions().get(datasource.getName());
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());
boolean isAdd = false;
if (!CollectionUtils.isEmpty(dimensionList)) {
@@ -195,7 +195,7 @@ public class SourceRender extends Renderer {
}
}
for (String where : fields) {
List<Dimension> dimensionList = schema.getDimension().get(datasource.getName());
List<Dimension> dimensionList = schema.getDimensions().get(datasource.getName());
boolean isAdd = false;
if (!CollectionUtils.isEmpty(dimensionList)) {
for (Dimension dim : dimensionList) {
@@ -262,8 +262,8 @@ public class SourceRender extends Renderer {
dimensions.add(oriField);
return;
}
if (schema.getDimension().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim = schema.getDimension().get(datasource.getName())
if (schema.getDimensions().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim = schema.getDimensions().get(datasource.getName())
.stream().filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
if (dataSourceDim.isPresent()) {
dimensions.add(oriField);
@@ -300,8 +300,8 @@ public class SourceRender extends Renderer {
if (identify.isPresent()) {
return true;
}
if (schema.getDimension().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim = schema.getDimension().get(datasource.getName())
if (schema.getDimensions().containsKey(datasource.getName())) {
Optional<Dimension> dataSourceDim = schema.getDimensions().get(datasource.getName())
.stream().filter(d -> d.getName().equalsIgnoreCase(name)).findFirst();
if (dataSourceDim.isPresent()) {
return true;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java
View File

@@ -40,7 +40,7 @@ public class SqlVariableParseConverter implements QueryConverter {
modelResp.getModelDetail().getSqlVariables(),
queryStatement.getQueryParam().getParams());
DataModel dataModel =
queryStatement.getOntology().getDatasourceMap().get(modelResp.getBizName());
queryStatement.getOntology().getDataModelMap().get(modelResp.getBizName());
dataModel.setSqlQuery(sqlParsed);
}
}

35
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -72,10 +72,10 @@ public class SemanticSchemaManager {
getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataModel> dataSourceMap =
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect(
Map<String, DataModel> dataModelMap =
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDataModel).collect(
Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1));
ontology.setDatasourceMap(dataSourceMap);
ontology.setDataModelMap(dataModelMap);
}
if (!dimensionYamlTpls.isEmpty()) {
Map<String, List<Dimension>> dimensionMap = new HashMap<>();
@@ -103,9 +103,8 @@ public class SemanticSchemaManager {
}
tagMap.get(tagResp.getModelId()).add(tagResp);
}
if (Objects.nonNull(ontology.getDatasourceMap())
&& !ontology.getDatasourceMap().isEmpty()) {
for (Map.Entry<String, DataModel> entry : ontology.getDatasourceMap().entrySet()) {
if (Objects.nonNull(ontology.getDataModelMap()) && !ontology.getDataModelMap().isEmpty()) {
for (Map.Entry<String, DataModel> entry : ontology.getDataModelMap().entrySet()) {
List<Dimension> modelDimensions = new ArrayList<>();
if (!ontology.getDimensionMap().containsKey(entry.getKey())) {
ontology.getDimensionMap().put(entry.getKey(), modelDimensions);
@@ -175,30 +174,30 @@ public class SemanticSchemaManager {
return getDimension(t);
}
public static DataModel getDatasource(final DataModelYamlTpl d) {
DataModel datasource = DataModel.builder().id(d.getId()).modelId(d.getSourceId())
public static DataModel getDataModel(final DataModelYamlTpl d) {
DataModel dataModel = DataModel.builder().id(d.getId()).modelId(d.getSourceId())
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
.tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasureParams(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions())).build();
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
dataModel.setAggTime(getDataModelAggTime(dataModel.getDimensions()));
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
}
if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
Set<String> measures = datasource.getMeasures().stream().map(mm -> mm.getName())
Set<String> measures = dataModel.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet());
for (Field f : d.getFields()) {
if (!measures.contains(f.getFieldName())) {
datasource.getMeasures().add(Measure.builder().expr(f.getFieldName())
dataModel.getMeasures().add(Measure.builder().expr(f.getFieldName())
.name(f.getFieldName()).agg("").build());
}
}
}
return datasource;
return dataModel;
}
private static String getDataSourceAggTime(List<Dimension> dimensions) {
private static String getDataModelAggTime(List<Dimension> dimensions) {
Optional<Dimension> timeDimension = dimensions.stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
@@ -364,12 +363,12 @@ public class SemanticSchemaManager {
if (schema != null) {
String dataSourceName = datasourceYamlTpl.getName();
Optional<Entry<String, DataModel>> datasourceYamlTplMap =
schema.getDatasource().entrySet().stream()
schema.getDataModels().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
if (datasourceYamlTplMap.isPresent()) {
datasourceYamlTplMap.get().setValue(datasourceYamlTpl);
} else {
schema.getDatasource().put(dataSourceName, datasourceYamlTpl);
schema.getDataModels().put(dataSourceName, datasourceYamlTpl);
}
}
}
@@ -378,14 +377,14 @@ public class SemanticSchemaManager {
List<Dimension> dimensionYamlTpls) throws Exception {
if (schema != null) {
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema
.getDimension().entrySet().stream()
.getDimensions().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
if (datasourceYamlTplMap.isPresent()) {
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
} else {
List<Dimension> dimensions = new ArrayList<>();
updateDimension(dimensionYamlTpls, dimensions);
schema.getDimension().put(datasourceBizName, dimensions);
schema.getDimensions().put(datasourceBizName, dimensions);
}
}
}

8
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -120,7 +120,7 @@ class HeadlessParserServiceTest {
S2CalciteSchema semanticSchema = S2CalciteSchema.builder().build();
SemanticSchemaManager.update(semanticSchema,
SemanticSchemaManager.getDatasource(datasource));
SemanticSchemaManager.getDataModel(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("page");
@@ -233,8 +233,8 @@ class HeadlessParserServiceTest {
identifies.add(identify);
datasource.setIdentifiers(identifies);
semanticSchema.getDatasource().put("user_department",
SemanticSchemaManager.getDatasource(datasource));
semanticSchema.getDataModels().put("user_department",
SemanticSchemaManager.getDataModel(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("department");
@@ -243,7 +243,7 @@ class HeadlessParserServiceTest {
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1);
semanticSchema.getDimension().put("user_department",
semanticSchema.getDimensions().put("user_department",
SemanticSchemaManager.getDimensions(dimensionYamlTpls));
}
}