topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 13:07:32 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[release](project)update version 0.7.4 backend (#66)

Browse Source
This commit is contained in:
daikon
2023-09-10 21:26:46 +08:00
committed by GitHub
parent 02068f58c7
commit a8add4c013
172 changed files with 2180 additions and 1082 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java
View File

@@ -79,7 +79,7 @@ public class QueryParser {
} else {
sql = sqlCommend.getSql();
for (String[] tb : tables) {
sql = sql.replaceAll(tb[0], "(" + tb[1] + ")");
sql = StringUtils.replace(sql, tb[0], "(" + tb[1] + ")", -1);
}
}
queryStatement.setSql(sql);

14
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java
View File

@@ -11,6 +11,7 @@ import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DimensionTimeTypeParams;
@@ -26,6 +27,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -65,7 +67,7 @@ public class SemanticSchemaManager {
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DatasourceYamlTpl> datasourceYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
catalog.getModelYamlTplByMoldelIds(modelIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls);
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, datasourceYamlTpls, metricYamlTpls);
if (!datasourceYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = datasourceYamlTpls.stream().map(d -> getDatasource(d))
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
@@ -114,9 +116,19 @@ public class SemanticSchemaManager {
datasource.setIdentifiers(getIdentify(d.getIdentifiers()));
datasource.setDimensions(getDimensions(d.getDimensions()));
datasource.setMeasures(getMeasures(d.getMeasures()));
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
return datasource;
}
private static String getDataSourceAggTime(List<Dimension> dimensions) {
Optional<Dimension> timeDimension = dimensions.stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).findFirst();
if (timeDimension.isPresent() && Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
}
return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE;
}
private static List<Metric> getMetricsByMetricYamlTpl(List<MetricYamlTpl> metricYamlTpls) {
List<Metric> metrics = new ArrayList<>();
for (MetricYamlTpl metricYamlTpl : metricYamlTpls) {

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java
View File

@@ -9,5 +9,8 @@ public class Constants {
public static final String JOIN_TABLE_PREFIX = "src1_";
public static final String JOIN_TABLE_OUT_PREFIX = "src11_";
public static final String JOIN_TABLE_LEFT_PREFIX = "src12_";
public static final String DIMENSION_TYPE_TIME_GRANULARITY_NONE = "none";
public static final String DIMENSION_TYPE_TIME = "time";
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java
View File

@@ -20,4 +20,6 @@ public class DataSource {
private List<Dimension> dimensions;
private List<Measure> measures;
private String aggTime;
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java
View File

@@ -2,12 +2,13 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.FilterRender;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.render.OutputRender;
@@ -16,6 +17,7 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Objects;
import java.util.Stack;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;
@@ -42,6 +44,10 @@ public class AggPlanner implements Planner {
if (datasource == null || datasource.isEmpty()) {
throw new Exception("datasource not found");
}
if (Objects.nonNull(datasource.get(0).getAggTime()) && !datasource.get(0).getAggTime().equalsIgnoreCase(
Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) {
isAgg = true;
}
sourceId = String.valueOf(datasource.get(0).getSourceId());
// build level by level

37
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryReqConverter.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -1,19 +1,26 @@
package com.tencent.supersonic.semantic.query.service;
package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptor;
import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptorFactory;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -21,6 +28,7 @@ import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
public class QueryReqConverter {
@Autowired
@@ -30,14 +38,18 @@ public class QueryReqConverter {
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private Catalog catalog;
public QueryStatement convert(QueryDslReq databaseReq, List<ModelSchemaResp> domainSchemas) throws Exception {
List<MetricTable> tables = new ArrayList<>();
MetricTable metricTable = new MetricTable();
String sql = databaseReq.getSql();
List<String> allFields = SqlParserSelectHelper.getAllFields(sql);
String tableName = SqlParserSelectHelper.getTableName(sql);
List<String> allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql());
String tableName = SqlParserSelectHelper.getTableName(databaseReq.getSql());
functionNameCorrector(databaseReq);
if (CollectionUtils.isEmpty(domainSchemas) || StringUtils.isEmpty(tableName)) {
return new QueryStatement();
@@ -56,7 +68,7 @@ public class QueryReqConverter {
Set<String> collect = allFields.stream().filter(entry -> dimensions.contains(entry.toLowerCase()))
.map(String::toLowerCase).collect(Collectors.toSet());
for (String internalCol : QueryStructUtils.internalCols) {
if (sql.contains(internalCol)) {
if (databaseReq.getSql().contains(internalCol)) {
collect.add(internalCol);
}
}
@@ -80,4 +92,19 @@ public class QueryReqConverter {
return queryStatement;
}
private void functionNameCorrector(QueryDslReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId());
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
}
String type = database.getType();
EngineAdaptor engineAdaptor = EngineAdaptorFactory.getEngineAdaptor(type.toLowerCase());
log.info("type:{},engineAdaptor:{}", type, engineAdaptor);
if (Objects.nonNull(engineAdaptor)) {
String functionNameCorrector = engineAdaptor.functionNameCorrector(databaseReq.getSql());
log.info("sql:{} ,after corrector", databaseReq.getSql(), functionNameCorrector);
databaseReq.setSql(functionNameCorrector);
}
}
}

1
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.QueryUtils;
import com.tencent.supersonic.semantic.query.utils.StatUtils;

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -24,7 +24,6 @@ import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -169,7 +168,7 @@ public class DataPermissionAOP {
String modelName = modelResp.getName();
List<String> admins = modelService.getModelAdmin(modelResp.getId());
String message = String.format("您没有主题域[%s]权限，请联系管理员%s开通", modelName, admins);
throw new InvalidArgumentException(message);
throw new InvalidPermissionException(message);
}
}

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java
View File

@@ -97,6 +97,9 @@ public class QueryUtils {
column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType());
column.setDataFormat(metricRespMap.get(nameEn).getDataFormat());
}
if (StringUtils.isEmpty(column.getShowType())) {
column.setShowType("NUMBER");
}
});
}
@@ -147,6 +150,9 @@ public class QueryUtils {
|| type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")) {
return true;
}
if (type.toLowerCase().startsWith("uint") || type.toLowerCase().startsWith("int")) {
return true;
}
return false;
}