topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 02:46:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Optimized the code for populating column information and resolved the issue with number type fields (#1633)

Browse Source
This commit is contained in:
lexluo09
2024-09-05 17:44:37 +08:00
committed by GitHub
parent 7dbd23d751
commit 45777f0abc
2 changed files with 68 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -172,7 +172,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
for (QueryExecutor queryExecutor : queryExecutors) {
if (queryExecutor.accept(queryStatement)) {
queryResp = queryExecutor.execute(queryStatement);
queryUtils.fillItemNameInfo(queryResp, queryStatement.getSemanticSchemaResp());
queryUtils.populateQueryColumns(queryResp, queryStatement.getSemanticSchemaResp());
}
}

142
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
@@ -17,13 +16,9 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -40,68 +35,86 @@ public class QueryUtils {
private static final String no_quotation_pattern = "\\((.*?)\\)";
private final Set<Pattern> patterns = new HashSet<>();
@Value("${s2.query-optimizer.enable:true}")
private Boolean optimizeEnable;
@PostConstruct
public void fillPattern() {
Set<String> aggFunctions = new HashSet<>(Arrays.asList("MAX", "MIN", "SUM", "AVG"));
String patternStr = "\\s*(%s\\((.*)\\)) AS";
for (String agg : aggFunctions) {
patterns.add(Pattern.compile(String.format(patternStr, agg)));
}
}
public void fillItemNameInfo(SemanticQueryResp semanticQueryResp, SemanticSchemaResp semanticSchemaResp) {
List<MetricSchemaResp> metricDescList = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimSchemaResps = semanticSchemaResp.getDimensions();
Map<String, MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
public void populateQueryColumns(SemanticQueryResp semanticQueryResp, SemanticSchemaResp semanticSchemaResp) {
Map<String, MetricResp> metricRespMap = createMetricRespMap(semanticSchemaResp);
Map<String, String> namePair = new HashMap<>();
Map<String, String> nameTypePair = new HashMap<>();
addSysTimeDimension(namePair, nameTypePair);
metricDescList.forEach(metricDesc -> {
populateNamePairs(semanticSchemaResp, namePair, nameTypePair);
List<QueryColumn> columns = semanticQueryResp.getColumns();
columns.forEach(column -> processColumn(column, namePair, nameTypePair, metricRespMap));
}
private Map<String, MetricResp> createMetricRespMap(SemanticSchemaResp semanticSchemaResp) {
List<MetricSchemaResp> metrics = semanticSchemaResp.getMetrics();
return metrics.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
}
private void populateNamePairs(SemanticSchemaResp semanticSchemaResp,
Map<String, String> namePair,
Map<String, String> nameTypePair) {
for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) {
namePair.put(timeDimensionEnum.getName(), "date");
nameTypePair.put(timeDimensionEnum.getName(), "DATE");
}
semanticSchemaResp.getMetrics().forEach(metricDesc -> {
namePair.put(metricDesc.getBizName(), metricDesc.getName());
nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name());
});
dimSchemaResps.forEach(dimensionDesc -> {
semanticSchemaResp.getDimensions().forEach(dimensionDesc -> {
namePair.put(dimensionDesc.getBizName(), dimensionDesc.getName());
nameTypePair.put(dimensionDesc.getBizName(), dimensionDesc.getSemanticType());
});
List<QueryColumn> columns = semanticQueryResp.getColumns();
columns.forEach(column -> {
String nameEn = getName(column.getNameEn().toLowerCase());
if (nameEn.contains(JOIN_UNDERLINE)) {
nameEn = nameEn.split(JOIN_UNDERLINE)[1];
}
private void processColumn(QueryColumn column,
Map<String, String> namePair,
Map<String, String> nameTypePair,
Map<String, MetricResp> metricRespMap) {
String nameEn = getName(column.getNameEn().toLowerCase());
if (nameEn.contains(JOIN_UNDERLINE)) {
nameEn = nameEn.split(JOIN_UNDERLINE)[1];
}
//set name
if (namePair.containsKey(nameEn)) {
column.setName(namePair.get(nameEn));
} else {
String nameEnByRegex = getNameEnByRegex(nameEn, pattern);
if (StringUtils.isEmpty(nameEnByRegex)) {
nameEnByRegex = getNameEnByRegex(nameEn, no_quotation_pattern);
}
if (namePair.containsKey(nameEn)) {
column.setName(namePair.get(nameEn));
} else {
String nameEnByRegex = getNameEnByRegex(nameEn, pattern);
if (StringUtils.isEmpty(nameEnByRegex)) {
nameEnByRegex = getNameEnByRegex(nameEn, no_quotation_pattern);
}
if (StringUtils.isNotEmpty(nameEnByRegex) && StringUtils.isNotEmpty(namePair.get(nameEnByRegex))) {
String filedName = namePair.get(nameEnByRegex);
column.setName(nameEn.replaceAll(nameEnByRegex, filedName));
}
if (StringUtils.isNotEmpty(nameEnByRegex) && StringUtils.isNotEmpty(namePair.get(nameEnByRegex))) {
String filedName = namePair.get(nameEnByRegex);
column.setName(nameEn.replaceAll(nameEnByRegex, filedName));
}
if (nameTypePair.containsKey(nameEn)) {
column.setShowType(nameTypePair.get(nameEn));
}
if (!nameTypePair.containsKey(nameEn) && isNumberType(column.getType())) {
column.setShowType(SemanticType.NUMBER.name());
}
if (metricRespMap.containsKey(nameEn)) {
column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType());
column.setDataFormat(metricRespMap.get(nameEn).getDataFormat());
}
if (StringUtils.isEmpty(column.getShowType())) {
column.setShowType(SemanticType.CATEGORY.name());
}
});
}
//set showType
if (nameTypePair.containsKey(nameEn)) {
column.setShowType(nameTypePair.get(nameEn));
}
if (!nameTypePair.containsKey(nameEn) && isNumberType(column.getType())) {
column.setShowType(SemanticType.NUMBER.name());
}
if (StringUtils.isEmpty(column.getShowType())) {
column.setShowType(SemanticType.CATEGORY.name());
}
//set dataFormat/dataFormatType
if (metricRespMap.containsKey(nameEn)) {
column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType());
column.setDataFormat(metricRespMap.get(nameEn).getDataFormat());
}
}
private boolean isNumberType(String type) {
if (StringUtils.isBlank(type)) {
return false;
}
return type.equalsIgnoreCase("int") || type.equalsIgnoreCase("bigint")
|| type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")
|| type.equalsIgnoreCase("numeric")
|| type.toLowerCase().startsWith("uint") || type.toLowerCase().startsWith("int");
}
private String getName(String nameEn) {
@@ -124,27 +137,6 @@ public class QueryUtils {
return null;
}
private boolean isNumberType(String type) {
if (StringUtils.isBlank(type)) {
return false;
}
if (type.equalsIgnoreCase("int") || type.equalsIgnoreCase("bigint")
|| type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")) {
return true;
}
if (type.toLowerCase().startsWith("uint") || type.toLowerCase().startsWith("int")) {
return true;
}
return false;
}
private static void addSysTimeDimension(Map<String, String> namePair, Map<String, String> nameTypePair) {
for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) {
namePair.put(timeDimensionEnum.getName(), "date");
nameTypePair.put(timeDimensionEnum.getName(), "DATE");
}
}
public QueryStatement sqlParserUnion(QueryMultiStructReq queryMultiStructCmd, List<QueryStatement> sqlParsers) {
QueryStatement sqlParser = new QueryStatement();
StringBuilder unionSqlBuilder = new StringBuilder();