topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 04:27:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) native query when metric default agg is null (#976) (#992)

Browse Source
This commit is contained in:
jipeli
2024-05-14 12:16:23 +08:00
committed by GitHub
parent d5e5c7d415
commit 9792cde6aa
1 changed files with 25 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -28,14 +28,6 @@ import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@@ -45,6 +37,14 @@ import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component @Component
@Slf4j @Slf4j
@@ -57,7 +57,7 @@ public class QueryReqConverter {
private SqlGenerateUtils sqlGenerateUtils; private SqlGenerateUtils sqlGenerateUtils;
public QueryStatement convert(QuerySqlReq querySQLReq, public QueryStatement convert(QuerySqlReq querySQLReq,
SemanticSchemaResp semanticSchemaResp) throws Exception { SemanticSchemaResp semanticSchemaResp) throws Exception {
if (semanticSchemaResp == null) { if (semanticSchemaResp == null) {
return new QueryStatement(); return new QueryStatement();
@@ -75,7 +75,8 @@ public class QueryReqConverter {
} }
//4.build MetricTables //4.build MetricTables
List<String> allFields = SqlSelectHelper.getAllFields(querySQLReq.getSql()); List<String> allFields = SqlSelectHelper.getAllFields(querySQLReq.getSql());
List<String> metrics = getMetrics(semanticSchemaResp, allFields); List<MetricSchemaResp> metricSchemas = getMetrics(semanticSchemaResp, allFields);
List<String> metrics = metricSchemas.stream().map(m -> m.getBizName()).collect(Collectors.toList());
QueryStructReq queryStructReq = new QueryStructReq(); QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable(); MetricTable metricTable = new MetricTable();
metricTable.setMetrics(metrics); metricTable.setMetrics(metrics);
@@ -95,7 +96,7 @@ public class QueryReqConverter {
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect( metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
Collectors.toList())); Collectors.toList()));
} }
AggOption aggOption = getAggOption(querySQLReq); AggOption aggOption = getAggOption(querySQLReq, metricSchemas);
metricTable.setAggOption(aggOption); metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>(); List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable); tables.add(metricTable);
@@ -138,7 +139,7 @@ public class QueryReqConverter {
queryParam.setGroups(queryStructReq.getGroups()); queryParam.setGroups(queryStructReq.getGroups());
} }
private AggOption getAggOption(QuerySqlReq databaseReq) { private AggOption getAggOption(QuerySqlReq databaseReq, List<MetricSchemaResp> metricSchemas) {
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql(); String sql = databaseReq.getSql();
@@ -150,6 +151,12 @@ public class QueryReqConverter {
if (databaseReq.isInnerLayerNative()) { if (databaseReq.isInnerLayerNative()) {
return AggOption.NATIVE; return AggOption.NATIVE;
} }
long defaultAggNullCnt = metricSchemas.stream()
.filter(m -> Objects.isNull(m.getDefaultAgg()) || Strings.isBlank(m.getDefaultAgg())).count();
if (defaultAggNullCnt > 0) {
log.info("getAggOption find null defaultAgg metric set to NATIVE");
return AggOption.NATIVE;
}
return AggOption.DEFAULT; return AggOption.DEFAULT;
} }
@@ -174,9 +181,9 @@ public class QueryReqConverter {
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet()); .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet());
} }
private List<String> getMetrics(SemanticSchemaResp semanticSchemaResp, List<String> allFields) { private List<MetricSchemaResp> getMetrics(SemanticSchemaResp semanticSchemaResp, List<String> allFields) {
Map<String, String> metricLowerToNameMap = semanticSchemaResp.getMetrics().stream() Map<String, MetricSchemaResp> metricLowerToNameMap = semanticSchemaResp.getMetrics().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName)); .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry));
return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList()); .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
} }
@@ -239,7 +246,7 @@ public class QueryReqConverter {
} }
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
DataSetQueryParam viewQueryParam) { DataSetQueryParam viewQueryParam) {
String sql = viewQueryParam.getSql(); String sql = viewQueryParam.getSql();
for (MetricTable metricTable : viewQueryParam.getTables()) { for (MetricTable metricTable : viewQueryParam.getTables()) {
Set<String> measures = new HashSet<>(); Set<String> measures = new HashSet<>();
@@ -261,8 +268,8 @@ public class QueryReqConverter {
} }
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
List<String> metrics, List<String> dimensions, List<String> metrics, List<String> dimensions,
Set<String> measures, Map<String, String> replaces) { Set<String> measures, Map<String, String> replaces) {
List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics(); List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions(); List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
// check metrics has derived // check metrics has derived