topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) logic sql show in chinese and convert to bizName in execute (#156)

Browse Source
This commit is contained in:
lexluo09
2023-09-27 17:27:31 +08:00
committed by GitHub
parent f931951ad5
commit 617db611c3
17 changed files with 138 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -1,6 +1,10 @@
package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
@@ -17,6 +21,7 @@ import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -31,6 +36,7 @@ import org.springframework.util.CollectionUtils;
@Slf4j
public class QueryReqConverter {
public static final String TABLE_PREFIX = "t_";
@Autowired
private ModelService domainService;
@Autowired
@@ -41,38 +47,36 @@ public class QueryReqConverter {
@Autowired
private Catalog catalog;
public QueryStatement convert(QueryDslReq databaseReq, List<ModelSchemaResp> domainSchemas) throws Exception {
public QueryStatement convert(QueryDslReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception {
List<MetricTable> tables = new ArrayList<>();
MetricTable metricTable = new MetricTable();
List<String> allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql());
if (Objects.isNull(modelSchemaResp)) {
return new QueryStatement();
}
//1.convert name to bizName
convertNameToBizName(databaseReq, modelSchemaResp);
//2.functionName corrector
functionNameCorrector(databaseReq);
//3.correct tableName
correctTableName(databaseReq);
String tableName = SqlParserSelectHelper.getTableName(databaseReq.getSql());
functionNameCorrector(databaseReq);
if (CollectionUtils.isEmpty(domainSchemas) || StringUtils.isEmpty(tableName)) {
if (StringUtils.isEmpty(tableName)) {
return new QueryStatement();
}
Set<String> dimensions = domainSchemas.get(0).getDimensions().stream()
.map(entry -> entry.getBizName().toLowerCase())
.collect(Collectors.toSet());
dimensions.addAll(QueryStructUtils.internalCols);
List<String> allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql());
Set<String> metrics = domainSchemas.get(0).getMetrics().stream().map(entry -> entry.getBizName().toLowerCase())
.collect(Collectors.toSet());
List<String> metrics = getMetrics(modelSchemaResp, allFields);
metricTable.setMetrics(metrics);
Set<String> dimensions = getDimensions(modelSchemaResp, allFields);
metricTable.setDimensions(new ArrayList<>(dimensions));
metricTable.setMetrics(allFields.stream().filter(entry -> metrics.contains(entry.toLowerCase()))
.map(String::toLowerCase).collect(Collectors.toList()));
Set<String> collect = allFields.stream().filter(entry -> dimensions.contains(entry.toLowerCase()))
.map(String::toLowerCase).collect(Collectors.toSet());
for (String internalCol : QueryStructUtils.internalCols) {
if (databaseReq.getSql().contains(internalCol)) {
collect.add(internalCol);
}
}
metricTable.setDimensions(new ArrayList<>(collect));
metricTable.setAlias(tableName.toLowerCase());
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
@@ -92,6 +96,33 @@ public class QueryReqConverter {
return queryStatement;
}
private void convertNameToBizName(QueryDslReq databaseReq, ModelSchemaResp modelSchemaResp) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
String replaceFields = SqlParserUpdateHelper.replaceFields(sql, fieldNameToBizNameMap, true);
log.info("convert name to bizName after:{}", replaceFields);
databaseReq.setSql(replaceFields);
}
private Set<String> getDimensions(ModelSchemaResp modelSchemaResp, List<String> allFields) {
Set<String> allDimensions = modelSchemaResp.getDimensions().stream()
.map(entry -> entry.getBizName().toLowerCase())
.collect(Collectors.toSet());
allDimensions.addAll(QueryStructUtils.internalCols);
Set<String> collect = allFields.stream().filter(entry -> allDimensions.contains(entry.toLowerCase()))
.map(String::toLowerCase).collect(Collectors.toSet());
return collect;
}
private List<String> getMetrics(ModelSchemaResp modelSchemaResp, List<String> allFields) {
Set<String> allMetrics = modelSchemaResp.getMetrics().stream().map(entry -> entry.getBizName().toLowerCase())
.collect(Collectors.toSet());
List<String> metrics = allFields.stream().filter(entry -> allMetrics.contains(entry.toLowerCase()))
.map(String::toLowerCase).collect(Collectors.toList());
return metrics;
}
private void functionNameCorrector(QueryDslReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId());
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
@@ -107,4 +138,21 @@ public class QueryReqConverter {
}
}
protected Map<String, String> getFieldNameToBizNameMap(ModelSchemaResp modelSchemaResp) {
List<SchemaItem> allSchemaItems = new ArrayList<>();
allSchemaItems.addAll(modelSchemaResp.getDimensions());
allSchemaItems.addAll(modelSchemaResp.getMetrics());
Map<String, String> result = allSchemaItems.stream()
.collect(Collectors.toMap(SchemaItem::getName, a -> a.getBizName(), (k1, k2) -> k1));
result.put(DateUtils.DATE_FIELD, TimeDimensionEnum.DAY.getName());
return result;
}
public void correctTableName(QueryDslReq databaseReq) {
String sql = SqlParserUpdateHelper.replaceTable(databaseReq.getSql(), TABLE_PREFIX + databaseReq.getModelId());
databaseReq.setSql(sql);
}
}

7
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -90,8 +90,11 @@ public class QueryServiceImpl implements QueryService {
filter.setModelIds(modelIds);
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> domainSchemas = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchemas);
ModelSchemaResp domainSchema = null;
if (CollectionUtils.isNotEmpty(domainSchemas)) {
domainSchema = domainSchemas.get(0);
}
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchema);
queryStatement.setModelId(querySqlCmd.getModelId());
return queryStatement;
}