topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(Fix)(headless)Fix expression replacement issue. (#2024)
Some checks failed
supersonic CentOS CI / build (21) (push) Has been cancelled
Details
supersonic mac CI / build (21) (push) Has been cancelled
Details
supersonic ubuntu CI / build (21) (push) Has been cancelled
Details
supersonic windows CI / build (21) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Jun Zhang
2025-01-30 08:54:42 +08:00
committed by GitHub
parent be5eeae707
commit de92b357df
18 changed files with 44 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioCalcProcessor.java
View File

@@ -142,7 +142,7 @@ public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
return new HashSet<>();
}
return queryResult.getQueryColumns().stream()
.flatMap(c -> SqlSelectHelper.getFieldsFromExpr(c.getNameEn()).stream())
.flatMap(c -> SqlSelectHelper.getFieldsFromExpr(c.getBizName()).stream())
.collect(Collectors.toSet());
}
@@ -167,16 +167,16 @@ public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
Map<String, Object> result = queryResp.getResultList().get(0);
Optional<QueryColumn> valueColumn = queryResp.getColumns().stream()
.filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst();
.filter(c -> c.getBizName().equals(metric.getBizName())).findFirst();
if (!valueColumn.isPresent()) {
return metricInfo;
}
String valueField = String.format("%s_%s", valueColumn.get().getNameEn(),
String valueField = String.format("%s_%s", valueColumn.get().getBizName(),
aggOperatorEnum.getOperator());
if (result.containsKey(valueColumn.get().getNameEn())) {
if (result.containsKey(valueColumn.get().getBizName())) {
DecimalFormat df = new DecimalFormat("#.####");
metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn())));
metricInfo.setValue(df.format(result.get(valueColumn.get().getBizName())));
}
String ratio = "";
if (Objects.nonNull(result.get(valueField))) {

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java
View File

@@ -28,7 +28,7 @@ public class ResultFormatter {
}
for (Map<String, Object> row : queryResults) {
for (QueryColumn column : queryColumns) {
String columnKey = column.getNameEn();
String columnKey = column.getBizName();
Object value = row.get(columnKey);
table.append("| ").append(value != null ? value.toString() : "").append(" ");
}

27
common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java
View File

@@ -18,26 +18,8 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
}
protected void visitBinaryExpression(BinaryExpression expr) {
Expression left = expr.getLeftExpression();
String toReplace = "";
if (left instanceof Function) {
Function leftFunc = (Function) left;
if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) {
toReplace = getReplaceExpr(leftFunc, fieldExprMap);
}
}
if (left instanceof Column) {
toReplace = getReplaceExpr((Column) left, fieldExprMap);
}
if (!toReplace.isEmpty()) {
Expression expression = getExpression(toReplace);
if (Objects.nonNull(expression)) {
expr.setLeftExpression(expression);
return;
}
}
expr.getLeftExpression().accept(this);
expr.getRightExpression().accept(this);
expr.setLeftExpression(replace(expr.getLeftExpression(), fieldExprMap));
expr.setRightExpression(replace(expr.getRightExpression(), fieldExprMap));
}
public void visit(SelectItem selectExpressionItem) {
@@ -59,6 +41,11 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
columnName = column.getColumnName();
toReplace = getReplaceExpr((Column) expression, fieldExprMap);
}
if (expression instanceof BinaryExpression) {
BinaryExpression binaryExpression = (BinaryExpression) expression;
visitBinaryExpression(binaryExpression);
}
if (!toReplace.isEmpty()) {
Expression toReplaceExpr = getExpression(toReplace);
if (Objects.nonNull(toReplaceExpr)) {

12
common/src/main/java/com/tencent/supersonic/common/pojo/QueryColumn.java
View File

@@ -11,7 +11,7 @@ public class QueryColumn {
private String name;
private String type;
private String nameEn;
private String bizName;
private String showType;
private Boolean authorized = true;
private String dataFormatType;
@@ -19,16 +19,16 @@ public class QueryColumn {
private String comment;
private Long modelId;
public QueryColumn(String nameEn, String type) {
public QueryColumn(String bizName, String type) {
this.type = type;
this.nameEn = nameEn;
this.name = nameEn;
this.bizName = bizName;
this.name = bizName;
}
public QueryColumn(String name, String type, String nameEn) {
public QueryColumn(String name, String type, String bizName) {
this.name = name;
this.type = type;
this.nameEn = nameEn;
this.bizName = bizName;
this.showType = "CATEGORY";
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java
View File

@@ -1,5 +1,5 @@
package com.tencent.supersonic.headless.api.pojo;
public enum SchemaElementType {
DATASET, METRIC, DIMENSION, VALUE, ID, DATE, TAG, TERM
DATASET, MODEL, METRIC, DIMENSION, VALUE, ID, DATE, TAG, TERM
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java
View File

@@ -12,7 +12,7 @@ import java.util.List;
public class SqlExecuteReq {
public static final String LIMIT_WRAPPER = " SELECT * FROM ( %s ) a LIMIT %d ";
@NotNull(message = "modelId can not be null")
@NotNull(message = "databaseId can not be null")
private Long id;
@NotBlank(message = "sql can not be blank")

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java
View File

@@ -58,12 +58,12 @@ public abstract class BaseMatchStrategy<T extends MapResult> implements MatchStr
boolean isDeleted = existResults.removeIf(existResult -> {
boolean delete = existResult.lessSimilar(oneRoundResult);
if (delete) {
log.info("deleted existResult:{}", existResult);
log.debug("deleted existResult:{}", existResult);
}
return delete;
});
if (isDeleted) {
log.info("deleted, add oneRoundResult:{}", oneRoundResult);
log.debug("deleted, add oneRoundResult:{}", oneRoundResult);
existResults.add(oneRoundResult);
}
} else {

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java
View File

@@ -127,7 +127,7 @@ public class KeywordMapper extends BaseMapper {
.similarity(EditDistanceUtils.getSimilarity(match.getDetectWord(),
schemaElement.getName()))
.build();
log.info("add to schema, elementMatch {}", schemaElementMatch);
log.debug("add to schema, elementMatch {}", schemaElementMatch);
addToSchemaMap(chatQueryContext.getMapInfo(), schemaElement.getDataSetId(),
schemaElementMatch);
}

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java
View File

@@ -55,12 +55,12 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception {
OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
log.info("parse with ontology: [{}]", ontologyQuery);
if (Objects.isNull(ontologyQuery) || StringUtils.isBlank(ontologyQuery.getSql())) {
throw new Exception(String.format("parse ontology table [%s] error [%s]",
queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg()));
throw new Exception(String.format("parse ontology sql [%s] error [%s]",
StringUtils.normalizeSpace(queryStatement.getSqlQuery().getSql()),
queryStatement.getErrMsg()));
}
log.info("parse with ontologyQuery fields: [{}]", ontologyQuery.getFields());
SqlQuery sqlQuery = queryStatement.getSqlQuery();
String ontologyOuterSql = sqlQuery.getSql();

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java
View File

@@ -139,7 +139,7 @@ public class SqlUtils {
throws SQLException {
Map<String, Object> map = new LinkedHashMap<>();
for (QueryColumn queryColumn : queryColumns) {
String colName = queryColumn.getNameEn();
String colName = queryColumn.getBizName();
Object value = rs.getObject(colName);
map.put(colName, getValue(value));
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -200,7 +200,7 @@ public class DimValueAspect {
}
for (QueryColumn queryColumn : columns) {
if (dimAndTechNameAndBizNamePair.containsKey(queryColumn.getNameEn())) {
if (dimAndTechNameAndBizNamePair.containsKey(queryColumn.getBizName())) {
return true;
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -227,7 +227,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
private List<QueryColumn> createQueryColumns(DimensionValueReq dimensionValueReq) {
QueryColumn queryColumn = new QueryColumn();
queryColumn.setNameEn(dimensionValueReq.getBizName());
queryColumn.setBizName(dimensionValueReq.getBizName());
queryColumn.setShowType(SemanticType.CATEGORY.name());
queryColumn.setAuthorized(true);
queryColumn.setType("CHAR");

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java
View File

@@ -73,7 +73,7 @@ public class DatabaseController {
public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user);
return databaseService.executeSql(sqlExecuteReq, user);
}
@RequestMapping("/getDbNames")

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java
View File

@@ -20,7 +20,7 @@ public interface DatabaseService {
List<DatabaseResp> getDatabaseByType(DataType dataType);
SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user);
SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, User user);
DatabaseResp getDatabase(Long id, User user);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -138,8 +138,8 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
}
@Override
public SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user) {
DatabaseResp databaseResp = getDatabase(id);
public SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, User user) {
DatabaseResp databaseResp = getDatabase(sqlExecuteReq.getId());
if (databaseResp == null) {
return new SemanticQueryResp();
}
@@ -257,7 +257,7 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
List<DBColumn> dbColumns = Lists.newArrayList();
for (QueryColumn queryColumn : semanticQueryResp.getColumns()) {
DBColumn dbColumn = new DBColumn();
dbColumn.setColumnName(queryColumn.getNameEn());
dbColumn.setColumnName(queryColumn.getBizName());
dbColumn.setDataType(queryColumn.getType());
dbColumns.add(dbColumn);
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -179,7 +179,7 @@ public class DownloadServiceImpl implements DownloadService {
for (Map<String, Object> row : semanticQueryResp.getResultList()) {
List<String> rowData = new ArrayList<>();
for (QueryColumn column : semanticQueryResp.getColumns()) {
rowData.add(String.valueOf(row.get(column.getNameEn())));
rowData.add(String.valueOf(row.get(column.getBizName())));
}
data.add(rowData);
}
@@ -217,7 +217,7 @@ public class DownloadServiceImpl implements DownloadService {
QueryColumn metric = metricColumns.get(0);
List<String> groups = queryStructReq.getGroups();
List<Map<String, Object>> dataTransformed =
DataTransformUtils.transform(queryResult.getResultList(), metric.getNameEn(),
DataTransformUtils.transform(queryResult.getResultList(), metric.getBizName(),
groups, queryStructReq.getDateInfo());
List<List<String>> headers =
buildHeader(dimensionColumns, queryStructReq.getDateInfo().getDateList());
@@ -264,7 +264,7 @@ public class DownloadServiceImpl implements DownloadService {
private Map<String, String> getDimensionNameMap(List<QueryColumn> queryColumns) {
return queryColumns.stream()
.collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn));
.collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getBizName));
}
private List<DimensionResp> getMetricRelaDimensions(MetricResp metricResp,

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/FlightServiceImpl.java
View File

@@ -309,7 +309,7 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
int columnNum = resp.getColumns().size();
rowSetMetaData.setColumnCount(columnNum);
for (int i = 1; i <= columnNum; i++) {
String columnName = resp.getColumns().get(i - 1).getNameEn();
String columnName = resp.getColumns().get(i - 1).getBizName();
rowSetMetaData.setColumnName(i, columnName);
Optional<Map<String, Object>> valOpt = resp.getResultList().stream()
.filter(r -> r.containsKey(columnName) && Objects.nonNull(r.get(columnName)))
@@ -329,7 +329,7 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
for (Map<String, Object> row : resp.getResultList()) {
rowset.moveToInsertRow();
for (int i = 1; i <= columnNum; i++) {
String columnName = resp.getColumns().get(i - 1).getNameEn();
String columnName = resp.getColumns().get(i - 1).getBizName();
if (row.containsKey(columnName)) {
rowset.updateObject(i, row.get(columnName));
} else {

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -72,7 +72,7 @@ public class QueryUtils {
private void processColumn(QueryColumn column, Map<String, String> namePair,
Map<String, String> nameTypePair, Map<String, MetricResp> metricRespMap,
Map<String, DimensionResp> dimensionRespMap) {
String nameEn = getName(column.getNameEn());
String nameEn = getName(column.getBizName());
if (nameEn.contains(JOIN_UNDERLINE)) {
nameEn = nameEn.split(JOIN_UNDERLINE)[1];
}
@@ -110,8 +110,8 @@ public class QueryUtils {
column.setModelId(dimensionRespMap.get(nameEn).getModelId());
}
// set name by NameEn
if (StringUtils.isBlank(column.getName()) && StringUtils.isNotBlank(column.getNameEn())) {
column.setName(column.getNameEn());
if (StringUtils.isBlank(column.getName()) && StringUtils.isNotBlank(column.getBizName())) {
column.setName(column.getBizName());
}
}