topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:28:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(Fix)(headless)Fix expression replacement issue. (#2024)
Some checks failed
supersonic CentOS CI / build (21) (push) Has been cancelled
Details
supersonic mac CI / build (21) (push) Has been cancelled
Details
supersonic ubuntu CI / build (21) (push) Has been cancelled
Details
supersonic windows CI / build (21) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Jun Zhang
2025-01-30 08:54:42 +08:00
committed by GitHub
parent be5eeae707
commit de92b357df
18 changed files with 44 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioCalcProcessor.java
View File

@@ -142,7 +142,7 @@ public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
return new HashSet<>(); return new HashSet<>();
} }
return queryResult.getQueryColumns().stream() return queryResult.getQueryColumns().stream()
.flatMap(c -> SqlSelectHelper.getFieldsFromExpr(c.getNameEn()).stream()) .flatMap(c -> SqlSelectHelper.getFieldsFromExpr(c.getBizName()).stream())
.collect(Collectors.toSet()); .collect(Collectors.toSet());
} }
@@ -167,16 +167,16 @@ public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
Map<String, Object> result = queryResp.getResultList().get(0); Map<String, Object> result = queryResp.getResultList().get(0);
Optional<QueryColumn> valueColumn = queryResp.getColumns().stream() Optional<QueryColumn> valueColumn = queryResp.getColumns().stream()
.filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst(); .filter(c -> c.getBizName().equals(metric.getBizName())).findFirst();
if (!valueColumn.isPresent()) { if (!valueColumn.isPresent()) {
return metricInfo; return metricInfo;
} }
String valueField = String.format("%s_%s", valueColumn.get().getNameEn(), String valueField = String.format("%s_%s", valueColumn.get().getBizName(),
aggOperatorEnum.getOperator()); aggOperatorEnum.getOperator());
if (result.containsKey(valueColumn.get().getNameEn())) { if (result.containsKey(valueColumn.get().getBizName())) {
DecimalFormat df = new DecimalFormat("#.####"); DecimalFormat df = new DecimalFormat("#.####");
metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn()))); metricInfo.setValue(df.format(result.get(valueColumn.get().getBizName())));
} }
String ratio = ""; String ratio = "";
if (Objects.nonNull(result.get(valueField))) { if (Objects.nonNull(result.get(valueField))) {

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ResultFormatter.java
View File

@@ -28,7 +28,7 @@ public class ResultFormatter {
} }
for (Map<String, Object> row : queryResults) { for (Map<String, Object> row : queryResults) {
for (QueryColumn column : queryColumns) { for (QueryColumn column : queryColumns) {
String columnKey = column.getNameEn(); String columnKey = column.getBizName();
Object value = row.get(columnKey); Object value = row.get(columnKey);
table.append("| ").append(value != null ? value.toString() : "").append(" "); table.append("| ").append(value != null ? value.toString() : "").append(" ");
} }

27
common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java
View File

@@ -18,26 +18,8 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
} }
protected void visitBinaryExpression(BinaryExpression expr) { protected void visitBinaryExpression(BinaryExpression expr) {
Expression left = expr.getLeftExpression(); expr.setLeftExpression(replace(expr.getLeftExpression(), fieldExprMap));
String toReplace = ""; expr.setRightExpression(replace(expr.getRightExpression(), fieldExprMap));
if (left instanceof Function) {
Function leftFunc = (Function) left;
if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) {
toReplace = getReplaceExpr(leftFunc, fieldExprMap);
}
}
if (left instanceof Column) {
toReplace = getReplaceExpr((Column) left, fieldExprMap);
}
if (!toReplace.isEmpty()) {
Expression expression = getExpression(toReplace);
if (Objects.nonNull(expression)) {
expr.setLeftExpression(expression);
return;
}
}
expr.getLeftExpression().accept(this);
expr.getRightExpression().accept(this);
} }
public void visit(SelectItem selectExpressionItem) { public void visit(SelectItem selectExpressionItem) {
@@ -59,6 +41,11 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
columnName = column.getColumnName(); columnName = column.getColumnName();
toReplace = getReplaceExpr((Column) expression, fieldExprMap); toReplace = getReplaceExpr((Column) expression, fieldExprMap);
} }
if (expression instanceof BinaryExpression) {
BinaryExpression binaryExpression = (BinaryExpression) expression;
visitBinaryExpression(binaryExpression);
}
if (!toReplace.isEmpty()) { if (!toReplace.isEmpty()) {
Expression toReplaceExpr = getExpression(toReplace); Expression toReplaceExpr = getExpression(toReplace);
if (Objects.nonNull(toReplaceExpr)) { if (Objects.nonNull(toReplaceExpr)) {

12
common/src/main/java/com/tencent/supersonic/common/pojo/QueryColumn.java
View File

@@ -11,7 +11,7 @@ public class QueryColumn {
private String name; private String name;
private String type; private String type;
private String nameEn; private String bizName;
private String showType; private String showType;
private Boolean authorized = true; private Boolean authorized = true;
private String dataFormatType; private String dataFormatType;
@@ -19,16 +19,16 @@ public class QueryColumn {
private String comment; private String comment;
private Long modelId; private Long modelId;
public QueryColumn(String nameEn, String type) { public QueryColumn(String bizName, String type) {
this.type = type; this.type = type;
this.nameEn = nameEn; this.bizName = bizName;
this.name = nameEn; this.name = bizName;
} }
public QueryColumn(String name, String type, String nameEn) { public QueryColumn(String name, String type, String bizName) {
this.name = name; this.name = name;
this.type = type; this.type = type;
this.nameEn = nameEn; this.bizName = bizName;
this.showType = "CATEGORY"; this.showType = "CATEGORY";
} }

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java
View File

@@ -1,5 +1,5 @@
package com.tencent.supersonic.headless.api.pojo; package com.tencent.supersonic.headless.api.pojo;
public enum SchemaElementType { public enum SchemaElementType {
DATASET, METRIC, DIMENSION, VALUE, ID, DATE, TAG, TERM DATASET, MODEL, METRIC, DIMENSION, VALUE, ID, DATE, TAG, TERM
} }

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java
View File

@@ -12,7 +12,7 @@ import java.util.List;
public class SqlExecuteReq { public class SqlExecuteReq {
public static final String LIMIT_WRAPPER = " SELECT * FROM ( %s ) a LIMIT %d "; public static final String LIMIT_WRAPPER = " SELECT * FROM ( %s ) a LIMIT %d ";
@NotNull(message = "modelId can not be null") @NotNull(message = "databaseId can not be null")
private Long id; private Long id;
@NotBlank(message = "sql can not be blank") @NotBlank(message = "sql can not be blank")

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/BaseMatchStrategy.java
View File

@@ -58,12 +58,12 @@ public abstract class BaseMatchStrategy<T extends MapResult> implements MatchStr
boolean isDeleted = existResults.removeIf(existResult -> { boolean isDeleted = existResults.removeIf(existResult -> {
boolean delete = existResult.lessSimilar(oneRoundResult); boolean delete = existResult.lessSimilar(oneRoundResult);
if (delete) { if (delete) {
log.info("deleted existResult:{}", existResult); log.debug("deleted existResult:{}", existResult);
} }
return delete; return delete;
}); });
if (isDeleted) { if (isDeleted) {
log.info("deleted, add oneRoundResult:{}", oneRoundResult); log.debug("deleted, add oneRoundResult:{}", oneRoundResult);
existResults.add(oneRoundResult); existResults.add(oneRoundResult);
} }
} else { } else {

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/KeywordMapper.java
View File

@@ -127,7 +127,7 @@ public class KeywordMapper extends BaseMapper {
.similarity(EditDistanceUtils.getSimilarity(match.getDetectWord(), .similarity(EditDistanceUtils.getSimilarity(match.getDetectWord(),
schemaElement.getName())) schemaElement.getName()))
.build(); .build();
log.info("add to schema, elementMatch {}", schemaElementMatch); log.debug("add to schema, elementMatch {}", schemaElementMatch);
addToSchemaMap(chatQueryContext.getMapInfo(), schemaElement.getDataSetId(), addToSchemaMap(chatQueryContext.getMapInfo(), schemaElement.getDataSetId(),
schemaElementMatch); schemaElementMatch);
} }

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java
View File

@@ -55,12 +55,12 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception { private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception {
OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
log.info("parse with ontology: [{}]", ontologyQuery);
if (Objects.isNull(ontologyQuery) || StringUtils.isBlank(ontologyQuery.getSql())) { if (Objects.isNull(ontologyQuery) || StringUtils.isBlank(ontologyQuery.getSql())) {
throw new Exception(String.format("parse ontology table [%s] error [%s]", throw new Exception(String.format("parse ontology sql [%s] error [%s]",
queryStatement.getSqlQuery().getTable(), queryStatement.getErrMsg())); StringUtils.normalizeSpace(queryStatement.getSqlQuery().getSql()),
queryStatement.getErrMsg()));
} }
log.info("parse with ontologyQuery fields: [{}]", ontologyQuery.getFields());
SqlQuery sqlQuery = queryStatement.getSqlQuery(); SqlQuery sqlQuery = queryStatement.getSqlQuery();
String ontologyOuterSql = sqlQuery.getSql(); String ontologyOuterSql = sqlQuery.getSql();

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java
View File

@@ -139,7 +139,7 @@ public class SqlUtils {
throws SQLException { throws SQLException {
Map<String, Object> map = new LinkedHashMap<>(); Map<String, Object> map = new LinkedHashMap<>();
for (QueryColumn queryColumn : queryColumns) { for (QueryColumn queryColumn : queryColumns) {
String colName = queryColumn.getNameEn(); String colName = queryColumn.getBizName();
Object value = rs.getObject(colName); Object value = rs.getObject(colName);
map.put(colName, getValue(value)); map.put(colName, getValue(value));
} }

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -200,7 +200,7 @@ public class DimValueAspect {
} }
for (QueryColumn queryColumn : columns) { for (QueryColumn queryColumn : columns) {
if (dimAndTechNameAndBizNamePair.containsKey(queryColumn.getNameEn())) { if (dimAndTechNameAndBizNamePair.containsKey(queryColumn.getBizName())) {
return true; return true;
} }
} }

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -227,7 +227,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
private List<QueryColumn> createQueryColumns(DimensionValueReq dimensionValueReq) { private List<QueryColumn> createQueryColumns(DimensionValueReq dimensionValueReq) {
QueryColumn queryColumn = new QueryColumn(); QueryColumn queryColumn = new QueryColumn();
queryColumn.setNameEn(dimensionValueReq.getBizName()); queryColumn.setBizName(dimensionValueReq.getBizName());
queryColumn.setShowType(SemanticType.CATEGORY.name()); queryColumn.setShowType(SemanticType.CATEGORY.name());
queryColumn.setAuthorized(true); queryColumn.setAuthorized(true);
queryColumn.setType("CHAR"); queryColumn.setType("CHAR");

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java
View File

@@ -73,7 +73,7 @@ public class DatabaseController {
public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq, public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
HttpServletRequest request, HttpServletResponse response) { HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response); User user = UserHolder.findUser(request, response);
return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user); return databaseService.executeSql(sqlExecuteReq, user);
} }
@RequestMapping("/getDbNames") @RequestMapping("/getDbNames")

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java
View File

@@ -20,7 +20,7 @@ public interface DatabaseService {
List<DatabaseResp> getDatabaseByType(DataType dataType); List<DatabaseResp> getDatabaseByType(DataType dataType);
SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user); SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, User user);
DatabaseResp getDatabase(Long id, User user); DatabaseResp getDatabase(Long id, User user);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -138,8 +138,8 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
} }
@Override @Override
public SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user) { public SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, User user) {
DatabaseResp databaseResp = getDatabase(id); DatabaseResp databaseResp = getDatabase(sqlExecuteReq.getId());
if (databaseResp == null) { if (databaseResp == null) {
return new SemanticQueryResp(); return new SemanticQueryResp();
} }
@@ -257,7 +257,7 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
List<DBColumn> dbColumns = Lists.newArrayList(); List<DBColumn> dbColumns = Lists.newArrayList();
for (QueryColumn queryColumn : semanticQueryResp.getColumns()) { for (QueryColumn queryColumn : semanticQueryResp.getColumns()) {
DBColumn dbColumn = new DBColumn(); DBColumn dbColumn = new DBColumn();
dbColumn.setColumnName(queryColumn.getNameEn()); dbColumn.setColumnName(queryColumn.getBizName());
dbColumn.setDataType(queryColumn.getType()); dbColumn.setDataType(queryColumn.getType());
dbColumns.add(dbColumn); dbColumns.add(dbColumn);
} }

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -179,7 +179,7 @@ public class DownloadServiceImpl implements DownloadService {
for (Map<String, Object> row : semanticQueryResp.getResultList()) { for (Map<String, Object> row : semanticQueryResp.getResultList()) {
List<String> rowData = new ArrayList<>(); List<String> rowData = new ArrayList<>();
for (QueryColumn column : semanticQueryResp.getColumns()) { for (QueryColumn column : semanticQueryResp.getColumns()) {
rowData.add(String.valueOf(row.get(column.getNameEn()))); rowData.add(String.valueOf(row.get(column.getBizName())));
} }
data.add(rowData); data.add(rowData);
} }
@@ -217,7 +217,7 @@ public class DownloadServiceImpl implements DownloadService {
QueryColumn metric = metricColumns.get(0); QueryColumn metric = metricColumns.get(0);
List<String> groups = queryStructReq.getGroups(); List<String> groups = queryStructReq.getGroups();
List<Map<String, Object>> dataTransformed = List<Map<String, Object>> dataTransformed =
DataTransformUtils.transform(queryResult.getResultList(), metric.getNameEn(), DataTransformUtils.transform(queryResult.getResultList(), metric.getBizName(),
groups, queryStructReq.getDateInfo()); groups, queryStructReq.getDateInfo());
List<List<String>> headers = List<List<String>> headers =
buildHeader(dimensionColumns, queryStructReq.getDateInfo().getDateList()); buildHeader(dimensionColumns, queryStructReq.getDateInfo().getDateList());
@@ -264,7 +264,7 @@ public class DownloadServiceImpl implements DownloadService {
private Map<String, String> getDimensionNameMap(List<QueryColumn> queryColumns) { private Map<String, String> getDimensionNameMap(List<QueryColumn> queryColumns) {
return queryColumns.stream() return queryColumns.stream()
.collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn)); .collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getBizName));
} }
private List<DimensionResp> getMetricRelaDimensions(MetricResp metricResp, private List<DimensionResp> getMetricRelaDimensions(MetricResp metricResp,

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/FlightServiceImpl.java
View File

@@ -309,7 +309,7 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
int columnNum = resp.getColumns().size(); int columnNum = resp.getColumns().size();
rowSetMetaData.setColumnCount(columnNum); rowSetMetaData.setColumnCount(columnNum);
for (int i = 1; i <= columnNum; i++) { for (int i = 1; i <= columnNum; i++) {
String columnName = resp.getColumns().get(i - 1).getNameEn(); String columnName = resp.getColumns().get(i - 1).getBizName();
rowSetMetaData.setColumnName(i, columnName); rowSetMetaData.setColumnName(i, columnName);
Optional<Map<String, Object>> valOpt = resp.getResultList().stream() Optional<Map<String, Object>> valOpt = resp.getResultList().stream()
.filter(r -> r.containsKey(columnName) && Objects.nonNull(r.get(columnName))) .filter(r -> r.containsKey(columnName) && Objects.nonNull(r.get(columnName)))
@@ -329,7 +329,7 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
for (Map<String, Object> row : resp.getResultList()) { for (Map<String, Object> row : resp.getResultList()) {
rowset.moveToInsertRow(); rowset.moveToInsertRow();
for (int i = 1; i <= columnNum; i++) { for (int i = 1; i <= columnNum; i++) {
String columnName = resp.getColumns().get(i - 1).getNameEn(); String columnName = resp.getColumns().get(i - 1).getBizName();
if (row.containsKey(columnName)) { if (row.containsKey(columnName)) {
rowset.updateObject(i, row.get(columnName)); rowset.updateObject(i, row.get(columnName));
} else { } else {

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -72,7 +72,7 @@ public class QueryUtils {
private void processColumn(QueryColumn column, Map<String, String> namePair, private void processColumn(QueryColumn column, Map<String, String> namePair,
Map<String, String> nameTypePair, Map<String, MetricResp> metricRespMap, Map<String, String> nameTypePair, Map<String, MetricResp> metricRespMap,
Map<String, DimensionResp> dimensionRespMap) { Map<String, DimensionResp> dimensionRespMap) {
String nameEn = getName(column.getNameEn()); String nameEn = getName(column.getBizName());
if (nameEn.contains(JOIN_UNDERLINE)) { if (nameEn.contains(JOIN_UNDERLINE)) {
nameEn = nameEn.split(JOIN_UNDERLINE)[1]; nameEn = nameEn.split(JOIN_UNDERLINE)[1];
} }
@@ -110,8 +110,8 @@ public class QueryUtils {
column.setModelId(dimensionRespMap.get(nameEn).getModelId()); column.setModelId(dimensionRespMap.get(nameEn).getModelId());
} }
// set name by NameEn // set name by NameEn
if (StringUtils.isBlank(column.getName()) && StringUtils.isNotBlank(column.getNameEn())) { if (StringUtils.isBlank(column.getName()) && StringUtils.isNotBlank(column.getBizName())) {
column.setName(column.getNameEn()); column.setName(column.getBizName());
} }
} }