topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 04:27:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Support flexible modification of time and like,improve evaluation (#972)

Browse Source
This commit is contained in:
mainmain
2024-05-10 19:11:55 +08:00
committed by GitHub
parent ad5b28cf35
commit 942fd9beed
9 changed files with 482 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/corrector/GroupByCorrector.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.headless.core.chat.corrector;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -8,6 +9,8 @@ import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.env.Environment;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Set;
@@ -53,6 +56,11 @@ public class GroupByCorrector extends BaseSemanticCorrector {
log.info("not add group by ,exist group by in correctS2SQL:{}", correctS2SQL);
return false;
}
Environment environment = ContextUtils.getBean(Environment.class);
String correctorAdditionalInfo = environment.getProperty("corrector.additional.information");
if (StringUtils.isNotBlank(correctorAdditionalInfo) && !Boolean.parseBoolean(correctorAdditionalInfo)) {
return false;
}
return true;
}

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java
View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlsReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.ChatQueryService;
import com.tencent.supersonic.headless.server.service.QueryService;
import lombok.extern.slf4j.Slf4j;
@@ -19,6 +20,7 @@ import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@RestController
@@ -56,7 +58,17 @@ public class SqlQueryApiController {
chatQueryService.correct(querySqlReq, user);
return querySqlReq;
}).collect(Collectors.toList());
return queryService.queryByReqs(semanticQueryReqs, user);
List<CompletableFuture<SemanticQueryResp>> futures = semanticQueryReqs.stream()
.map(querySqlReq -> CompletableFuture.supplyAsync(() -> {
try {
return queryService.queryByReq(querySqlReq, user);
} catch (Exception e) {
e.printStackTrace();
return new SemanticQueryResp();
}
}))
.collect(Collectors.toList());
return futures.stream().map(CompletableFuture::join).collect(Collectors.toList());
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
View File

@@ -14,7 +14,7 @@ import java.util.List;
public interface QueryService {
SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;
List<SemanticQueryResp> queryByReqs(List<SemanticQueryReq> queryReqs, User user) throws Exception;
//List<SemanticQueryResp> queryByReqs(List<SemanticQueryReq> queryReqs, User user) throws Exception;
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);

82
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java
View File

@@ -64,7 +64,7 @@ import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.InExpression;
@@ -337,59 +337,39 @@ public class ChatQueryServiceImpl implements ChatQueryService {
if (Objects.isNull(queryData.getDateInfo())) {
return;
}
Map<String, String> map = new HashMap<>();
String dateField = TimeDimensionEnum.DAY.getChName();
if (queryData.getDateInfo().getUnit() > 1) {
queryData.getDateInfo().setStartDate(DateUtils.getBeforeDate(queryData.getDateInfo().getUnit() + 1));
queryData.getDateInfo().setEndDate(DateUtils.getBeforeDate(1));
}
// startDate equals to endDate
if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) {
for (FieldExpression fieldExpression : fieldExpressionList) {
if (TimeDimensionEnum.DAY.getChName().equals(fieldExpression.getFieldName())) {
//sql where condition exists 'equals' operator about date,just replace
if (fieldExpression.getOperator().equals(FilterOperatorEnum.EQUALS)) {
dateField = fieldExpression.getFieldName();
map.put(fieldExpression.getFieldValue().toString(),
queryData.getDateInfo().getStartDate());
filedNameToValueMap.put(dateField, map);
} else {
// first remove,then add
removeFieldNames.add(TimeDimensionEnum.DAY.getChName());
EqualsTo equalsTo = new EqualsTo();
Column column = new Column(TimeDimensionEnum.DAY.getChName());
StringValue stringValue = new StringValue(queryData.getDateInfo().getStartDate());
equalsTo.setLeftExpression(column);
equalsTo.setRightExpression(stringValue);
addConditions.add(equalsTo);
}
break;
}
for (FieldExpression fieldExpression : fieldExpressionList) {
if (TimeDimensionEnum.DAY.getChName().equals(fieldExpression.getFieldName())) {
// first remove,then add
removeFieldNames.add(TimeDimensionEnum.DAY.getChName());
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addTimeFilters(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions);
MinorThanEquals minorThanEquals = new MinorThanEquals();
addTimeFilters(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions);
break;
}
} else {
for (FieldExpression fieldExpression : fieldExpressionList) {
if (TimeDimensionEnum.DAY.getChName().equals(fieldExpression.getFieldName())) {
dateField = fieldExpression.getFieldName();
//just replace
if (FilterOperatorEnum.GREATER_THAN_EQUALS.getValue().equals(fieldExpression.getOperator())
|| FilterOperatorEnum.GREATER_THAN.getValue().equals(fieldExpression.getOperator())) {
map.put(fieldExpression.getFieldValue().toString(),
queryData.getDateInfo().getStartDate());
}
for (FieldExpression fieldExpression : fieldExpressionList) {
for (QueryFilter queryFilter : queryData.getDimensionFilters()) {
if (queryFilter.getOperator().equals(FilterOperatorEnum.LIKE)
&& FilterOperatorEnum.LIKE.getValue().toLowerCase().equals(
fieldExpression.getOperator().toLowerCase())) {
Map<String, String> replaceMap = new HashMap<>();
String preValue = fieldExpression.getFieldValue().toString();
String curValue = queryFilter.getValue().toString();
if (preValue.startsWith("%")) {
curValue = "%" + curValue;
}
if (FilterOperatorEnum.MINOR_THAN_EQUALS.getValue().equals(fieldExpression.getOperator())
|| FilterOperatorEnum.MINOR_THAN.getValue().equals(fieldExpression.getOperator())) {
map.put(fieldExpression.getFieldValue().toString(),
queryData.getDateInfo().getEndDate());
}
filedNameToValueMap.put(dateField, map);
// first remove,then add
if (FilterOperatorEnum.EQUALS.getValue().equals(fieldExpression.getOperator())) {
removeFieldNames.add(TimeDimensionEnum.DAY.getChName());
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
addTimeFilters(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions);
MinorThanEquals minorThanEquals = new MinorThanEquals();
addTimeFilters(queryData.getDateInfo().getEndDate(), minorThanEquals, addConditions);
if (preValue.endsWith("%")) {
curValue = curValue + "%";
}
replaceMap.put(preValue, curValue);
filedNameToValueMap.put(fieldExpression.getFieldName(), replaceMap);
break;
}
}
}
@@ -450,8 +430,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
Set<QueryFilter> contextMetricFilters,
List<Expression> addConditions) {
Column column = new Column(dslQueryFilter.getName());
ExpressionList expressionList = new ExpressionList();
List<Expression> expressions = new ArrayList<>();
ParenthesedExpressionList parenthesedExpressionList = new ParenthesedExpressionList<>();
List<String> valueList = JsonUtil.toList(
JsonUtil.toString(dslQueryFilter.getValue()), String.class);
if (CollectionUtils.isEmpty(valueList)) {
@@ -459,11 +438,10 @@ public class ChatQueryServiceImpl implements ChatQueryService {
}
valueList.stream().forEach(o -> {
StringValue stringValue = new StringValue(o);
expressions.add(stringValue);
parenthesedExpressionList.add(stringValue);
});
expressionList.setExpressions(expressions);
inExpression.setLeftExpression(column);
inExpression.setRightExpression(expressionList);
inExpression.setRightExpression(parenthesedExpressionList);
addConditions.add(inExpression);
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {
@@ -639,4 +617,4 @@ public class ChatQueryServiceImpl implements ChatQueryService {
querySqlReq.setSql(sqlInfo.getCorrectS2SQL());
}
}
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -46,8 +46,6 @@ import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@Service
@@ -124,14 +122,6 @@ public class QueryServiceImpl implements QueryService {
}
}
@Override
public List<SemanticQueryResp> queryByReqs(List<SemanticQueryReq> queryReqs, User user) throws Exception {
List<CompletableFuture<SemanticQueryResp>> futures = queryReqs.stream()
.map(querySqlReq -> CompletableFuture.supplyAsync(() -> queryByReq(querySqlReq, user)))
.collect(Collectors.toList());
return futures.stream().map(CompletableFuture::join).collect(Collectors.toList());
}
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception {
//If dataSetId or DataSetName is empty, parse dataSetId from the SQL
if (querySqlReq.needGetDataSetId()) {