(improvement)(headless)Remove unnecessary sourceId from TranslateResp.

This commit is contained in:
jerryjzhang
2024-07-09 10:35:08 +08:00
parent c9c049a20f
commit 7a376bd9a3
9 changed files with 21 additions and 49 deletions

View File

@@ -16,8 +16,7 @@ import java.io.Serializable;
public class TranslateResp implements Serializable { public class TranslateResp implements Serializable {
private String sql; private String sql;
private boolean isOk;
private String sourceId; private String errMsg;
} }

View File

@@ -34,10 +34,6 @@ public class JdbcExecutor implements QueryExecutor {
} }
SqlUtils sqlUtils = ContextUtils.getBean(SqlUtils.class); SqlUtils sqlUtils = ContextUtils.getBean(SqlUtils.class);
if (StringUtils.isEmpty(queryStatement.getSourceId())) {
log.warn("data base id is empty");
return null;
}
log.info("executing SQL: {}", StringUtils.normalizeSpace(queryStatement.getSql())); log.info("executing SQL: {}", StringUtils.normalizeSpace(queryStatement.getSql()));
Database database = queryStatement.getSemanticModel().getDatabase(); Database database = queryStatement.getSemanticModel().getDatabase();
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();

View File

@@ -4,6 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel;
import lombok.Data; import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple; import org.apache.commons.lang3.tuple.Triple;
@@ -14,10 +15,8 @@ public class QueryStatement {
private Long dataSetId; private Long dataSetId;
private List<Long> modelIds; private List<Long> modelIds;
private String sql = ""; private String sql;
private String sourceId = ""; private String errMsg;
private String errMsg = "";
private Boolean ok;
private QueryParam queryParam; private QueryParam queryParam;
private MetricQueryParam metricQueryParam; private MetricQueryParam metricQueryParam;
private DataSetQueryParam dataSetQueryParam; private DataSetQueryParam dataSetQueryParam;
@@ -26,20 +25,17 @@ public class QueryStatement {
private List<ImmutablePair<String, String>> timeRanges; private List<ImmutablePair<String, String>> timeRanges;
private Boolean enableOptimize = true; private Boolean enableOptimize = true;
private Triple<String, String, String> minMaxTime; private Triple<String, String, String> minMaxTime;
private String dataSetSql = ""; private String dataSetSql;
private String dataSetAlias = ""; private String dataSetAlias;
private String dataSetSimplifySql = ""; private String dataSetSimplifySql;
private Boolean enableLimitWrapper = false; private Boolean enableLimitWrapper = false;
private SemanticModel semanticModel; private SemanticModel semanticModel;
private SemanticSchemaResp semanticSchemaResp; private SemanticSchemaResp semanticSchemaResp;
private Integer limit = 1000; private Integer limit = 1000;
private Boolean isTranslated = false; private Boolean isTranslated = false;
public boolean isOk() { public boolean isOk() {
this.ok = "".equals(errMsg) && !"".equals(sql); return StringUtils.isBlank(errMsg) && StringUtils.isNotBlank(sql);
return ok;
} }
public boolean isTranslated() { public boolean isTranslated() {

View File

@@ -28,9 +28,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
try { try {
parse(queryStatement); parse(queryStatement);
optimize(queryStatement); optimize(queryStatement);
queryStatement.setOk(true);
} catch (Exception e) { } catch (Exception e) {
queryStatement.setOk(false); queryStatement.setErrMsg(e.getMessage());
} }
} }
@@ -63,8 +62,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
queryStatement.getMetricQueryParam().setNativeQuery(queryParam.getQueryType().isNativeAggQuery()); queryStatement.getMetricQueryParam().setNativeQuery(queryParam.getQueryType().isNativeAggQuery());
doParse(queryStatement); doParse(queryStatement);
} }
if (StringUtils.isEmpty(queryStatement.getSql()) if (StringUtils.isEmpty(queryStatement.getSql())) {
|| StringUtils.isEmpty(queryStatement.getSourceId())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg()); throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
} }
if (StringUtils.isNotBlank(queryStatement.getSql()) if (StringUtils.isNotBlank(queryStatement.getSql())
@@ -156,7 +154,6 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
throw new Exception(String.format("parser table [%s] error [%s]", metricTable.getAlias(), throw new Exception(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
tableSql.getErrMsg())); tableSql.getErrMsg()));
} }
queryStatement.setSourceId(tableSql.getSourceId());
return tableSql; return tableSql;
} }

View File

@@ -35,7 +35,6 @@ public class CalciteQueryParser implements QueryParser {
aggBuilder.explain(queryStatement, isAgg); aggBuilder.explain(queryStatement, isAgg);
EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType()); EngineType engineType = EngineType.fromString(semanticSchema.getSemanticModel().getDatabase().getType());
queryStatement.setSql(aggBuilder.getSql(engineType)); queryStatement.setSql(aggBuilder.getSql(engineType));
queryStatement.setSourceId(aggBuilder.getSourceId());
if (Objects.nonNull(queryStatement.getEnableOptimize()) && queryStatement.getEnableOptimize() if (Objects.nonNull(queryStatement.getEnableOptimize()) && queryStatement.getEnableOptimize()
&& Objects.nonNull(queryStatement.getDataSetAlias()) && !queryStatement.getDataSetAlias().isEmpty()) { && Objects.nonNull(queryStatement.getDataSetAlias()) && !queryStatement.getDataSetAlias().isEmpty()) {
// simplify model sql with query sql // simplify model sql with query sql

View File

@@ -266,10 +266,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
TranslateSqlReq<Object> translateSqlReq = TranslateSqlReq.builder().queryReq(semanticQueryReq) TranslateSqlReq<Object> translateSqlReq = TranslateSqlReq.builder().queryReq(semanticQueryReq)
.queryTypeEnum(QueryMethod.SQL).build(); .queryTypeEnum(QueryMethod.SQL).build();
TranslateResp explain = semanticLayerService.translate(translateSqlReq, user); TranslateResp explain = semanticLayerService.translate(translateSqlReq, user);
if (StringUtils.isNotBlank(explain.getSql())) { parseInfo.getSqlInfo().setQuerySQL(explain.getSql());
parseInfo.getSqlInfo().setQuerySQL(explain.getSql());
parseInfo.getSqlInfo().setSourceId(explain.getSourceId());
}
} else { } else {
log.info("rule begin replace metrics and revise filters!"); log.info("rule begin replace metrics and revise filters!");
//remove unvalid filters //remove unvalid filters

View File

@@ -175,7 +175,6 @@ public class S2SemanticLayerService implements SemanticLayerService {
if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo()) && StringUtils.isNotBlank( if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo()) && StringUtils.isNotBlank(
semanticQueryReq.getSqlInfo().getQuerySQL())) { semanticQueryReq.getSqlInfo().getQuerySQL())) {
queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL()); queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
queryStatement.setSourceId(semanticQueryReq.getSqlInfo().getSourceId());
queryStatement.setDataSetId(semanticQueryReq.getDataSetId()); queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
queryStatement.setIsTranslated(true); queryStatement.setIsTranslated(true);
} }
@@ -233,14 +232,11 @@ public class S2SemanticLayerService implements SemanticLayerService {
T queryReq = translateSqlReq.getQueryReq(); T queryReq = translateSqlReq.getQueryReq();
QueryStatement queryStatement = buildQueryStatement((SemanticQueryReq) queryReq, user); QueryStatement queryStatement = buildQueryStatement((SemanticQueryReq) queryReq, user);
semanticTranslator.translate(queryStatement); semanticTranslator.translate(queryStatement);
return TranslateResp.builder()
String sql = ""; .sql(queryStatement.getSql())
String sorceId = ""; .isOk(queryStatement.isOk())
if (Objects.nonNull(queryStatement)) { .errMsg(queryStatement.getErrMsg())
sql = queryStatement.getSql(); .build();
sorceId = queryStatement.getSourceId();
}
return TranslateResp.builder().sql(sql).sourceId(sorceId).build();
} }
public List<ItemResp> getDomainDataSetTree() { public List<ItemResp> getDomainDataSetTree() {

View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
import com.tencent.supersonic.headless.api.pojo.enums.ChatWorkflowState; import com.tencent.supersonic.headless.api.pojo.enums.ChatWorkflowState;
import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod; import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
@@ -127,19 +126,13 @@ public class ChatWorkflowEngine {
TranslateSqlReq<Object> translateSqlReq = TranslateSqlReq.builder().queryReq(semanticQueryReq) TranslateSqlReq<Object> translateSqlReq = TranslateSqlReq.builder().queryReq(semanticQueryReq)
.queryTypeEnum(QueryMethod.SQL).build(); .queryTypeEnum(QueryMethod.SQL).build();
TranslateResp explain = queryService.translate(translateSqlReq, chatQueryContext.getUser()); TranslateResp explain = queryService.translate(translateSqlReq, chatQueryContext.getUser());
String querySql = explain.getSql(); parseInfo.getSqlInfo().setQuerySQL(explain.getSql());
if (StringUtils.isBlank(querySql)) {
return;
}
SqlInfo sqlInfo = parseInfo.getSqlInfo();
sqlInfo.setQuerySQL(querySql);
sqlInfo.setSourceId(explain.getSourceId());
keyPipelineLog.info("SqlInfoProcessor results:\n" keyPipelineLog.info("SqlInfoProcessor results:\n"
+ "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}", + "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
StringUtils.normalizeSpace(sqlInfo.getS2SQL()), StringUtils.normalizeSpace(parseInfo.getSqlInfo().getS2SQL()),
StringUtils.normalizeSpace(sqlInfo.getCorrectS2SQL()), StringUtils.normalizeSpace(parseInfo.getSqlInfo().getCorrectS2SQL()),
StringUtils.normalizeSpace(querySql)); StringUtils.normalizeSpace(parseInfo.getSqlInfo().getQuerySQL()));
} catch (Exception e) { } catch (Exception e) {
log.warn("get sql info failed:{}", parseInfo, e); log.warn("get sql info failed:{}", parseInfo, e);
} }

View File

@@ -157,7 +157,6 @@ public class QueryUtils {
} }
String unionSql = unionSqlBuilder.substring(0, unionSqlBuilder.length() - Constants.UNIONALL.length()); String unionSql = unionSqlBuilder.substring(0, unionSqlBuilder.length() - Constants.UNIONALL.length());
sqlParser.setSql(unionSql); sqlParser.setSql(unionSql);
sqlParser.setSourceId(sqlParsers.get(0).getSourceId());
log.info("union sql parser:{}", sqlParser); log.info("union sql parser:{}", sqlParser);
return sqlParser; return sqlParser;
} }