diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index e8edc777b..e6da4826f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -37,7 +37,8 @@ public class SemanticParseInfo { private List elementMatches = new ArrayList<>(); private Map properties = new HashMap<>(); private EntityInfo entityInfo; - private String sql; + private String logicSql; + private String querySql; public Long getModelId() { return model != null ? model.getId() : 0L; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java index 9b4e80e41..63edb2767 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.corrector; import com.tencent.supersonic.chat.api.component.SemanticCorrector; import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.knowledge.service.SchemaService; @@ -14,7 +15,14 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class BaseSemanticCorrector implements SemanticCorrector { + public static final String DATE_FIELD = "数据日期"; + + + public void correct(SemanticCorrectInfo semanticCorrectInfo) { + semanticCorrectInfo.setPreSql(semanticCorrectInfo.getSql()); + } + protected Map getFieldToBizName(Long modelId) { SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalCorrector.java index ec9e819b0..fbe6b0a83 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalCorrector.java @@ -21,6 +21,7 @@ public class GlobalCorrector extends BaseSemanticCorrector { @Override public void correct(SemanticCorrectInfo semanticCorrectInfo) { + super.correct(semanticCorrectInfo); replaceAlias(semanticCorrectInfo); @@ -74,9 +75,8 @@ public class GlobalCorrector extends BaseSemanticCorrector { Collectors.groupingBy(ElementValue::getFieldValue, Collectors.mapping(ElementValue::getFieldName, Collectors.toSet()))); - String preSql = semanticCorrectInfo.getSql(); - semanticCorrectInfo.setPreSql(preSql); - String sql = SqlParserUpdateHelper.replaceFieldNameByValue(preSql, fieldValueToFieldNames); + String sql = SqlParserUpdateHelper.replaceFieldNameByValue(semanticCorrectInfo.getSql(), + fieldValueToFieldNames); semanticCorrectInfo.setSql(sql); } } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java index 62407df25..303a498e5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java @@ -17,31 +17,30 @@ public class SelectCorrector extends BaseSemanticCorrector { @Override public void correct(SemanticCorrectInfo semanticCorrectInfo) { - String preSql = semanticCorrectInfo.getSql(); + super.correct(semanticCorrectInfo); + String sql = semanticCorrectInfo.getSql(); - if (SqlParserSelectHelper.hasAggregateFunction(preSql)) { - Expression havingExpression = SqlParserSelectHelper.getHavingExpression(preSql); + if (SqlParserSelectHelper.hasAggregateFunction(sql)) { + Expression havingExpression = SqlParserSelectHelper.getHavingExpression(sql); if (Objects.nonNull(havingExpression)) { - String replaceSql = SqlParserUpdateHelper.addFunctionToSelect(preSql, havingExpression); - semanticCorrectInfo.setPreSql(preSql); + String replaceSql = SqlParserUpdateHelper.addFunctionToSelect(sql, havingExpression); semanticCorrectInfo.setSql(replaceSql); } return; } - Set selectFields = new HashSet<>(SqlParserSelectHelper.getSelectFields(preSql)); - Set whereFields = new HashSet<>(SqlParserSelectHelper.getWhereFields(preSql)); + Set selectFields = new HashSet<>(SqlParserSelectHelper.getSelectFields(sql)); + Set whereFields = new HashSet<>(SqlParserSelectHelper.getWhereFields(sql)); if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(whereFields)) { return; } - whereFields.addAll(SqlParserSelectHelper.getOrderByFields(preSql)); + whereFields.addAll(SqlParserSelectHelper.getOrderByFields(sql)); whereFields.removeAll(selectFields); whereFields.remove(TimeDimensionEnum.DAY.getName()); whereFields.remove(TimeDimensionEnum.WEEK.getName()); whereFields.remove(TimeDimensionEnum.MONTH.getName()); - String replaceFields = SqlParserUpdateHelper.addFieldsToSelect(preSql, new ArrayList<>(whereFields)); - semanticCorrectInfo.setPreSql(preSql); + String replaceFields = SqlParserUpdateHelper.addFieldsToSelect(sql, new ArrayList<>(whereFields)); semanticCorrectInfo.setSql(replaceFields); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableCorrector.java index 1a64727c3..16008197e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/TableCorrector.java @@ -11,10 +11,9 @@ public class TableCorrector extends BaseSemanticCorrector { @Override public void correct(SemanticCorrectInfo semanticCorrectInfo) { + super.correct(semanticCorrectInfo); Long modelId = semanticCorrectInfo.getParseInfo().getModelId(); - String preSql = semanticCorrectInfo.getSql(); - semanticCorrectInfo.setPreSql(preSql); - String sql = SqlParserUpdateHelper.replaceTable(preSql, TABLE_PREFIX + modelId); + String sql = SqlParserUpdateHelper.replaceTable(semanticCorrectInfo.getSql(), TABLE_PREFIX + modelId); semanticCorrectInfo.setSql(sql); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java index fc607194a..dd16fa462 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java @@ -30,7 +30,9 @@ import org.springframework.util.CollectionUtils; public class WhereCorrector extends BaseSemanticCorrector { @Override - public void correct(SemanticCorrectInfo semanticCorrectInfo) throws JSQLParserException { + public void correct(SemanticCorrectInfo semanticCorrectInfo) { + + super.correct(semanticCorrectInfo); addDateIfNotExist(semanticCorrectInfo); @@ -41,24 +43,27 @@ public class WhereCorrector extends BaseSemanticCorrector { updateFieldValueByTechName(semanticCorrectInfo); } - private void addQueryFilter(SemanticCorrectInfo semanticCorrectInfo) throws JSQLParserException { + private void addQueryFilter(SemanticCorrectInfo semanticCorrectInfo) { String queryFilter = getQueryFilter(semanticCorrectInfo.getQueryFilters()); String preSql = semanticCorrectInfo.getSql(); if (StringUtils.isNotEmpty(queryFilter)) { log.info("add queryFilter to preSql :{}", queryFilter); - Expression expression = CCJSqlParserUtil.parseCondExpression(queryFilter); + Expression expression = null; + try { + expression = CCJSqlParserUtil.parseCondExpression(queryFilter); + } catch (JSQLParserException e) { + log.error("parseCondExpression", e); + } String sql = SqlParserUpdateHelper.addWhere(preSql, expression); - semanticCorrectInfo.setPreSql(preSql); semanticCorrectInfo.setSql(sql); } } private void parserDateDiffFunction(SemanticCorrectInfo semanticCorrectInfo) { - String preSql = semanticCorrectInfo.getSql(); - semanticCorrectInfo.setPreSql(preSql); - String sql = SqlParserUpdateHelper.replaceFunction(preSql); + String sql = semanticCorrectInfo.getSql(); + sql = SqlParserUpdateHelper.replaceFunction(sql); semanticCorrectInfo.setSql(sql); } @@ -98,9 +103,7 @@ public class WhereCorrector extends BaseSemanticCorrector { } Map> aliasAndBizNameToTechName = getAliasAndBizNameToTechName(dimensions); - String preSql = semanticCorrectInfo.getSql(); - semanticCorrectInfo.setPreSql(preSql); - String sql = SqlParserUpdateHelper.replaceValue(preSql, aliasAndBizNameToTechName); + String sql = SqlParserUpdateHelper.replaceValue(semanticCorrectInfo.getSql(), aliasAndBizNameToTechName); semanticCorrectInfo.setSql(sql); return; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java index b007f9ce6..5ac322b68 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java @@ -37,14 +37,12 @@ public class HanlpDictMapper implements SchemaMapper { String queryText = queryContext.getRequest().getQueryText(); List terms = HanlpHelper.getTerms(queryText); - for (Term term : terms) { - log.info("word:{},nature:{},frequency:{}", term.word, term.nature.toString(), term.getFrequency()); - } - QueryMatchStrategy matchStrategy = ContextUtils.getBean(QueryMatchStrategy.class); MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class); Set detectModelIds = mapperHelper.getModelIds(queryContext.getRequest()); + terms = filterByModelIds(terms, detectModelIds); + Map> matchResult = matchStrategy.match(queryContext.getRequest(), terms, detectModelIds); @@ -57,6 +55,26 @@ public class HanlpDictMapper implements SchemaMapper { convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo(), terms); } + private List filterByModelIds(List terms, Set detectModelIds) { + for (Term term : terms) { + log.info("before word:{},nature:{},frequency:{}", term.word, term.nature.toString(), term.getFrequency()); + } + if (CollectionUtils.isNotEmpty(detectModelIds)) { + terms = terms.stream().filter(term -> { + Long modelId = NatureHelper.getModelId(term.getNature().toString()); + if (Objects.nonNull(modelId)) { + return detectModelIds.contains(modelId); + } + return false; + }).collect(Collectors.toList()); + } + for (Term term : terms) { + log.info("after filter word:{},nature:{},frequency:{}", term.word, term.nature.toString(), + term.getFrequency()); + } + return terms; + } + private void convertTermsToSchemaMapInfo(List mapResults, SchemaMapInfo schemaMap, List terms) { if (CollectionUtils.isEmpty(mapResults)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java index 9b6c24422..b0a49338c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java @@ -132,6 +132,7 @@ public class LLMDslParser implements SemanticParser { if (StringUtils.isEmpty(correctorSql)) { correctorSql = semanticCorrectInfo.getSql(); } + parseInfo.setLogicSql(correctorSql); List expressions = SqlParserSelectHelper.getFilterExpression(correctorSql); //set dataInfo try { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 4f4d8f0f4..e2b3f4b28 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -165,7 +165,7 @@ public class QueryServiceImpl implements QueryService { if (Objects.isNull(explain)) { return; } - parseInfo.setSql(explain.getSql()); + parseInfo.setQuerySql(explain.getSql()); } @Override diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FiledFilterReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FiledFilterReplaceVisitor.java index 9950e8e43..abe60778f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FiledFilterReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FiledFilterReplaceVisitor.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.util.jsqlparser; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index aa0a5ff20..4835d1225 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -32,8 +32,8 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ com.tencent.supersonic.chat.api.component.SemanticCorrector=\ com.tencent.supersonic.chat.corrector.GlobalCorrector, \ - com.tencent.supersonic.chat.corrector.TableCorrector, \ com.tencent.supersonic.chat.corrector.GroupByCorrector, \ com.tencent.supersonic.chat.corrector.SelectCorrector, \ com.tencent.supersonic.chat.corrector.WhereCorrector, \ - com.tencent.supersonic.chat.corrector.HavingCorrector \ No newline at end of file + com.tencent.supersonic.chat.corrector.HavingCorrector, \ + com.tencent.supersonic.chat.corrector.TableCorrector \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 3e8c7a5d6..ed43f0e5a 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -32,8 +32,8 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ com.tencent.supersonic.chat.api.component.SemanticCorrector=\ com.tencent.supersonic.chat.corrector.GlobalCorrector, \ - com.tencent.supersonic.chat.corrector.TableCorrector, \ com.tencent.supersonic.chat.corrector.GroupByCorrector, \ com.tencent.supersonic.chat.corrector.SelectCorrector, \ com.tencent.supersonic.chat.corrector.WhereCorrector, \ - com.tencent.supersonic.chat.corrector.HavingCorrector + com.tencent.supersonic.chat.corrector.HavingCorrector, \ + com.tencent.supersonic.chat.corrector.TableCorrector \ No newline at end of file