From e0088e8f8fd549f1d3b477670e4f048cad1a43e7 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Thu, 9 Nov 2023 22:17:13 +0800 Subject: [PATCH] update work to doXXX and integrate the SemanticCorrector code (#351) --- .../chat/corrector/BaseSemanticCorrector.java | 4 +-- .../chat/corrector/GlobalAfterCorrector.java | 30 ------------------- .../chat/corrector/GroupByCorrector.java | 2 +- .../chat/corrector/HavingCorrector.java | 26 +++++++++++++++- ...oreCorrector.java => SchemaCorrector.java} | 4 +-- .../chat/corrector/SelectCorrector.java | 2 +- .../chat/corrector/WhereCorrector.java | 2 +- .../supersonic/chat/mapper/BaseMapper.java | 4 +-- .../chat/mapper/EmbeddingMapper.java | 2 +- .../supersonic/chat/mapper/EntityMapper.java | 2 +- .../chat/mapper/FuzzyNameMapper.java | 2 +- .../chat/mapper/HanlpDictMapper.java | 2 +- .../main/resources/META-INF/spring.factories | 5 ++-- .../main/resources/META-INF/spring.factories | 5 ++-- 14 files changed, 42 insertions(+), 50 deletions(-) delete mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalAfterCorrector.java rename chat/core/src/main/java/com/tencent/supersonic/chat/corrector/{GlobalBeforeCorrector.java => SchemaCorrector.java} (96%) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java index 4231740cf..30f0ab9d8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java @@ -30,7 +30,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getCorrectS2SQL())) { return; } - work(queryReq, semanticParseInfo); + doCorrect(queryReq, semanticParseInfo); log.info("sqlCorrection:{} sql:{}", this.getClass().getSimpleName(), semanticParseInfo.getSqlInfo()); } catch (Exception e) { log.error(String.format("correct error,sqlInfo:%s", semanticParseInfo.getSqlInfo()), e); @@ -38,7 +38,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { } - public abstract void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo); + public abstract void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo); protected Map getFieldNameMap(Long modelId) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalAfterCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalAfterCorrector.java deleted file mode 100644 index 3a137b222..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalAfterCorrector.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.supersonic.chat.corrector; - -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryReq; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.expression.Expression; - -@Slf4j -public class GlobalAfterCorrector extends BaseSemanticCorrector { - - @Override - public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { - - String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); - if (!SqlParserSelectFunctionHelper.hasAggregateFunction(logicSql)) { - return; - } - Expression havingExpression = SqlParserSelectHelper.getHavingExpression(logicSql); - if (Objects.nonNull(havingExpression)) { - String replaceSql = SqlParserAddHelper.addFunctionToSelect(logicSql, havingExpression); - semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceSql); - } - return; - } - -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java index d04c12e9d..bbd00d197 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java @@ -20,7 +20,7 @@ import org.springframework.util.CollectionUtils; public class GroupByCorrector extends BaseSemanticCorrector { @Override - public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { + public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { addGroupByFields(semanticParseInfo); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java index deb0e8093..3de527134 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java @@ -5,19 +5,30 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.knowledge.service.SchemaService; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.Expression; import org.springframework.util.CollectionUtils; @Slf4j public class HavingCorrector extends BaseSemanticCorrector { @Override - public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { + public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { //add aggregate to all metric + addHaving(semanticParseInfo); + + //add having expression filed to select + addHavingToSelect(semanticParseInfo); + } + + private void addHaving(SemanticParseInfo semanticParseInfo) { Long modelId = semanticParseInfo.getModel().getModel(); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); @@ -32,4 +43,17 @@ public class HavingCorrector extends BaseSemanticCorrector { semanticParseInfo.getSqlInfo().setCorrectS2SQL(havingSql); } + private void addHavingToSelect(SemanticParseInfo semanticParseInfo) { + String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); + if (!SqlParserSelectFunctionHelper.hasAggregateFunction(logicSql)) { + return; + } + Expression havingExpression = SqlParserSelectHelper.getHavingExpression(logicSql); + if (Objects.nonNull(havingExpression)) { + String replaceSql = SqlParserAddHelper.addFunctionToSelect(logicSql, havingExpression); + semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceSql); + } + return; + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SchemaCorrector.java similarity index 96% rename from chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SchemaCorrector.java index 7c71abbd0..5c54da6fb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SchemaCorrector.java @@ -18,10 +18,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @Slf4j -public class GlobalBeforeCorrector extends BaseSemanticCorrector { +public class SchemaCorrector extends BaseSemanticCorrector { @Override - public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { + public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { replaceAlias(semanticParseInfo); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java index eb4c9814c..3135ac420 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java @@ -11,7 +11,7 @@ import org.springframework.util.CollectionUtils; public class SelectCorrector extends BaseSemanticCorrector { @Override - public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { + public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL(); List aggregateFields = SqlParserSelectHelper.getAggregateFields(logicSql); List selectFields = SqlParserSelectHelper.getSelectFields(logicSql); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java index 69799068f..ed5a7f037 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java @@ -32,7 +32,7 @@ import org.springframework.util.CollectionUtils; public class WhereCorrector extends BaseSemanticCorrector { @Override - public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { + public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { addDateIfNotExist(semanticParseInfo); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/BaseMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/BaseMapper.java index c50becc54..a8040e3ef 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/BaseMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/BaseMapper.java @@ -32,7 +32,7 @@ public abstract class BaseMapper implements SchemaMapper { log.info("before {},mapInfo:{}", simpleName, queryContext.getMapInfo().getModelElementMatches()); try { - work(queryContext); + doMap(queryContext); } catch (Exception e) { log.error("work error", e); } @@ -41,7 +41,7 @@ public abstract class BaseMapper implements SchemaMapper { log.info("after {},cost:{},mapInfo:{}", simpleName, cost, queryContext.getMapInfo().getModelElementMatches()); } - public abstract void work(QueryContext queryContext); + public abstract void doMap(QueryContext queryContext); public void addToSchemaMap(SchemaMapInfo schemaMap, Long modelId, SchemaElementMatch schemaElementMatch) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java index e6665db9a..6c41881ed 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EmbeddingMapper.java @@ -21,7 +21,7 @@ import org.apache.commons.lang3.StringUtils; public class EmbeddingMapper extends BaseMapper { @Override - public void work(QueryContext queryContext) { + public void doMap(QueryContext queryContext) { //1. query from embedding by queryText String queryText = queryContext.getRequest().getQueryText(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java index a9ccf3768..8fae3e0f8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java @@ -22,7 +22,7 @@ import org.springframework.util.CollectionUtils; public class EntityMapper extends BaseMapper { @Override - public void work(QueryContext queryContext) { + public void doMap(QueryContext queryContext) { SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); for (Long modelId : schemaMapInfo.getMatchedModels()) { List schemaElementMatchList = schemaMapInfo.getMatchedElements(modelId); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java index 399a88ae0..df0184d90 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java @@ -23,7 +23,7 @@ import org.springframework.util.CollectionUtils; public class FuzzyNameMapper extends BaseMapper { @Override - public void work(QueryContext queryContext) { + public void doMap(QueryContext queryContext) { List terms = HanlpHelper.getTerms(queryContext.getRequest().getQueryText()); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java index 3054b90e2..ef64a52a2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java @@ -25,7 +25,7 @@ import org.apache.commons.collections.CollectionUtils; public class HanlpDictMapper extends BaseMapper { @Override - public void work(QueryContext queryContext) { + public void doMap(QueryContext queryContext) { String queryText = queryContext.getRequest().getQueryText(); List terms = HanlpHelper.getTerms(queryText); diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index 920f83de7..669b18857 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -11,12 +11,11 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser com.tencent.supersonic.chat.api.component.SemanticCorrector=\ - com.tencent.supersonic.chat.corrector.GlobalBeforeCorrector, \ + com.tencent.supersonic.chat.corrector.SchemaCorrector, \ com.tencent.supersonic.chat.corrector.SelectCorrector, \ com.tencent.supersonic.chat.corrector.WhereCorrector, \ com.tencent.supersonic.chat.corrector.GroupByCorrector, \ - com.tencent.supersonic.chat.corrector.HavingCorrector, \ - com.tencent.supersonic.chat.corrector.GlobalAfterCorrector + com.tencent.supersonic.chat.corrector.HavingCorrector com.tencent.supersonic.chat.api.component.SemanticInterpreter=\ com.tencent.supersonic.knowledge.semantic.RemoteSemanticInterpreter diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 7eabfbcc3..395a4d9c0 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -12,12 +12,11 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser com.tencent.supersonic.chat.api.component.SemanticCorrector=\ - com.tencent.supersonic.chat.corrector.GlobalBeforeCorrector, \ + com.tencent.supersonic.chat.corrector.SchemaCorrector, \ com.tencent.supersonic.chat.corrector.SelectCorrector, \ com.tencent.supersonic.chat.corrector.WhereCorrector, \ com.tencent.supersonic.chat.corrector.GroupByCorrector, \ - com.tencent.supersonic.chat.corrector.HavingCorrector, \ - com.tencent.supersonic.chat.corrector.GlobalAfterCorrector + com.tencent.supersonic.chat.corrector.HavingCorrector com.tencent.supersonic.chat.api.component.SemanticInterpreter=\ com.tencent.supersonic.knowledge.semantic.LocalSemanticInterpreter