From b6e4577784e206787087caec76e19a2db9750117 Mon Sep 17 00:00:00 2001 From: jipeli <54889677+jipeli@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:53:49 +0800 Subject: [PATCH 1/3] [improvement][headless] Fix interval time sql error(#1717) (#1786) --- .../common/calcite/SemanticSqlDialect.java | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlDialect.java b/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlDialect.java index c92fe49ec..d8323d5be 100644 --- a/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlDialect.java +++ b/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlDialect.java @@ -1,14 +1,16 @@ package com.tencent.supersonic.common.calcite; import com.google.common.base.Preconditions; -import org.apache.calcite.sql.SqlDialect; -import org.apache.calcite.sql.SqlIntervalLiteral; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.sql.*; +import org.apache.calcite.sql.fun.SqlMonotonicBinaryOperator; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlConformance; import org.checkerframework.checker.nullness.qual.Nullable; -/** customize the SqlDialect */ +/** + * customize the SqlDialect + */ public class SemanticSqlDialect extends SqlDialect { private static final SqlConformance tagTdwSqlConformance = new SemanticSqlConformance(); @@ -85,4 +87,36 @@ public class SemanticSqlDialect extends SqlDialect { public boolean supportsNestedAggregations() { return false; } + + + public void unparseCall(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { + if (modifyIntervalTime(call, writer, leftPrec, rightPrec)) { + return; + } + super.unparseCall(writer, call, leftPrec, rightPrec); + + } + + private Boolean modifyIntervalTime(SqlCall call, SqlWriter writer, int leftPrec, + int rightPrec) { + SqlOperator operator = call.getOperator(); + if (operator instanceof SqlMonotonicBinaryOperator + && call.getKind().equals(SqlKind.TIMES)) { + if (call.getOperandList() != null && call.getOperandList().size() == 2 + && call.getOperandList().get(1) instanceof SqlIntervalLiteral) { + SqlIntervalLiteral intervalOperand = + (SqlIntervalLiteral) call.getOperandList().get(1); + SqlIntervalLiteral.IntervalValue interval = + (SqlIntervalLiteral.IntervalValue) intervalOperand.getValue(); + call.setOperand(1, SqlNumericLiteral.createExactNumeric(interval.toString(), + SqlParserPos.ZERO)); + writer.keyword(SqlKind.INTERVAL.name()); + call.unparse(writer, leftPrec, rightPrec); + unparseSqlIntervalQualifier(writer, interval.getIntervalQualifier(), + RelDataTypeSystem.DEFAULT); + return true; + } + } + return false; + } } From 0a86a63937220daba89403fde92d388fe7ab36a5 Mon Sep 17 00:00:00 2001 From: daikon <1059907724@qq.com> Date: Fri, 11 Oct 2024 22:27:46 +0800 Subject: [PATCH 2/3] opt queryDictValue (#1787) --- .../headless/api/pojo/DimValueMap.java | 2 ++ .../api/pojo/response/DictValueDimResp.java | 2 +- .../chat/knowledge/file/FileHandlerImpl.java | 20 +++++++++------- .../service/impl/DictTaskServiceImpl.java | 23 +++++++++++++++++-- .../server/utils/DimensionConverter.java | 11 ++++++--- 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java index 3fc060fbe..f6c3ca0af 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java @@ -16,4 +16,6 @@ public class DimValueMap { /** dimension value for user query */ private List alias = new ArrayList<>(); + + private String value; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java index f2cb1ce11..cbe8c38ed 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java @@ -13,7 +13,7 @@ import java.util.List; @Data @ToString public class DictValueDimResp extends DictValueResp { - /** dimension value for result show */ + /** dimension value */ private String bizName; /** dimension value for user query */ diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java index f92afc531..48c207da5 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java @@ -28,9 +28,6 @@ import java.util.stream.Stream; @Component public class FileHandlerImpl implements FileHandler { - @Value("${dict.value.max.count.page:1000}") - private int dictValueMaxCountPage; - public static final String FILE_SPILT = File.separator; private final LocalFileConfig localFileConfig; @@ -89,7 +86,7 @@ public class FileHandlerImpl implements FileHandler { } private PageInfo getDictValueRespPagWithKey(String fileName, - DictValueReq dictValueReq) { + DictValueReq dictValueReq) { PageInfo dictValueRespPageInfo = new PageInfo<>(); dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize()); dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent()); @@ -121,11 +118,11 @@ public class FileHandlerImpl implements FileHandler { } private PageInfo getDictValueRespPagWithoutKey(String fileName, - DictValueReq dictValueReq) { + DictValueReq dictValueReq) { PageInfo dictValueRespPageInfo = new PageInfo<>(); String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName; - Long fileLineNum = Math.min(dictValueMaxCountPage, getFileLineNum(filePath)); - Integer startLine = 1; + Long fileLineNum = getFileLineNum(filePath); + Integer startLine = Math.max(1, (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1); Integer endLine = Integer.valueOf( Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + ""); List dictValueRespList = getFileData(filePath, startLine, endLine); @@ -135,7 +132,7 @@ public class FileHandlerImpl implements FileHandler { dictValueRespPageInfo.setTotal(fileLineNum); dictValueRespPageInfo.setList(dictValueRespList); dictValueRespPageInfo.setHasNextPage(endLine >= fileLineNum ? false : true); - dictValueRespPageInfo.setHasPreviousPage(startLine <= 0 ? false : true); + dictValueRespPageInfo.setHasPreviousPage(startLine <= 1 ? false : true); return dictValueRespPageInfo; } @@ -149,6 +146,13 @@ public class FileHandlerImpl implements FileHandler { return null; } + /** + * + * @param filePath + * @param startLine 1开始 + * @param endLine + * @return + */ private List getFileData(String filePath, Integer startLine, Integer endLine) { List fileData = new ArrayList<>(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java index 43c50321a..04ae85233 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java @@ -59,8 +59,8 @@ public class DictTaskServiceImpl implements DictTaskService { private final DimensionService dimensionService; public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter, - DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService, - DimensionService dimensionService) { + DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService, + DimensionService dimensionService) { this.dictRepository = dictRepository; this.dictConverter = dictConverter; this.dictUtils = dictUtils; @@ -178,9 +178,28 @@ public class DictTaskServiceImpl implements DictTaskService { PageInfo dictValueRespList = fileHandler.queryDictValue(fileName, dictValueReq); PageInfo result = convert2DictValueDimRespPage(dictValueRespList); + fillDimMapInfo(result.getList(), dictValueReq.getItemId()); return result; } + private void fillDimMapInfo(List list, Long dimId) { + DimensionResp dimResp = dimensionService.getDimension(dimId); + if (CollectionUtils.isEmpty(dimResp.getDimValueMaps())) { + return; + } + Map valueAndMap = dimResp.getDimValueMaps().stream() + .collect(Collectors.toMap(dim -> dim.getValue(), v -> v, (v1, v2) -> v2)); + if (CollectionUtils.isEmpty(valueAndMap)) { + return; + } + list.stream().forEach(dictValueDimResp -> { + String dimValue = dictValueDimResp.getValue(); + if (valueAndMap.containsKey(dimValue) && Objects.nonNull(valueAndMap.get(dimValue))) { + dictValueDimResp.setAlias(valueAndMap.get(dimValue).getAlias()); + } + }); + } + private PageInfo convert2DictValueDimRespPage( PageInfo dictValueRespPage) { PageInfo result = new PageInfo<>(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java index 5a8e3379f..4610db2c9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java @@ -34,6 +34,11 @@ public class DimensionConverter { dimensionDO.setDefaultValues(JSONObject.toJSONString(dimensionReq.getDefaultValues())); } if (!CollectionUtils.isEmpty(dimensionReq.getDimValueMaps())) { + List dimValueMaps = dimensionReq.getDimValueMaps(); + dimValueMaps.stream().forEach(dimValueMap -> { + dimValueMap.setTechName(dimValueMap.getValue()); + }); + dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimensionReq.getDimValueMaps())); } else { dimensionDO.setDimValueMaps(JSONObject.toJSONString(new ArrayList<>())); @@ -70,7 +75,7 @@ public class DimensionConverter { } public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO, - Map modelRespMap) { + Map modelRespMap) { DimensionResp dimensionResp = new DimensionResp(); BeanUtils.copyProperties(dimensionDO, dimensionResp); dimensionResp.setModelName( @@ -118,11 +123,11 @@ public class DimensionConverter { } public static List filterByDataSet(List dimensionResps, - DataSetResp dataSetResp) { + DataSetResp dataSetResp) { return dimensionResps.stream() .filter(dimensionResp -> dataSetResp.dimensionIds().contains(dimensionResp.getId()) || dataSetResp.getAllIncludeAllModels() - .contains(dimensionResp.getModelId())) + .contains(dimensionResp.getModelId())) .collect(Collectors.toList()); } } From 72d01bb4198cd1792851f37af1f9885c13e3196f Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Fri, 11 Oct 2024 23:09:13 +0800 Subject: [PATCH 3/3] [improvement][chat] Optimize the MemoryReviewTask task by adding individual exception handling (#1788) --- .../chat/server/memory/MemoryReviewTask.java | 13 ++++++++----- .../chat/server/service/impl/AgentServiceImpl.java | 3 +-- .../supersonic/headless/api/pojo/ModelSchema.java | 3 ++- .../headless/chat/corrector/LLMSqlCorrector.java | 3 ++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java index 8014a9256..7578b35ae 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java @@ -49,11 +49,14 @@ public class MemoryReviewTask { @Scheduled(fixedDelay = 60 * 1000) public void review() { - try { - memoryService.getMemoriesForLlmReview().stream().forEach(this::processMemory); - } catch (Exception e) { - log.error("Exception occurred during memory review task", e); - } + memoryService.getMemoriesForLlmReview().stream().forEach(memory -> { + try { + processMemory(memory); + } catch (Exception e) { + log.error("Exception occurred while processing memory with id {}: {}", + memory.getId(), e.getMessage(), e); + } + }); } private void processMemory(ChatMemoryDO m) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index a183a94e0..702c473e6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -105,8 +105,7 @@ public class AgentServiceImpl extends ServiceImpl implem } private synchronized void doExecuteAgentExamples(Agent agent) { - if (!agent.containsDatasetTool() - || !agent.enableMemoryReview() + if (!agent.containsDatasetTool() || !agent.enableMemoryReview() || !ModelConfigHelper.testConnection( ModelConfigHelper.getChatModelConfig(agent, ChatModelType.TEXT_TO_SQL)) || CollectionUtils.isEmpty(agent.getExamples())) { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java index fdf7750b6..1eafb8a44 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.headless.api.pojo; -import java.util.List; import lombok.Data; +import java.util.List; + @Data public class ModelSchema { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java index 31d1467e3..f1ba1c49b 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java @@ -56,7 +56,8 @@ public class LLMSqlCorrector extends BaseSemanticCorrector { return; } - ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatQueryContext.getModelConfig()); + ChatLanguageModel chatLanguageModel = + ModelProvider.getChatModel(chatQueryContext.getModelConfig()); SemanticSqlExtractor extractor = AiServices.create(SemanticSqlExtractor.class, chatLanguageModel); Prompt prompt = generatePrompt(chatQueryContext.getQueryText(), semanticParseInfo);