[feature][headless-chat]Introduce ChatApp to support more flexible chat model config.#1739

2025-12-10 19:51:00 +00:00 · 2024-10-12 11:58:34 +08:00
parent 2717a1603c 72d01bb419
commit 0cce0a76b4
9 changed files with 93 additions and 27 deletions
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java
@@ -55,11 +55,14 @@ public class MemoryReviewTask {

    @Scheduled(fixedDelay = 60 * 1000)
    public void review() {
-        try {
-            memoryService.getMemoriesForLlmReview().stream().forEach(this::processMemory);
-        } catch (Exception e) {
-            log.error("Exception occurred during memory review task", e);
-        }
+        memoryService.getMemoriesForLlmReview().stream().forEach(memory -> {
+            try {
+                processMemory(memory);
+            } catch (Exception e) {
+                log.error("Exception occurred while processing memory with id {}: {}",
+                        memory.getId(), e.getMessage(), e);
+            }
+        });
    }

    private void processMemory(ChatMemoryDO m) {
--- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java
+++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java
@@ -16,9 +16,7 @@ import com.tencent.supersonic.chat.server.service.ChatModelService;
 import com.tencent.supersonic.chat.server.service.ChatQueryService;
 import com.tencent.supersonic.chat.server.service.MemoryService;
 import com.tencent.supersonic.common.pojo.ChatApp;
-import com.tencent.supersonic.common.pojo.ChatModelConfig;
 import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.headless.chat.parser.llm.OnePassSCSqlGenStrategy;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
--- a/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlDialect.java
+++ b/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlDialect.java
@@ -1,14 +1,16 @@
 package com.tencent.supersonic.common.calcite;

 import com.google.common.base.Preconditions;
-import org.apache.calcite.sql.SqlDialect;
-import org.apache.calcite.sql.SqlIntervalLiteral;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlWriter;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.*;
+import org.apache.calcite.sql.fun.SqlMonotonicBinaryOperator;
+import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.sql.validate.SqlConformance;
 import org.checkerframework.checker.nullness.qual.Nullable;

-/** customize the SqlDialect */
+/**
+ * customize the SqlDialect
+ */
 public class SemanticSqlDialect extends SqlDialect {

    private static final SqlConformance tagTdwSqlConformance = new SemanticSqlConformance();
@@ -85,4 +87,36 @@ public class SemanticSqlDialect extends SqlDialect {
    public boolean supportsNestedAggregations() {
        return false;
    }
+
+
+    public void unparseCall(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
+        if (modifyIntervalTime(call, writer, leftPrec, rightPrec)) {
+            return;
+        }
+        super.unparseCall(writer, call, leftPrec, rightPrec);
+
+    }
+
+    private Boolean modifyIntervalTime(SqlCall call, SqlWriter writer, int leftPrec,
+            int rightPrec) {
+        SqlOperator operator = call.getOperator();
+        if (operator instanceof SqlMonotonicBinaryOperator
+                && call.getKind().equals(SqlKind.TIMES)) {
+            if (call.getOperandList() != null && call.getOperandList().size() == 2
+                    && call.getOperandList().get(1) instanceof SqlIntervalLiteral) {
+                SqlIntervalLiteral intervalOperand =
+                        (SqlIntervalLiteral) call.getOperandList().get(1);
+                SqlIntervalLiteral.IntervalValue interval =
+                        (SqlIntervalLiteral.IntervalValue) intervalOperand.getValue();
+                call.setOperand(1, SqlNumericLiteral.createExactNumeric(interval.toString(),
+                        SqlParserPos.ZERO));
+                writer.keyword(SqlKind.INTERVAL.name());
+                call.unparse(writer, leftPrec, rightPrec);
+                unparseSqlIntervalQualifier(writer, interval.getIntervalQualifier(),
+                        RelDataTypeSystem.DEFAULT);
+                return true;
+            }
+        }
+        return false;
+    }
 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java
@@ -16,4 +16,6 @@ public class DimValueMap {

    /** dimension value for user query */
    private List<String> alias = new ArrayList<>();
+
+    private String value;
 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java
@@ -1,8 +1,9 @@
 package com.tencent.supersonic.headless.api.pojo;

-import java.util.List;
 import lombok.Data;

+import java.util.List;
+
@Data
 public class ModelSchema {

--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java
@@ -13,7 +13,7 @@ import java.util.List;
@Data
@ToString
 public class DictValueDimResp extends DictValueResp {
-    /** dimension value for result show */
+    /** dimension value */
    private String bizName;

    /** dimension value for user query */
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java
@@ -28,9 +28,6 @@ import java.util.stream.Stream;
@Component
 public class FileHandlerImpl implements FileHandler {

-    @Value("${dict.value.max.count.page:1000}")
-    private int dictValueMaxCountPage;
-
    public static final String FILE_SPILT = File.separator;

    private final LocalFileConfig localFileConfig;
@@ -89,7 +86,7 @@ public class FileHandlerImpl implements FileHandler {
    }

    private PageInfo<DictValueResp> getDictValueRespPagWithKey(String fileName,
-            DictValueReq dictValueReq) {
+                                                               DictValueReq dictValueReq) {
        PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
        dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize());
        dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
@@ -121,11 +118,11 @@ public class FileHandlerImpl implements FileHandler {
    }

    private PageInfo<DictValueResp> getDictValueRespPagWithoutKey(String fileName,
-            DictValueReq dictValueReq) {
+                                                                  DictValueReq dictValueReq) {
        PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
        String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
-        Long fileLineNum = Math.min(dictValueMaxCountPage, getFileLineNum(filePath));
-        Integer startLine = 1;
+        Long fileLineNum = getFileLineNum(filePath);
+        Integer startLine = Math.max(1, (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1);
        Integer endLine = Integer.valueOf(
                Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + "");
        List<DictValueResp> dictValueRespList = getFileData(filePath, startLine, endLine);
@@ -135,7 +132,7 @@ public class FileHandlerImpl implements FileHandler {
        dictValueRespPageInfo.setTotal(fileLineNum);
        dictValueRespPageInfo.setList(dictValueRespList);
        dictValueRespPageInfo.setHasNextPage(endLine >= fileLineNum ? false : true);
-        dictValueRespPageInfo.setHasPreviousPage(startLine <= 0 ? false : true);
+        dictValueRespPageInfo.setHasPreviousPage(startLine <= 1 ? false : true);
        return dictValueRespPageInfo;
    }

@@ -149,6 +146,13 @@ public class FileHandlerImpl implements FileHandler {
        return null;
    }

+    /**
+     *
+     * @param filePath
+     * @param startLine 1开始
+     * @param endLine
+     * @return
+     */
    private List<DictValueResp> getFileData(String filePath, Integer startLine, Integer endLine) {
        List<DictValueResp> fileData = new ArrayList<>();

--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
@@ -59,8 +59,8 @@ public class DictTaskServiceImpl implements DictTaskService {
    private final DimensionService dimensionService;

    public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter,
-            DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService,
-            DimensionService dimensionService) {
+                               DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService,
+                               DimensionService dimensionService) {
        this.dictRepository = dictRepository;
        this.dictConverter = dictConverter;
        this.dictUtils = dictUtils;
@@ -178,9 +178,28 @@ public class DictTaskServiceImpl implements DictTaskService {
        PageInfo<DictValueResp> dictValueRespList =
                fileHandler.queryDictValue(fileName, dictValueReq);
        PageInfo<DictValueDimResp> result = convert2DictValueDimRespPage(dictValueRespList);
+        fillDimMapInfo(result.getList(), dictValueReq.getItemId());
        return result;
    }

+    private void fillDimMapInfo(List<DictValueDimResp> list, Long dimId) {
+        DimensionResp dimResp = dimensionService.getDimension(dimId);
+        if (CollectionUtils.isEmpty(dimResp.getDimValueMaps())) {
+            return;
+        }
+        Map<String, DimValueMap> valueAndMap = dimResp.getDimValueMaps().stream()
+                .collect(Collectors.toMap(dim -> dim.getValue(), v -> v, (v1, v2) -> v2));
+        if (CollectionUtils.isEmpty(valueAndMap)) {
+            return;
+        }
+        list.stream().forEach(dictValueDimResp -> {
+            String dimValue = dictValueDimResp.getValue();
+            if (valueAndMap.containsKey(dimValue) && Objects.nonNull(valueAndMap.get(dimValue))) {
+                dictValueDimResp.setAlias(valueAndMap.get(dimValue).getAlias());
+            }
+        });
+    }
+
    private PageInfo<DictValueDimResp> convert2DictValueDimRespPage(
            PageInfo<DictValueResp> dictValueRespPage) {
        PageInfo<DictValueDimResp> result = new PageInfo<>();
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java
@@ -34,6 +34,11 @@ public class DimensionConverter {
            dimensionDO.setDefaultValues(JSONObject.toJSONString(dimensionReq.getDefaultValues()));
        }
        if (!CollectionUtils.isEmpty(dimensionReq.getDimValueMaps())) {
+            List<DimValueMap> dimValueMaps = dimensionReq.getDimValueMaps();
+            dimValueMaps.stream().forEach(dimValueMap -> {
+                dimValueMap.setTechName(dimValueMap.getValue());
+            });
+
            dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimensionReq.getDimValueMaps()));
        } else {
            dimensionDO.setDimValueMaps(JSONObject.toJSONString(new ArrayList<>()));
@@ -70,7 +75,7 @@ public class DimensionConverter {
    }

    public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO,
-            Map<Long, ModelResp> modelRespMap) {
+                                                      Map<Long, ModelResp> modelRespMap) {
        DimensionResp dimensionResp = new DimensionResp();
        BeanUtils.copyProperties(dimensionDO, dimensionResp);
        dimensionResp.setModelName(
@@ -118,11 +123,11 @@ public class DimensionConverter {
    }

    public static List<DimensionResp> filterByDataSet(List<DimensionResp> dimensionResps,
-            DataSetResp dataSetResp) {
+                                                      DataSetResp dataSetResp) {
        return dimensionResps.stream()
                .filter(dimensionResp -> dataSetResp.dimensionIds().contains(dimensionResp.getId())
                        || dataSetResp.getAllIncludeAllModels()
-                                .contains(dimensionResp.getModelId()))
+                        .contains(dimensionResp.getModelId()))
                .collect(Collectors.toList());
    }
 }