[feature][headless-chat]Introduce ChatApp to support more flexible chat model config.#1739

chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java

@@ -55,11 +55,14 @@ public class MemoryReviewTask {
 
     @Scheduled(fixedDelay = 60 * 1000)
     public void review() {
+        memoryService.getMemoriesForLlmReview().stream().forEach(memory -> {
             try {
-            memoryService.getMemoriesForLlmReview().stream().forEach(this::processMemory);
+                processMemory(memory);
             } catch (Exception e) {
-            log.error("Exception occurred during memory review task", e);
+                log.error("Exception occurred while processing memory with id {}: {}",
+                        memory.getId(), e.getMessage(), e);
             }
+        });
     }
 
     private void processMemory(ChatMemoryDO m) {

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java

@@ -16,9 +16,7 @@ import com.tencent.supersonic.chat.server.service.ChatModelService;
 import com.tencent.supersonic.chat.server.service.ChatQueryService;
 import com.tencent.supersonic.chat.server.service.MemoryService;
 import com.tencent.supersonic.common.pojo.ChatApp;
-import com.tencent.supersonic.common.pojo.ChatModelConfig;
 import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.headless.chat.parser.llm.OnePassSCSqlGenStrategy;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;

common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlDialect.java

@@ -1,14 +1,16 @@
 package com.tencent.supersonic.common.calcite;
 
 import com.google.common.base.Preconditions;
-import org.apache.calcite.sql.SqlDialect;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
-import org.apache.calcite.sql.SqlIntervalLiteral;
+import org.apache.calcite.sql.*;
-import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.fun.SqlMonotonicBinaryOperator;
-import org.apache.calcite.sql.SqlWriter;
+import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.sql.validate.SqlConformance;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
-/** customize the SqlDialect */
+/**
+ * customize the SqlDialect
+ */
 public class SemanticSqlDialect extends SqlDialect {
 
     private static final SqlConformance tagTdwSqlConformance = new SemanticSqlConformance();
@@ -85,4 +87,36 @@ public class SemanticSqlDialect extends SqlDialect {
     public boolean supportsNestedAggregations() {
         return false;
     }
+
+
+    public void unparseCall(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
+        if (modifyIntervalTime(call, writer, leftPrec, rightPrec)) {
+            return;
+        }
+        super.unparseCall(writer, call, leftPrec, rightPrec);
+
+    }
+
+    private Boolean modifyIntervalTime(SqlCall call, SqlWriter writer, int leftPrec,
+            int rightPrec) {
+        SqlOperator operator = call.getOperator();
+        if (operator instanceof SqlMonotonicBinaryOperator
+                && call.getKind().equals(SqlKind.TIMES)) {
+            if (call.getOperandList() != null && call.getOperandList().size() == 2
+                    && call.getOperandList().get(1) instanceof SqlIntervalLiteral) {
+                SqlIntervalLiteral intervalOperand =
+                        (SqlIntervalLiteral) call.getOperandList().get(1);
+                SqlIntervalLiteral.IntervalValue interval =
+                        (SqlIntervalLiteral.IntervalValue) intervalOperand.getValue();
+                call.setOperand(1, SqlNumericLiteral.createExactNumeric(interval.toString(),
+                        SqlParserPos.ZERO));
+                writer.keyword(SqlKind.INTERVAL.name());
+                call.unparse(writer, leftPrec, rightPrec);
+                unparseSqlIntervalQualifier(writer, interval.getIntervalQualifier(),
+                        RelDataTypeSystem.DEFAULT);
+                return true;
+            }
+        }
+        return false;
+    }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DimValueMap.java

@@ -16,4 +16,6 @@ public class DimValueMap {
 
     /** dimension value for user query */
     private List<String> alias = new ArrayList<>();
+
+    private String value;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java

@@ -1,8 +1,9 @@
 package com.tencent.supersonic.headless.api.pojo;
 
-import java.util.List;
 import lombok.Data;
 
+import java.util.List;
+
 @Data
 public class ModelSchema {
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java

@@ -13,7 +13,7 @@ import java.util.List;
 @Data
 @ToString
 public class DictValueDimResp extends DictValueResp {
-    /** dimension value for result show */
+    /** dimension value */
     private String bizName;
 
     /** dimension value for user query */

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java

@@ -28,9 +28,6 @@ import java.util.stream.Stream;
 @Component
 public class FileHandlerImpl implements FileHandler {
 
-    @Value("${dict.value.max.count.page:1000}")
-    private int dictValueMaxCountPage;
-
     public static final String FILE_SPILT = File.separator;
 
     private final LocalFileConfig localFileConfig;
@@ -124,8 +121,8 @@ public class FileHandlerImpl implements FileHandler {
                                                                   DictValueReq dictValueReq) {
         PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
         String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
-        Long fileLineNum = Math.min(dictValueMaxCountPage, getFileLineNum(filePath));
+        Long fileLineNum = getFileLineNum(filePath);
-        Integer startLine = 1;
+        Integer startLine = Math.max(1, (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1);
         Integer endLine = Integer.valueOf(
                 Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + "");
         List<DictValueResp> dictValueRespList = getFileData(filePath, startLine, endLine);
@@ -135,7 +132,7 @@ public class FileHandlerImpl implements FileHandler {
         dictValueRespPageInfo.setTotal(fileLineNum);
         dictValueRespPageInfo.setList(dictValueRespList);
         dictValueRespPageInfo.setHasNextPage(endLine >= fileLineNum ? false : true);
-        dictValueRespPageInfo.setHasPreviousPage(startLine <= 0 ? false : true);
+        dictValueRespPageInfo.setHasPreviousPage(startLine <= 1 ? false : true);
         return dictValueRespPageInfo;
     }
 
@@ -149,6 +146,13 @@ public class FileHandlerImpl implements FileHandler {
         return null;
     }
 
+    /**
+     *
+     * @param filePath
+     * @param startLine 1开始
+     * @param endLine
+     * @return
+     */
     private List<DictValueResp> getFileData(String filePath, Integer startLine, Integer endLine) {
         List<DictValueResp> fileData = new ArrayList<>();
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java

@@ -178,9 +178,28 @@ public class DictTaskServiceImpl implements DictTaskService {
         PageInfo<DictValueResp> dictValueRespList =
                 fileHandler.queryDictValue(fileName, dictValueReq);
         PageInfo<DictValueDimResp> result = convert2DictValueDimRespPage(dictValueRespList);
+        fillDimMapInfo(result.getList(), dictValueReq.getItemId());
         return result;
     }
 
+    private void fillDimMapInfo(List<DictValueDimResp> list, Long dimId) {
+        DimensionResp dimResp = dimensionService.getDimension(dimId);
+        if (CollectionUtils.isEmpty(dimResp.getDimValueMaps())) {
+            return;
+        }
+        Map<String, DimValueMap> valueAndMap = dimResp.getDimValueMaps().stream()
+                .collect(Collectors.toMap(dim -> dim.getValue(), v -> v, (v1, v2) -> v2));
+        if (CollectionUtils.isEmpty(valueAndMap)) {
+            return;
+        }
+        list.stream().forEach(dictValueDimResp -> {
+            String dimValue = dictValueDimResp.getValue();
+            if (valueAndMap.containsKey(dimValue) && Objects.nonNull(valueAndMap.get(dimValue))) {
+                dictValueDimResp.setAlias(valueAndMap.get(dimValue).getAlias());
+            }
+        });
+    }
+
     private PageInfo<DictValueDimResp> convert2DictValueDimRespPage(
             PageInfo<DictValueResp> dictValueRespPage) {
         PageInfo<DictValueDimResp> result = new PageInfo<>();

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java

@@ -34,6 +34,11 @@ public class DimensionConverter {
             dimensionDO.setDefaultValues(JSONObject.toJSONString(dimensionReq.getDefaultValues()));
         }
         if (!CollectionUtils.isEmpty(dimensionReq.getDimValueMaps())) {
+            List<DimValueMap> dimValueMaps = dimensionReq.getDimValueMaps();
+            dimValueMaps.stream().forEach(dimValueMap -> {
+                dimValueMap.setTechName(dimValueMap.getValue());
+            });
+
             dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimensionReq.getDimValueMaps()));
         } else {
             dimensionDO.setDimValueMaps(JSONObject.toJSONString(new ArrayList<>()));