(improvement)(knowledge) add keyValue parameter for queryDictValue (#1749)

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java

@@ -13,4 +13,6 @@ public class DictValueReq extends PageBaseReq {
     private Long itemId;
 
     private TypeEnums type = TypeEnums.DIMENSION;
+
+    private String keyValue;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java

@@ -0,0 +1,21 @@
+package com.tencent.supersonic.headless.api.pojo.response;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author: kanedai
+ * @date: 2024/9/29
+ */
+@Data
+@ToString
+public class DictValueDimResp extends DictValueResp {
+    /** dimension value for result show */
+    private String bizName;
+
+    /** dimension value for user query */
+    private List<String> alias = new ArrayList<>();
+}

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java

@@ -77,10 +77,51 @@ public class FileHandlerImpl implements FileHandler {
 
     @Override
     public PageInfo<DictValueResp> queryDictValue(String fileName, DictValueReq dictValueReq) {
+        if (StringUtils.isEmpty(dictValueReq.getKeyValue())) {
+            return getDictValueRespPagWithoutKey(fileName, dictValueReq);
+        }
+        return getDictValueRespPagWithKey(fileName, dictValueReq);
+    }
+
+    private PageInfo<DictValueResp> getDictValueRespPagWithKey(String fileName,
+            DictValueReq dictValueReq) {
+        PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
+        dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize());
+        dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
+        String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
+        Long fileLineNum = getFileLineNum(filePath);
+        Integer startLine = 1;
+        List<DictValueResp> dictValueRespList =
+                getFileData(filePath, startLine, fileLineNum.intValue());
+        dictValueRespList = dictValueRespList.stream()
+                .filter(dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue()))
+                .collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(dictValueRespList)) {
+            dictValueRespPageInfo.setList(new ArrayList<>());
+            return dictValueRespPageInfo;
+        }
+
+        Integer startIndex =
+                Math.max((dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize(), 0);
+        Integer endIndex =
+                Integer.valueOf(Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(),
+                        dictValueRespList.size()) + "");
+        List<DictValueResp> list = dictValueRespList.subList(startIndex, endIndex);
+        dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize());
+        dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
+        dictValueRespPageInfo.setTotal(dictValueRespList.size());
+        dictValueRespPageInfo.setList(list);
+        dictValueRespPageInfo.setHasNextPage(endIndex >= dictValueRespList.size() ? false : true);
+        dictValueRespPageInfo.setHasPreviousPage(startLine <= 0 ? false : true);
+        return dictValueRespPageInfo;
+    }
+
+    private PageInfo<DictValueResp> getDictValueRespPagWithoutKey(String fileName,
+            DictValueReq dictValueReq) {
         PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
         String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
         Long fileLineNum = getFileLineNum(filePath);
-        Integer startLine = (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1;
+        Integer startLine = 1;
         Integer endLine = Integer.valueOf(
                 Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + "");
         List<DictValueResp> dictValueRespList = getFileData(filePath, startLine, endLine);

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java

@@ -15,7 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
 import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
-import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
+import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
 import com.tencent.supersonic.headless.server.service.DictConfService;
 import com.tencent.supersonic.headless.server.service.DictTaskService;
 import com.tencent.supersonic.headless.server.task.DictionaryReloadTask;
@@ -112,7 +112,9 @@ public class KnowledgeController {
         return taskService.deleteDictTask(taskReq, user);
     }
 
-    /** dailyDictTask-手动离线更新所有字典 */
+    /**
+     * dailyDictTask-手动离线更新所有字典
+     */
     @PutMapping("/task/all")
     public Boolean dailyDictTask() {
         return taskService.dailyDictTask();
@@ -155,7 +157,7 @@ public class KnowledgeController {
      * @param dictValueReq
      */
     @PostMapping("/dict/data")
-    public PageInfo<DictValueResp> queryDictValue(@RequestBody @Valid DictValueReq dictValueReq,
+    public PageInfo<DictValueDimResp> queryDictValue(@RequestBody @Valid DictValueReq dictValueReq,
             HttpServletRequest request, HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
         return taskService.queryDictValue(dictValueReq, user);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java

@@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
 import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
-import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
+import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
 
 /** Manage dictionary tasks */
 public interface DictTaskService {
@@ -17,7 +17,7 @@ public interface DictTaskService {
 
     DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq, User user);
 
-    PageInfo<DictValueResp> queryDictValue(DictValueReq dictValueReq, User user);
+    PageInfo<DictValueDimResp> queryDictValue(DictValueReq dictValueReq, User user);
 
     String queryDictFilePath(DictValueReq dictValueReq, User user);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java

@@ -5,25 +5,36 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.util.BeanMapper;
+import com.tencent.supersonic.headless.api.pojo.DimValueMap;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
 import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
+import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
 import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
+import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
+import com.tencent.supersonic.headless.chat.knowledge.DictWord;
 import com.tencent.supersonic.headless.chat.knowledge.file.FileHandler;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
 import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
 import com.tencent.supersonic.headless.server.service.DictTaskService;
+import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.utils.DictUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 @Service
 @Slf4j
@@ -45,14 +56,17 @@ public class DictTaskServiceImpl implements DictTaskService {
     private final DictUtils dictUtils;
     private final FileHandler fileHandler;
     private final DictWordService dictWordService;
+    private final DimensionService dimensionService;
 
     public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter,
-            DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService) {
+            DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService,
+            DimensionService dimensionService) {
         this.dictRepository = dictRepository;
         this.dictConverter = dictConverter;
         this.dictUtils = dictUtils;
         this.fileHandler = fileHandler;
         this.dictWordService = dictWordService;
+        this.dimensionService = dimensionService;
     }
 
     @Override
@@ -151,13 +165,96 @@ public class DictTaskServiceImpl implements DictTaskService {
     }
 
     @Override
-    public PageInfo<DictValueResp> queryDictValue(DictValueReq dictValueReq, User user) {
+    public PageInfo<DictValueDimResp> queryDictValue(DictValueReq dictValueReq, User user) {
+        // todo 优化读取内存结构
+        // return getDictValuePageFromMemory(dictValueReq);
+        return getDictValuePageFromFile(dictValueReq);
+    }
+
+    private PageInfo<DictValueDimResp> getDictValuePageFromFile(DictValueReq dictValueReq) {
         String fileName = String.format("dic_value_%d_%s_%s", dictValueReq.getModelId(),
                 dictValueReq.getType().name(), dictValueReq.getItemId()) + Constants.DOT
                 + dictFileType;
         PageInfo<DictValueResp> dictValueRespList =
                 fileHandler.queryDictValue(fileName, dictValueReq);
-        return dictValueRespList;
+        PageInfo<DictValueDimResp> result = convert2DictValueDimRespPage(dictValueRespList);
+        return result;
+    }
+
+    private PageInfo<DictValueDimResp> convert2DictValueDimRespPage(
+            PageInfo<DictValueResp> dictValueRespPage) {
+        PageInfo<DictValueDimResp> result = new PageInfo<>();
+        BeanMapper.mapper(dictValueRespPage, result);
+        if (CollectionUtils.isEmpty(dictValueRespPage.getList())) {
+            return result;
+        }
+
+        List<DictValueDimResp> list = getDictValueDimRespList(dictValueRespPage.getList());
+        result.setList(list);
+        return result;
+    }
+
+    private List<DictValueDimResp> getDictValueDimRespList(List<DictValueResp> dictValueRespList) {
+        List<DictValueDimResp> list =
+                dictValueRespList.stream().map(dictValue -> convert2DictValueInternal(dictValue))
+                        .collect(Collectors.toList());
+        return list;
+    }
+
+    private List<DictValueDimResp> getDictValueDimRespList(List<DictWord> dictWords, Long dimId) {
+        DimensionResp dimResp = dimensionService.getDimension(dimId);
+        List<DictValueDimResp> list =
+                dictWords.stream().map(dictWord -> convert2DictValueInternal(dictWord, dimResp))
+                        .collect(Collectors.toList());
+        return list;
+    }
+
+    private DictValueDimResp convert2DictValueInternal(DictWord dictWord, DimensionResp dimResp) {
+        DictValueDimResp dictValueDimResp = new DictValueDimResp();
+        BeanMapper.mapper(dictWord, dictValueDimResp);
+        if (Objects.nonNull(dimResp.getDimValueMaps())) {
+            Map<String, DimValueMap> techAndAliasMap = dimResp.getDimValueMaps().stream().collect(
+                    Collectors.toMap(dimValue -> dimValue.getTechName(), v -> v, (v1, v2) -> v2));
+            if (techAndAliasMap.containsKey(dictWord.getWord())) {
+                DimValueMap dimValueMap = techAndAliasMap.get(dictWord.getWord());
+                BeanMapper.mapper(dimValueMap, dictValueDimResp);
+            }
+        }
+        return dictValueDimResp;
+    }
+
+    private DictValueDimResp convert2DictValueInternal(DictValueResp dictValue) {
+        DictValueDimResp dictValueDimResp = new DictValueDimResp();
+        BeanMapper.mapper(dictValue, dictValueDimResp);
+        return dictValueDimResp;
+    }
+
+    private PageInfo<DictValueDimResp> getDictValuePageFromMemory(DictValueReq dictValueReq) {
+        PageInfo<DictValueDimResp> dictValueRespPageInfo = new PageInfo<>();
+        Set<Long> dimSet = new HashSet<>();
+        dimSet.add(dictValueReq.getItemId());
+        List<DictWord> dimDictWords = dictWordService.getDimDictWords(dimSet);
+        if (CollectionUtils.isEmpty(dimDictWords)) {
+            return dictValueRespPageInfo;
+        }
+        if (StringUtils.isNotEmpty(dictValueReq.getKeyValue())) {
+            dimDictWords = dimDictWords.stream()
+                    .filter(dimValue -> dimValue.getWord().contains(dictValueReq.getKeyValue()))
+                    .collect(Collectors.toList());
+        }
+
+        Integer pageSize = dictValueReq.getPageSize();
+        Integer current = dictValueReq.getCurrent();
+        dictValueRespPageInfo.setTotal(dimDictWords.size());
+        dictValueRespPageInfo.setPageSize(pageSize);
+        dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
+
+        // 分页
+        int startIndex = (current - 1) * pageSize;
+        int endIndex = Math.min(startIndex + pageSize, dimDictWords.size());
+        List<DictWord> data = dimDictWords.subList(startIndex, endIndex);
+        dictValueRespPageInfo.setList(getDictValueDimRespList(data, dictValueReq.getItemId()));
+        return dictValueRespPageInfo;
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java

@@ -15,6 +15,7 @@ import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -63,6 +64,15 @@ public class DictWordService {
         return words;
     }
 
+    public List<DictWord> getDimDictWords(Set<Long> dimIds) {
+        SemanticSchema semanticSchema = schemaService.getSemanticSchema();
+        List<SchemaElement> requiredDims = semanticSchema.getDimensionValues().stream()
+                .filter(dim -> dimIds.contains(dim.getId())).collect(Collectors.toList());
+        List<DictWord> words = new ArrayList<>();
+        addWordsByType(DictWordType.VALUE, requiredDims, words);
+        return words;
+    }
+
     private void addWordsByType(DictWordType value, List<SchemaElement> metas,
             List<DictWord> natures) {
         metas = distinct(metas);

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -320,6 +320,10 @@ public class DictUtils {
         long limit =
                 (Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount
                         : dictItemResp.getConfig().getLimit();
+        if (limit <= 0) {
+            limit = Integer.MAX_VALUE;
+        }
+
         String sql = String.format(sqlPattern, bizName, where, bizName, limit);
         Set<Long> modelIds = new HashSet<>();
         modelIds.add(dictItemResp.getModelId());