topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 12:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(knowledge) add keyValue parameter for queryDictValue (#1749)

Browse Source
This commit is contained in:
daikon
2024-10-08 00:10:34 +08:00
committed by GitHub
parent ff709aec60
commit ee6b8a1044
8 changed files with 186 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java
View File

@@ -13,4 +13,6 @@ public class DictValueReq extends PageBaseReq {
private Long itemId;
private TypeEnums type = TypeEnums.DIMENSION;
private String keyValue;
}

21
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java Normal file
View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.headless.api.pojo.response;
import lombok.Data;
import lombok.ToString;
import java.util.ArrayList;
import java.util.List;
/**
* @author: kanedai
* @date: 2024/9/29
*/
@Data
@ToString
public class DictValueDimResp extends DictValueResp {
/** dimension value for result show */
private String bizName;
/** dimension value for user query */
private List<String> alias = new ArrayList<>();
}

43
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java
View File

@@ -77,10 +77,51 @@ public class FileHandlerImpl implements FileHandler {
@Override
public PageInfo<DictValueResp> queryDictValue(String fileName, DictValueReq dictValueReq) {
if (StringUtils.isEmpty(dictValueReq.getKeyValue())) {
return getDictValueRespPagWithoutKey(fileName, dictValueReq);
}
return getDictValueRespPagWithKey(fileName, dictValueReq);
}
private PageInfo<DictValueResp> getDictValueRespPagWithKey(String fileName,
DictValueReq dictValueReq) {
PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize());
dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
Long fileLineNum = getFileLineNum(filePath);
Integer startLine = 1;
List<DictValueResp> dictValueRespList =
getFileData(filePath, startLine, fileLineNum.intValue());
dictValueRespList = dictValueRespList.stream()
.filter(dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(dictValueRespList)) {
dictValueRespPageInfo.setList(new ArrayList<>());
return dictValueRespPageInfo;
}
Integer startIndex =
Math.max((dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize(), 0);
Integer endIndex =
Integer.valueOf(Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(),
dictValueRespList.size()) + "");
List<DictValueResp> list = dictValueRespList.subList(startIndex, endIndex);
dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize());
dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
dictValueRespPageInfo.setTotal(dictValueRespList.size());
dictValueRespPageInfo.setList(list);
dictValueRespPageInfo.setHasNextPage(endIndex >= dictValueRespList.size() ? false : true);
dictValueRespPageInfo.setHasPreviousPage(startLine <= 0 ? false : true);
return dictValueRespPageInfo;
}
private PageInfo<DictValueResp> getDictValueRespPagWithoutKey(String fileName,
DictValueReq dictValueReq) {
PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
Long fileLineNum = getFileLineNum(filePath);
Integer startLine = (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1;
Integer startLine = 1;
Integer endLine = Integer.valueOf(
Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + "");
List<DictValueResp> dictValueRespList = getFileData(filePath, startLine, endLine);

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java
View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
import com.tencent.supersonic.headless.server.service.DictConfService;
import com.tencent.supersonic.headless.server.service.DictTaskService;
import com.tencent.supersonic.headless.server.task.DictionaryReloadTask;
@@ -112,7 +112,9 @@ public class KnowledgeController {
return taskService.deleteDictTask(taskReq, user);
}
/** dailyDictTask-手动离线更新所有字典 */
/**
* dailyDictTask-手动离线更新所有字典
*/
@PutMapping("/task/all")
public Boolean dailyDictTask() {
return taskService.dailyDictTask();
@@ -155,7 +157,7 @@ public class KnowledgeController {
* @param dictValueReq
*/
@PostMapping("/dict/data")
public PageInfo<DictValueResp> queryDictValue(@RequestBody @Valid DictValueReq dictValueReq,
public PageInfo<DictValueDimResp> queryDictValue(@RequestBody @Valid DictValueReq dictValueReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return taskService.queryDictValue(dictValueReq, user);

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
/** Manage dictionary tasks */
public interface DictTaskService {
@@ -17,7 +17,7 @@ public interface DictTaskService {
DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq, User user);
PageInfo<DictValueResp> queryDictValue(DictValueReq dictValueReq, User user);
PageInfo<DictValueDimResp> queryDictValue(DictValueReq dictValueReq, User user);
String queryDictFilePath(DictValueReq dictValueReq, User user);
}

103
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
View File

@@ -5,25 +5,36 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.chat.knowledge.file.FileHandler;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
import com.tencent.supersonic.headless.server.service.DictTaskService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.utils.DictUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Slf4j
@@ -45,14 +56,17 @@ public class DictTaskServiceImpl implements DictTaskService {
private final DictUtils dictUtils;
private final FileHandler fileHandler;
private final DictWordService dictWordService;
private final DimensionService dimensionService;
public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter,
DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService) {
DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService,
DimensionService dimensionService) {
this.dictRepository = dictRepository;
this.dictConverter = dictConverter;
this.dictUtils = dictUtils;
this.fileHandler = fileHandler;
this.dictWordService = dictWordService;
this.dimensionService = dimensionService;
}
@Override
@@ -151,13 +165,96 @@ public class DictTaskServiceImpl implements DictTaskService {
}
@Override
public PageInfo<DictValueResp> queryDictValue(DictValueReq dictValueReq, User user) {
public PageInfo<DictValueDimResp> queryDictValue(DictValueReq dictValueReq, User user) {
// todo 优化读取内存结构
// return getDictValuePageFromMemory(dictValueReq);
return getDictValuePageFromFile(dictValueReq);
}
private PageInfo<DictValueDimResp> getDictValuePageFromFile(DictValueReq dictValueReq) {
String fileName = String.format("dic_value_%d_%s_%s", dictValueReq.getModelId(),
dictValueReq.getType().name(), dictValueReq.getItemId()) + Constants.DOT
+ dictFileType;
PageInfo<DictValueResp> dictValueRespList =
fileHandler.queryDictValue(fileName, dictValueReq);
return dictValueRespList;
PageInfo<DictValueDimResp> result = convert2DictValueDimRespPage(dictValueRespList);
return result;
}
private PageInfo<DictValueDimResp> convert2DictValueDimRespPage(
PageInfo<DictValueResp> dictValueRespPage) {
PageInfo<DictValueDimResp> result = new PageInfo<>();
BeanMapper.mapper(dictValueRespPage, result);
if (CollectionUtils.isEmpty(dictValueRespPage.getList())) {
return result;
}
List<DictValueDimResp> list = getDictValueDimRespList(dictValueRespPage.getList());
result.setList(list);
return result;
}
private List<DictValueDimResp> getDictValueDimRespList(List<DictValueResp> dictValueRespList) {
List<DictValueDimResp> list =
dictValueRespList.stream().map(dictValue -> convert2DictValueInternal(dictValue))
.collect(Collectors.toList());
return list;
}
private List<DictValueDimResp> getDictValueDimRespList(List<DictWord> dictWords, Long dimId) {
DimensionResp dimResp = dimensionService.getDimension(dimId);
List<DictValueDimResp> list =
dictWords.stream().map(dictWord -> convert2DictValueInternal(dictWord, dimResp))
.collect(Collectors.toList());
return list;
}
private DictValueDimResp convert2DictValueInternal(DictWord dictWord, DimensionResp dimResp) {
DictValueDimResp dictValueDimResp = new DictValueDimResp();
BeanMapper.mapper(dictWord, dictValueDimResp);
if (Objects.nonNull(dimResp.getDimValueMaps())) {
Map<String, DimValueMap> techAndAliasMap = dimResp.getDimValueMaps().stream().collect(
Collectors.toMap(dimValue -> dimValue.getTechName(), v -> v, (v1, v2) -> v2));
if (techAndAliasMap.containsKey(dictWord.getWord())) {
DimValueMap dimValueMap = techAndAliasMap.get(dictWord.getWord());
BeanMapper.mapper(dimValueMap, dictValueDimResp);
}
}
return dictValueDimResp;
}
private DictValueDimResp convert2DictValueInternal(DictValueResp dictValue) {
DictValueDimResp dictValueDimResp = new DictValueDimResp();
BeanMapper.mapper(dictValue, dictValueDimResp);
return dictValueDimResp;
}
private PageInfo<DictValueDimResp> getDictValuePageFromMemory(DictValueReq dictValueReq) {
PageInfo<DictValueDimResp> dictValueRespPageInfo = new PageInfo<>();
Set<Long> dimSet = new HashSet<>();
dimSet.add(dictValueReq.getItemId());
List<DictWord> dimDictWords = dictWordService.getDimDictWords(dimSet);
if (CollectionUtils.isEmpty(dimDictWords)) {
return dictValueRespPageInfo;
}
if (StringUtils.isNotEmpty(dictValueReq.getKeyValue())) {
dimDictWords = dimDictWords.stream()
.filter(dimValue -> dimValue.getWord().contains(dictValueReq.getKeyValue()))
.collect(Collectors.toList());
}
Integer pageSize = dictValueReq.getPageSize();
Integer current = dictValueReq.getCurrent();
dictValueRespPageInfo.setTotal(dimDictWords.size());
dictValueRespPageInfo.setPageSize(pageSize);
dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
// 分页
int startIndex = (current - 1) * pageSize;
int endIndex = Math.min(startIndex + pageSize, dimDictWords.size());
List<DictWord> data = dimDictWords.subList(startIndex, endIndex);
dictValueRespPageInfo.setList(getDictValueDimRespList(data, dictValueReq.getItemId()));
return dictValueRespPageInfo;
}
@Override

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java
View File

@@ -15,6 +15,7 @@ import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -63,6 +64,15 @@ public class DictWordService {
return words;
}
public List<DictWord> getDimDictWords(Set<Long> dimIds) {
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
List<SchemaElement> requiredDims = semanticSchema.getDimensionValues().stream()
.filter(dim -> dimIds.contains(dim.getId())).collect(Collectors.toList());
List<DictWord> words = new ArrayList<>();
addWordsByType(DictWordType.VALUE, requiredDims, words);
return words;
}
private void addWordsByType(DictWordType value, List<SchemaElement> metas,
List<DictWord> natures) {
metas = distinct(metas);

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
View File

@@ -320,6 +320,10 @@ public class DictUtils {
long limit =
(Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount
: dictItemResp.getConfig().getLimit();
if (limit <= 0) {
limit = Integer.MAX_VALUE;
}
String sql = String.format(sqlPattern, bizName, where, bizName, limit);
Set<Long> modelIds = new HashSet<>();
modelIds.add(dictItemResp.getModelId());