topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 20:25:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Remove redundant datasetName in e MetaDiscovery interface (#993)

Browse Source
This commit is contained in:
LXW
2024-05-14 17:14:43 +08:00
committed by GitHub
parent e0d45a77ce
commit 5e3cc89da6
4 changed files with 26 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Term.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.api.pojo; package com.tencent.supersonic.headless.api.pojo;
import com.google.common.collect.Lists;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
@@ -10,6 +11,6 @@ public class Term {
private String description; private String description;
private List<String> similarTerms; private List<String> similarTerms = Lists.newArrayList();
} }

5
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MapInfoResp.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.Term; import com.tencent.supersonic.headless.api.pojo.Term;
import lombok.Data; import lombok.Data;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -11,8 +12,8 @@ public class MapInfoResp {
private String queryText; private String queryText;
private Map<String, DataSetMapInfo> dataSetMapInfo; private Map<String, DataSetMapInfo> dataSetMapInfo = new HashMap<>();
private Map<String, List<Term>> terms; private Map<String, List<Term>> terms = new HashMap<>();
} }

9
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ChatQueryApiController.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.rest.api;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq; import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq; import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.server.service.ChatQueryService; import com.tencent.supersonic.headless.server.service.ChatQueryService;
import com.tencent.supersonic.headless.server.service.SearchService; import com.tencent.supersonic.headless.server.service.SearchService;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -32,6 +33,14 @@ public class ChatQueryApiController {
return searchService.search(queryReq); return searchService.search(queryReq);
} }
@PostMapping("/chat/map")
public MapResp map(@RequestBody QueryReq queryReq,
HttpServletRequest request,
HttpServletResponse response) {
queryReq.setUser(UserHolder.findUser(request, response));
return chatQueryService.performMapping(queryReq);
}
@PostMapping("/chat/parse") @PostMapping("/chat/parse")
public Object parse(@RequestBody QueryReq queryReq, public Object parse(@RequestBody QueryReq queryReq,
HttpServletRequest request, HttpServletRequest request,

20
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.server.service.impl; package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
@@ -21,7 +22,6 @@ import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService; import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
import com.tencent.supersonic.headless.server.service.TermService; import com.tencent.supersonic.headless.server.service.TermService;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -61,16 +61,16 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet()); Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
queryReq.setDataSetIds(dataSetIds); queryReq.setDataSetIds(dataSetIds);
MapResp mapResp = chatQueryService.performMapping(queryReq); MapResp mapResp = chatQueryService.performMapping(queryReq);
return convert(mapResp, queryMapReq.getTopN()); dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet());
return convert(mapResp, queryMapReq.getTopN(), dataSetIds);
} }
private MapInfoResp convert(MapResp mapResp, Integer topN) { private MapInfoResp convert(MapResp mapResp, Integer topN, Set<Long> dataSetIds) {
MapInfoResp mapInfoResp = new MapInfoResp(); MapInfoResp mapInfoResp = new MapInfoResp();
if (Objects.isNull(mapResp)) { if (Objects.isNull(mapResp)) {
return mapInfoResp; return mapInfoResp;
} }
BeanUtils.copyProperties(mapResp, mapInfoResp); BeanUtils.copyProperties(mapResp, mapInfoResp);
Set<Long> dataSetIds = mapResp.getMapInfo().getDataSetElementMatches().keySet();
MetaFilter metaFilter = new MetaFilter(); MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(new ArrayList<>(dataSetIds)); metaFilter.setIds(new ArrayList<>(dataSetIds));
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter); List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
@@ -92,9 +92,12 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
if (dataSetResp == null) { if (dataSetResp == null) {
continue; continue;
} }
if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) {
continue;
}
DataSetMapInfo dataSetMapInfo = new DataSetMapInfo(); DataSetMapInfo dataSetMapInfo = new DataSetMapInfo();
dataSetMapInfo.setMapFields(mapFields.get(dataSetId)); dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setTopFields(topFields.get(dataSetId)); dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setName(dataSetResp.getName()); dataSetMapInfo.setName(dataSetResp.getName());
dataSetMapInfo.setDescription(dataSetResp.getDescription()); dataSetMapInfo.setDescription(dataSetResp.getDescription());
map.put(dataSetMapInfo.getName(), dataSetMapInfo); map.put(dataSetMapInfo.getName(), dataSetMapInfo);
@@ -123,10 +126,11 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) { for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
Long dataSetId = entry.getKey(); Long dataSetId = entry.getKey();
List<SchemaElementMatch> values = entry.getValue(); List<SchemaElementMatch> values = entry.getValue();
String dataSetName = dataSetMap.get(dataSetId).getName(); DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (StringUtils.isBlank(dataSetName) || CollectionUtils.isEmpty(values)) { if (dataSetResp == null || CollectionUtils.isEmpty(values)) {
continue; continue;
} }
String dataSetName = dataSetResp.getName();
//topN dimensions //topN dimensions
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId) Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())