topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Optimize return structure of MetaDiscovery interface (#986)

Browse Source
This commit is contained in:
LXW
2024-05-13 14:29:30 +08:00
committed by GitHub
parent 0e28d6cbcc
commit 1f4039a453
5 changed files with 64 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetMapInfo.java Normal file
View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import lombok.Data;
import java.util.List;
@Data
public class DataSetMapInfo {
private String name;
private String description;
private List<SchemaElementMatch> mapFields;
private List<SchemaElementMatch> topFields;
}

5
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MapInfoResp.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.Term;
import lombok.Data;
@@ -12,9 +11,7 @@ public class MapInfoResp {
private String queryText;
private Map<String, List<SchemaElementMatch>> mapFields;
private Map<String, List<SchemaElementMatch>> topFields;
private Map<String, DataSetMapInfo> dataSetMapInfo;
private Map<String, List<Term>> terms;

51
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java
View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.Term;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
@@ -73,36 +74,56 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(new ArrayList<>(dataSetIds));
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
Map<Long, String> dataSetMap = dataSetList.stream()
.collect(Collectors.toMap(DataSetResp::getId, DataSetResp::getName));
mapInfoResp.setMapFields(getMapFields(mapResp.getMapInfo(), dataSetMap));
mapInfoResp.setTopFields(getTopFields(topN, mapResp.getMapInfo(), dataSetMap));
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
mapInfoResp.setTerms(getTerms(dataSetList, dataSetMap));
return mapInfoResp;
}
private Map<String, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
Map<Long, String> dataSetMap) {
Map<String, List<SchemaElementMatch>> result = new HashMap<>();
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap,
Integer topN) {
Map<String, DataSetMapInfo> map = new HashMap<>();
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) {
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null) {
continue;
}
DataSetMapInfo dataSetMapInfo = new DataSetMapInfo();
dataSetMapInfo.setMapFields(mapFields.get(dataSetId));
dataSetMapInfo.setTopFields(topFields.get(dataSetId));
dataSetMapInfo.setName(dataSetResp.getName());
dataSetMapInfo.setDescription(dataSetResp.getDescription());
map.put(dataSetMapInfo.getName(), dataSetMapInfo);
}
return map;
}
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values = entry.getValue();
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
result.put(dataSetMap.get(entry.getKey()), values);
result.put(entry.getKey(), values);
}
}
return result;
}
private Map<String, List<SchemaElementMatch>> getTopFields(Integer topN,
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN,
SchemaMapInfo mapInfo,
Map<Long, String> dataSetMap) {
Map<String, List<SchemaElementMatch>> result = new HashMap<>();
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
Long dataSetId = entry.getKey();
List<SchemaElementMatch> values = entry.getValue();
String dataSetName = dataSetMap.get(dataSetId);
String dataSetName = dataSetMap.get(dataSetId).getName();
if (StringUtils.isBlank(dataSetName) || CollectionUtils.isEmpty(values)) {
continue;
}
@@ -120,19 +141,19 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
.limit(topN).map(mergeFunction()).collect(Collectors.toSet());
dimensions.addAll(metrics);
result.put(dataSetName, new ArrayList<>(dimensions));
result.put(dataSetId, new ArrayList<>(dimensions));
}
return result;
}
private Map<String, List<Term>> getTerms(List<DataSetResp> dataSets, Map<Long, String> dataSetNameMap) {
private Map<String, List<Term>> getTerms(List<DataSetResp> dataSets, Map<Long, DataSetResp> dataSetNameMap) {
Set<Long> domainIds = dataSets.stream().map(DataSetResp::getDomainId).collect(Collectors.toSet());
Map<Long, Long> dataSetDomainIdMap = dataSets.stream()
.collect(Collectors.toMap(DataSetResp::getId, DataSetResp::getDomainId));
Map<Long, List<Term>> domainTermSetMap = termService.getTermSets(domainIds);
Map<String, List<Term>> dataSetTermSetMap = new HashMap<>();
for (DataSetResp dataSet : dataSets) {
dataSetTermSetMap.put(dataSetNameMap.get(dataSet.getId()),
dataSetTermSetMap.put(dataSetNameMap.get(dataSet.getId()).getName(),
domainTermSetMap.get(dataSetDomainIdMap.get(dataSet.getId())));
}
return dataSetTermSetMap;

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -37,6 +37,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
@@ -292,11 +293,12 @@ public class MetricServiceImpl implements MetricService {
queryMapReq.setUser(user);
queryMapReq.setMapModeEnum(MapModeEnum.LOOSE);
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
Map<String, List<SchemaElementMatch>> mapFields = mapMeta.getMapFields();
if (CollectionUtils.isEmpty(mapFields)) {
Map<String, DataSetMapInfo> dataSetMapInfo = mapMeta.getDataSetMapInfo();
if (CollectionUtils.isEmpty(dataSetMapInfo)) {
return metricRespPageInfo;
}
Map<Long, Double> result = mapFields.values().stream()
Map<Long, Double> result = dataSetMapInfo.values().stream()
.map(DataSetMapInfo::getMapFields)
.flatMap(Collection::stream).filter(schemaElementMatch ->
SchemaElementType.METRIC.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toMap(schemaElementMatch ->

7
launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java
View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
import org.junit.Assert;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -25,9 +26,9 @@ public class MetaDiscoveryTest extends BaseTest {
queryMapReq.setDataSetNames(Collections.singletonList("超音数"));
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
Assert.assertNotNull(mapMeta);
Assert.assertNotEquals(0, mapMeta.getMapFields());
Assert.assertNotEquals(0, mapMeta.getTopFields());
Assertions.assertNotNull(mapMeta);
Assertions.assertNotEquals(0, mapMeta.getDataSetMapInfo().get("超音数").getMapFields());
Assertions.assertNotEquals(0, mapMeta.getDataSetMapInfo().get("超音数").getTopFields());
}
@Test