topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)Remove redundant SemanticService and SearchService.

Browse Source
This commit is contained in:
jerryjzhang
2024-06-22 11:15:59 +08:00
parent 69bb64cdbf
commit 6466999dbc
20 changed files with 453 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
import org.springframework.util.CollectionUtils;
import java.util.Comparator;
@@ -40,7 +40,7 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
}
private List<SchemaElement> getDimensions(Long metricId, Long dataSetId) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(dataSetId);
List<Long> drillDownDimensions = Lists.newArrayList();
Set<SchemaElement> metricElements = dataSetSchema.getMetrics();

4
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/EntityInfoProcessor.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.chat.query.QueryManager;
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
import org.springframework.util.CollectionUtils;
import java.util.List;
@@ -30,7 +30,7 @@ public class EntityInfoProcessor implements ParseResultProcessor {
return;
}
//1. set entity info
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(parseInfo.getDataSetId());
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, chatParseContext.getUser());
if (QueryManager.isTagQuery(queryMode)

4
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatServiceImpl.java
View File

@@ -30,7 +30,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import com.tencent.supersonic.headless.server.service.ChatQueryService;
import com.tencent.supersonic.headless.server.service.SearchService;
import com.tencent.supersonic.headless.server.service.RetrieveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -47,7 +47,7 @@ public class ChatServiceImpl implements ChatService {
@Autowired
private ChatQueryService chatQueryService;
@Autowired
private SearchService searchService;
private RetrieveService searchService;
@Autowired
private SimilarQueryManager similarQueryManager;
private List<ChatParser> chatParsers = ComponentFactory.getChatParsers();

14
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java
View File

@@ -33,10 +33,9 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -56,18 +55,17 @@ public class ConfigServiceImpl implements ConfigService {
private final ChatConfigHelper chatConfigHelper;
private final DimensionService dimensionService;
private final MetricService metricService;
@Autowired
private SemanticService semanticService;
private final SemanticLayerService semanticLayerService;
public ConfigServiceImpl(ChatConfigRepository chatConfigRepository,
ChatConfigHelper chatConfigHelper,
DimensionService dimensionService,
MetricService metricService) {
ChatConfigHelper chatConfigHelper, DimensionService dimensionService,
MetricService metricService, SemanticLayerService semanticLayerService) {
this.chatConfigRepository = chatConfigRepository;
this.chatConfigHelper = chatConfigHelper;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.semanticLayerService = semanticLayerService;
}
@Override
@@ -208,7 +206,7 @@ public class ConfigServiceImpl implements ConfigService {
}
BeanUtils.copyProperties(chatConfigResp, chatConfigRich);
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(modelId);
DataSetSchema dataSetSchema = semanticLayerService.getDataSetSchema(modelId);
if (dataSetSchema == null) {
return chatConfigRich;
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java
View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.chat.ChatContext;
import com.tencent.supersonic.headless.chat.QueryContext;
import com.tencent.supersonic.headless.chat.query.SemanticQuery;
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -83,7 +83,7 @@ public class ParseInfoProcessor implements ResultProcessor {
log.error("set dimensionFilter error :", e);
}
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticService.class).getSemanticSchema();
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticLayerService.class).getSemanticSchema();
if (Objects.isNull(semanticSchema)) {
return;
}
@@ -195,7 +195,7 @@ public class ParseInfoProcessor implements ResultProcessor {
}
protected Map<String, SchemaElement> getNameToElement(Long dataSetId) {
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticService.class).getSemanticSchema();
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticLayerService.class).getSemanticSchema();
List<SchemaElement> dimensions = semanticSchema.getDimensions(dataSetId);
List<SchemaElement> metrics = semanticSchema.getMetrics(dataSetId);

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ChatQueryApiController.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.server.service.ChatQueryService;
import com.tencent.supersonic.headless.server.service.SearchService;
import com.tencent.supersonic.headless.server.service.RetrieveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
@@ -22,8 +22,9 @@ public class ChatQueryApiController {
@Autowired
private ChatQueryService chatQueryService;
@Autowired
private SearchService searchService;
private RetrieveService searchService;
@PostMapping("/chat/search")
public Object search(@RequestBody QueryReq queryReq,

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/MetaDiscoveryApiController.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.rest.api;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
import com.tencent.supersonic.headless.server.service.RetrieveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
@@ -20,14 +20,14 @@ import javax.servlet.http.HttpServletResponse;
public class MetaDiscoveryApiController {
@Autowired
private MetaDiscoveryService metaDiscovery;
private RetrieveService metaDiscovery;
@PostMapping("map")
public Object map(@RequestBody QueryMapReq queryMapReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
queryMapReq.setUser(user);
return metaDiscovery.getMapMeta(queryMapReq);
return metaDiscovery.map(queryMapReq);
}
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaDiscoveryService.java
View File

@@ -1,10 +0,0 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
public interface MetaDiscoveryService {
MapInfoResp getMapMeta(QueryMapReq queryMapReq);
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SearchService.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/RetrieveService.java
View File

@@ -1,16 +1,16 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import java.util.List;
public interface RetrieveService {
/**
* search service
*/
public interface SearchService {
MapInfoResp map(QueryMapReq queryMapReq);
List<SearchResult> search(QueryReq queryCtx);
}
}

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemanticLayerService.java
View File

@@ -2,6 +2,8 @@ package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
@@ -13,19 +15,24 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.util.List;
/**
* This interface abstracts functionalities provided by a semantic layer.
*/
public interface SemanticLayerService {
SemanticSchema getSemanticSchema();
DataSetSchema getDataSetSchema(Long id);
SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;
List<DataSetSchema> getDataSetSchema();
//List<SemanticQueryResp> queryByReqs(List<SemanticQueryReq> queryReqs, User user) throws Exception;
SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user);
}

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java
View File

@@ -93,14 +93,12 @@ import java.util.stream.Collectors;
public class ChatQueryServiceImpl implements ChatQueryService {
@Autowired
private SemanticService semanticService;
private SemanticLayerService semanticLayerService;
@Autowired
private ChatContextService chatContextService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private SemanticLayerService queryService;
@Autowired
private DataSetService dataSetService;
@Autowired
private WorkflowService workflowService;
@@ -137,7 +135,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
public QueryContext buildQueryContext(QueryReq queryReq) {
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds();
QueryContext queryCtx = QueryContext.builder()
.queryFilters(queryReq.getQueryFilters())
@@ -184,7 +182,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
private QueryResult doExecution(SemanticQueryReq semanticQueryReq,
SemanticParseInfo parseInfo, User user) throws Exception {
SemanticQueryResp queryResp = queryService.queryByReq(semanticQueryReq, user);
SemanticQueryResp queryResp = semanticLayerService.queryByReq(semanticQueryReq, user);
QueryResult queryResult = new QueryResult();
if (queryResp != null) {
queryResult.setQueryAuthorization(queryResp.getQueryAuthorization());
@@ -214,7 +212,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Override
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws Exception {
SemanticParseInfo parseInfo = getSemanticParseInfo(queryData);
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
semanticQuery.setParseInfo(parseInfo);
@@ -239,7 +237,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
ExplainSqlReq<Object> explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq)
.queryTypeEnum(QueryMethod.SQL).build();
ExplainResp explain = queryService.explain(explainSqlReq, user);
ExplainResp explain = semanticLayerService.explain(explainSqlReq, user);
if (StringUtils.isNotBlank(explain.getSql())) {
parseInfo.getSqlInfo().setQuerySQL(explain.getSql());
}
@@ -258,7 +256,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
QueryResult queryResult = doExecution(semanticQueryReq, semanticQuery.getParseInfo(), user);
queryResult.setChatContext(semanticQuery.getParseInfo());
DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(parseInfo.getDataSetId());
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
queryResult.setEntityInfo(entityInfo);
return queryResult;
@@ -324,7 +322,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Override
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dataSetSchema =
semanticService.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
return semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
@@ -520,7 +518,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Override
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID());
Set<Long> detectDataSetIds = new HashSet<>();
@@ -589,7 +587,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
List<String> groups = new ArrayList<>();
groups.add(dimensionValueReq.getBizName());
queryStructReq.setGroups(groups);
return queryService.queryByReq(queryStructReq, user);
return semanticLayerService.queryByReq(queryStructReq, user);
}
public void correct(QuerySqlReq querySqlReq, User user) {
@@ -605,7 +603,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
private SemanticParseInfo correctSqlReq(QuerySqlReq querySqlReq, User user) {
QueryContext queryCtx = new QueryContext();
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
queryCtx.setSemanticSchema(semanticSchema);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
SqlInfo sqlInfo = new SqlInfo();

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.chat.knowledge.builder.WordBuilderFactory;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -24,7 +24,7 @@ import java.util.stream.Collectors;
public class DictWordService {
@Autowired
private SchemaService schemaService;
private SemanticLayerService semanticLayerService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@@ -49,7 +49,7 @@ public class DictWordService {
}
public List<DictWord> getAllDictWords() {
SemanticSchema semanticSchema = new SemanticSchema(schemaService.getDataSetSchema());
SemanticSchema semanticSchema = new SemanticSchema(semanticLayerService.getDataSetSchema());
List<DictWord> words = new ArrayList<>();

192
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java
View File

@@ -1,192 +0,0 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.ChatQueryService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
@Autowired
private DataSetService dataSetService;
@Autowired
private ChatQueryService chatQueryService;
@Autowired
private SemanticService semanticService;
@Override
public MapInfoResp getMapMeta(QueryMapReq queryMapReq) {
QueryReq queryReq = new QueryReq();
BeanUtils.copyProperties(queryMapReq, queryReq);
List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
queryReq.setDataSetIds(dataSetIds);
MapResp mapResp = chatQueryService.performMapping(queryReq);
dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet());
return convert(mapResp, queryMapReq.getTopN(), dataSetIds);
}
private MapInfoResp convert(MapResp mapResp, Integer topN, Set<Long> dataSetIds) {
MapInfoResp mapInfoResp = new MapInfoResp();
if (Objects.isNull(mapResp)) {
return mapInfoResp;
}
BeanUtils.copyProperties(mapResp, mapInfoResp);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(new ArrayList<>(dataSetIds));
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap));
return mapInfoResp;
}
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap,
Integer topN) {
Map<String, DataSetMapInfo> map = new HashMap<>();
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) {
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null) {
continue;
}
if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) {
continue;
}
DataSetMapInfo dataSetMapInfo = new DataSetMapInfo();
dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setName(dataSetResp.getName());
dataSetMapInfo.setDescription(dataSetResp.getDescription());
map.put(dataSetMapInfo.getName(), dataSetMapInfo);
}
return map;
}
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values = entry.getValue().stream()
.filter(schemaElementMatch ->
!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
result.put(entry.getKey(), values);
}
}
return result;
}
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN,
SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
if (0 == topN) {
return result;
}
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
Long dataSetId = entry.getKey();
List<SchemaElementMatch> values = entry.getValue();
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null || CollectionUtils.isEmpty(values)) {
continue;
}
String dataSetName = dataSetResp.getName();
//topN dimensions
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
dimensions.add(timeDimensionMatch);
//topN metrics
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN).map(mergeFunction()).collect(Collectors.toSet());
dimensions.addAll(metrics);
result.put(dataSetId, new ArrayList<>(dimensions));
}
return result;
}
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetNameMap) {
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
Map<Long, List<SchemaElementMatch>> dataSetElementMatches = mapInfo.getDataSetElementMatches();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey());
if (dataSetResp == null) {
continue;
}
List<SchemaElementMatch> terms = entry.getValue().stream().filter(schemaElementMatch
-> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
termMap.put(dataSetResp.getName(), terms);
}
return termMap;
}
/***
* get time dimension SchemaElementMatch
* @param dataSetId
* @param dataSetName
* @return
*/
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName)
.type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
.detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName())
.similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
return timeDimensionMatch;
}
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
.detectWord(schemaElement.getName()).build();
}
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -61,7 +61,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
import com.tencent.supersonic.headless.server.service.RetrieveService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
@@ -110,7 +110,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
private TagMetaService tagMetaService;
private MetaDiscoveryService metaDiscoveryService;
private RetrieveService metaDiscoveryService;
public MetricServiceImpl(MetricRepository metricRepository,
ModelService modelService,
@@ -120,7 +120,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService,
TagMetaService tagMetaService,
@Lazy MetaDiscoveryService metaDiscoveryService) {
@Lazy RetrieveService metaDiscoveryService) {
this.metricRepository = metricRepository;
this.modelService = modelService;
this.aliasGenerateHelper = aliasGenerateHelper;
@@ -298,7 +298,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
queryMapReq.setQueryText(pageMetricReq.getKey());
queryMapReq.setUser(user);
queryMapReq.setMapModeEnum(MapModeEnum.LOOSE);
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
Map<String, DataSetMapInfo> dataSetMapInfoMap = mapMeta.getDataSetMapInfo();
if (CollectionUtils.isEmpty(dataSetMapInfoMap)) {
return metricRespPageInfo;

224
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java
View File

@@ -3,71 +3,104 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import com.tencent.supersonic.headless.chat.mapper.MatchText;
import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType;
import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy;
import com.tencent.supersonic.headless.chat.QueryContext;
import com.tencent.supersonic.headless.chat.knowledge.DataSetInfoStat;
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.chat.mapper.MatchText;
import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType;
import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.ChatContextService;
import com.tencent.supersonic.headless.server.service.ChatQueryService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.SearchService;
import com.tencent.supersonic.headless.server.service.RetrieveService;
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* search service impl
*/
@Service
@Slf4j
public class SearchServiceImpl implements SearchService {
public class RetrieveServiceImpl implements RetrieveService {
private static final int RESULT_SIZE = 10;
@Autowired
private SemanticService semanticService;
@Autowired
private SearchMatchStrategy searchMatchStrategy;
@Autowired
private ChatContextService chatContextService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private DataSetService dataSetService;
@Autowired
private ChatQueryService chatQueryService;
@Autowired
private ChatContextService chatContextService;
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private SearchMatchStrategy searchMatchStrategy;
@Override
public MapInfoResp map(QueryMapReq queryMapReq) {
QueryReq queryReq = new QueryReq();
BeanUtils.copyProperties(queryMapReq, queryReq);
List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
queryReq.setDataSetIds(dataSetIds);
MapResp mapResp = chatQueryService.performMapping(queryReq);
dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet());
return convert(mapResp, queryMapReq.getTopN(), dataSetIds);
}
@Override
public List<SearchResult> search(QueryReq queryReq) {
String queryText = queryReq.getQueryText();
// 1.get meta info
SemanticSchema semanticSchemaDb = semanticService.getSemanticSchema();
SemanticSchema semanticSchemaDb = semanticLayerService.getSemanticSchema();
List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
final Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
Map<Long, List<Long>> modelIdToDataSetIds =
@@ -87,7 +120,7 @@ public class SearchServiceImpl implements SearchService {
regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue()));
// 3.get the most matching data
Optional<Entry<MatchText, List<HanlpMapResult>>> mostSimilarSearchResult = regTextMap.entrySet()
Optional<Map.Entry<MatchText, List<HanlpMapResult>>> mostSimilarSearchResult = regTextMap.entrySet()
.stream()
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
.reduce((entry1, entry2) ->
@@ -127,7 +160,7 @@ public class SearchServiceImpl implements SearchService {
}
private List<Long> getPossibleDataSets(QueryReq queryCtx, List<S2Term> originals,
DataSetInfoStat dataSetInfoStat, Set<Long> dataSetIds) {
DataSetInfoStat dataSetInfoStat, Set<Long> dataSetIds) {
if (CollectionUtils.isNotEmpty(dataSetIds)) {
return new ArrayList<>(dataSetIds);
}
@@ -152,13 +185,13 @@ public class SearchServiceImpl implements SearchService {
}
private Set<SearchResult> searchDimensionValue(List<SchemaElement> metricsDb,
Map<Long, String> modelToName,
long metricModelCount,
boolean existMetricAndDimension,
MatchText matchText,
Map<String, String> natureToNameMap,
Map.Entry<String, String> natureToNameEntry,
QueryFilters queryFilters) {
Map<Long, String> modelToName,
long metricModelCount,
boolean existMetricAndDimension,
MatchText matchText,
Map<String, String> natureToNameMap,
Map.Entry<String, String> natureToNameEntry,
QueryFilters queryFilters) {
Set<SearchResult> searchResults = new LinkedHashSet();
String nature = natureToNameEntry.getKey();
@@ -247,7 +280,7 @@ public class SearchServiceImpl implements SearchService {
* @return
*/
private Map<String, String> getNatureToNameMap(Map.Entry<MatchText, List<HanlpMapResult>> recommendTextListEntry,
Set<Long> possibleModels) {
Set<Long> possibleModels) {
List<HanlpMapResult> recommendValues = recommendTextListEntry.getValue();
return recommendValues.stream()
.flatMap(entry -> entry.getNatures().stream()
@@ -269,7 +302,8 @@ public class SearchServiceImpl implements SearchService {
}
private boolean searchMetricAndDimension(Set<Long> possibleDataSets, Map<Long, String> modelToName,
Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry, Set<SearchResult> searchResults) {
Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry,
Set<SearchResult> searchResults) {
boolean existMetric = false;
log.info("searchMetricAndDimension searchTextEntry:{}", searchTextEntry);
MatchText matchText = searchTextEntry.getKey();
@@ -317,4 +351,134 @@ public class SearchServiceImpl implements SearchService {
}
return possibleDataSets.contains(entry.getModel());
}
private MapInfoResp convert(MapResp mapResp, Integer topN, Set<Long> dataSetIds) {
MapInfoResp mapInfoResp = new MapInfoResp();
if (Objects.isNull(mapResp)) {
return mapInfoResp;
}
BeanUtils.copyProperties(mapResp, mapInfoResp);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(new ArrayList<>(dataSetIds));
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap));
return mapInfoResp;
}
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap,
Integer topN) {
Map<String, DataSetMapInfo> map = new HashMap<>();
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) {
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null) {
continue;
}
if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) {
continue;
}
DataSetMapInfo dataSetMapInfo = new DataSetMapInfo();
dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setName(dataSetResp.getName());
dataSetMapInfo.setDescription(dataSetResp.getDescription());
map.put(dataSetMapInfo.getName(), dataSetMapInfo);
}
return map;
}
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values = entry.getValue().stream()
.filter(schemaElementMatch ->
!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
result.put(entry.getKey(), values);
}
}
return result;
}
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN,
SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
if (0 == topN) {
return result;
}
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
Long dataSetId = entry.getKey();
List<SchemaElementMatch> values = entry.getValue();
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null || CollectionUtils.isEmpty(values)) {
continue;
}
String dataSetName = dataSetResp.getName();
//topN dimensions
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
dimensions.add(timeDimensionMatch);
//topN metrics
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN).map(mergeFunction()).collect(Collectors.toSet());
dimensions.addAll(metrics);
result.put(dataSetId, new ArrayList<>(dimensions));
}
return result;
}
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetNameMap) {
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
Map<Long, List<SchemaElementMatch>> dataSetElementMatches = mapInfo.getDataSetElementMatches();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey());
if (dataSetResp == null) {
continue;
}
List<SchemaElementMatch> terms = entry.getValue().stream().filter(schemaElementMatch
-> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
termMap.put(dataSetResp.getName(), terms);
}
return termMap;
}
/***
* get time dimension SchemaElementMatch
* @param dataSetId
* @param dataSetName
* @return
*/
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName)
.type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
.detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName())
.similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
return timeDimensionMatch;
}
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
.detectWord(schemaElement.getName()).build();
}
}

202
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticLayerServiceImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/S2SemanticLayerService.java
View File

@@ -3,15 +3,27 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataInfo;
import com.tencent.supersonic.headless.api.pojo.DataSetInfo;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
@@ -23,6 +35,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.headless.core.cache.QueryCache;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.translator.SemanticTranslator;
@@ -41,16 +54,23 @@ import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Slf4j
public class SemanticLayerServiceImpl implements SemanticLayerService {
public class S2SemanticLayerService implements SemanticLayerService {
private StatUtils statUtils;
private final QueryUtils queryUtils;
@@ -61,7 +81,7 @@ public class SemanticLayerServiceImpl implements SemanticLayerService {
private final SchemaService schemaService;
private final SemanticTranslator semanticTranslator;
public SemanticLayerServiceImpl(
public S2SemanticLayerService(
StatUtils statUtils,
QueryUtils queryUtils,
QueryReqConverter queryReqConverter,
@@ -88,6 +108,11 @@ public class SemanticLayerServiceImpl implements SemanticLayerService {
return schemaService.getDataSetSchema(id);
}
@Override
public List<DataSetSchema> getDataSetSchema() {
return schemaService.getDataSetSchema();
}
@Override
@S2DataPermission
@SneakyThrows
@@ -261,4 +286,177 @@ public class SemanticLayerServiceImpl implements SemanticLayerService {
throw e;
}
}
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) {
entityInfo.setMetrics(null);
entityInfo.setDimensions(null);
return entityInfo;
}
String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey();
if (StringUtils.isNotBlank(primaryKey)) {
String entityId = "";
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
.equals(primaryKey)) {
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
entityId = chatFilter.getValue().toString();
}
}
}
entityInfo.setEntityId(entityId);
try {
fillEntityInfoValue(entityInfo, dataSetSchema, user);
return entityInfo;
} catch (Exception e) {
log.error("setMainModel error", e);
}
}
}
return null;
}
private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) {
EntityInfo entityInfo = new EntityInfo();
if (dataSetSchema == null) {
return entityInfo;
}
Long dataSetId = dataSetSchema.getDataSet().getDataSet();
DataSetInfo dataSetInfo = new DataSetInfo();
dataSetInfo.setItemId(dataSetId.intValue());
dataSetInfo.setName(dataSetSchema.getDataSet().getName());
dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias());
dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName());
if (Objects.nonNull(dataSetSchema.getEntity())) {
dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName());
}
entityInfo.setDataSetInfo(dataSetInfo);
TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig();
if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) {
return entityInfo;
}
List<DataInfo> dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> {
SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
List<DataInfo> metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> {
SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
entityInfo.setDimensions(dimensions);
entityInfo.setMetrics(metrics);
return entityInfo;
}
private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
SemanticQueryResp queryResultWithColumns =
getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user);
if (queryResultWithColumns != null) {
if (!org.springframework.util.CollectionUtils.isEmpty(queryResultWithColumns.getResultList())
&& queryResultWithColumns.getResultList().size() > 0) {
Map<String, Object> result = queryResultWithColumns.getResultList().get(0);
for (Map.Entry<String, Object> entry : result.entrySet()) {
String entryKey = getEntryKey(entry);
if (entry.getValue() == null || entryKey == null) {
continue;
}
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
}
}
}
}
private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
DataSetSchema dataSetSchema, User user) {
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
semanticParseInfo.setQueryType(QueryType.DETAIL);
semanticParseInfo.setMetrics(getMetrics(entityInfo));
semanticParseInfo.setDimensions(getDimensions(entityInfo));
DateConf dateInfo = new DateConf();
int unit = 1;
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
if (Objects.nonNull(timeDefaultConfig)) {
unit = timeDefaultConfig.getUnit();
String date = LocalDate.now().plusDays(-unit).toString();
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(date);
dateInfo.setEndDate(date);
} else {
dateInfo.setUnit(unit);
dateInfo.setDateMode(DateConf.DateMode.RECENT);
}
semanticParseInfo.setDateInfo(dateInfo);
//add filter
QueryFilter chatFilter = getQueryFilter(entityInfo);
Set<QueryFilter> chatFilters = new LinkedHashSet();
chatFilters.add(chatFilter);
semanticParseInfo.setDimensionFilters(chatFilters);
SemanticQueryResp queryResultWithColumns = null;
try {
QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo);
queryResultWithColumns = queryByReq(queryStructReq, user);
} catch (Exception e) {
log.warn("setMainModel queryByStruct error, e:", e);
}
return queryResultWithColumns;
}
private QueryFilter getQueryFilter(EntityInfo entityInfo) {
QueryFilter chatFilter = new QueryFilter();
chatFilter.setValue(entityInfo.getEntityId());
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
chatFilter.setBizName(getEntityPrimaryName(entityInfo));
return chatFilter;
}
private Set<SchemaElement> getDimensions(EntityInfo modelInfo) {
Set<SchemaElement> dimensions = new LinkedHashSet();
for (DataInfo mainEntityDimension : modelInfo.getDimensions()) {
SchemaElement dimension = new SchemaElement();
dimension.setBizName(mainEntityDimension.getBizName());
dimensions.add(dimension);
}
return dimensions;
}
private String getEntryKey(Map.Entry<String, Object> entry) {
// metric parser special handle, TODO delete
String entryKey = entry.getKey();
if (entryKey.contains("__")) {
entryKey = entryKey.split("__")[1];
}
return entryKey;
}
private Set<SchemaElement> getMetrics(EntityInfo modelInfo) {
Set<SchemaElement> metrics = new LinkedHashSet();
for (DataInfo metricValue : modelInfo.getMetrics()) {
SchemaElement metric = new SchemaElement();
BeanUtils.copyProperties(metricValue, metric);
metrics.add(metric);
}
return metrics;
}
private String getEntityPrimaryName(EntityInfo entityInfo) {
return entityInfo.getDataSetInfo().getPrimaryKey();
}
}

229
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemanticService.java
View File

@@ -1,229 +0,0 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.DataInfo;
import com.tencent.supersonic.headless.api.pojo.DataSetInfo;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Slf4j
public class SemanticService {
@Autowired
private SchemaService schemaService;
@Autowired
private SemanticLayerService queryService;
public SemanticSchema getSemanticSchema() {
return new SemanticSchema(schemaService.getDataSetSchema());
}
public DataSetSchema getDataSetSchema(Long id) {
return schemaService.getDataSetSchema(id);
}
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) {
entityInfo.setMetrics(null);
entityInfo.setDimensions(null);
return entityInfo;
}
String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey();
if (StringUtils.isNotBlank(primaryKey)) {
String entityId = "";
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
.equals(primaryKey)) {
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
entityId = chatFilter.getValue().toString();
}
}
}
entityInfo.setEntityId(entityId);
try {
fillEntityInfoValue(entityInfo, dataSetSchema, user);
return entityInfo;
} catch (Exception e) {
log.error("setMainModel error", e);
}
}
}
return null;
}
private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) {
EntityInfo entityInfo = new EntityInfo();
if (dataSetSchema == null) {
return entityInfo;
}
Long dataSetId = dataSetSchema.getDataSet().getDataSet();
DataSetInfo dataSetInfo = new DataSetInfo();
dataSetInfo.setItemId(dataSetId.intValue());
dataSetInfo.setName(dataSetSchema.getDataSet().getName());
dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias());
dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName());
if (Objects.nonNull(dataSetSchema.getEntity())) {
dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName());
}
entityInfo.setDataSetInfo(dataSetInfo);
TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig();
if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) {
return entityInfo;
}
List<DataInfo> dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> {
SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
List<DataInfo> metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> {
SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
entityInfo.setDimensions(dimensions);
entityInfo.setMetrics(metrics);
return entityInfo;
}
public void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
SemanticQueryResp queryResultWithColumns =
getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user);
if (queryResultWithColumns != null) {
if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList())
&& queryResultWithColumns.getResultList().size() > 0) {
Map<String, Object> result = queryResultWithColumns.getResultList().get(0);
for (Map.Entry<String, Object> entry : result.entrySet()) {
String entryKey = getEntryKey(entry);
if (entry.getValue() == null || entryKey == null) {
continue;
}
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
}
}
}
}
public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
DataSetSchema dataSetSchema, User user) {
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
semanticParseInfo.setQueryType(QueryType.DETAIL);
semanticParseInfo.setMetrics(getMetrics(entityInfo));
semanticParseInfo.setDimensions(getDimensions(entityInfo));
DateConf dateInfo = new DateConf();
int unit = 1;
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
if (Objects.nonNull(timeDefaultConfig)) {
unit = timeDefaultConfig.getUnit();
String date = LocalDate.now().plusDays(-unit).toString();
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(date);
dateInfo.setEndDate(date);
} else {
dateInfo.setUnit(unit);
dateInfo.setDateMode(DateConf.DateMode.RECENT);
}
semanticParseInfo.setDateInfo(dateInfo);
//add filter
QueryFilter chatFilter = getQueryFilter(entityInfo);
Set<QueryFilter> chatFilters = new LinkedHashSet();
chatFilters.add(chatFilter);
semanticParseInfo.setDimensionFilters(chatFilters);
SemanticQueryResp queryResultWithColumns = null;
try {
QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo);
queryResultWithColumns = queryService.queryByReq(queryStructReq, user);
} catch (Exception e) {
log.warn("setMainModel queryByStruct error, e:", e);
}
return queryResultWithColumns;
}
private QueryFilter getQueryFilter(EntityInfo entityInfo) {
QueryFilter chatFilter = new QueryFilter();
chatFilter.setValue(entityInfo.getEntityId());
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
chatFilter.setBizName(getEntityPrimaryName(entityInfo));
return chatFilter;
}
private Set<SchemaElement> getDimensions(EntityInfo modelInfo) {
Set<SchemaElement> dimensions = new LinkedHashSet();
for (DataInfo mainEntityDimension : modelInfo.getDimensions()) {
SchemaElement dimension = new SchemaElement();
dimension.setBizName(mainEntityDimension.getBizName());
dimensions.add(dimension);
}
return dimensions;
}
private String getEntryKey(Map.Entry<String, Object> entry) {
// metric parser special handle, TODO delete
String entryKey = entry.getKey();
if (entryKey.contains("__")) {
entryKey = entryKey.split("__")[1];
}
return entryKey;
}
private Set<SchemaElement> getMetrics(EntityInfo modelInfo) {
Set<SchemaElement> metrics = new LinkedHashSet();
for (DataInfo metricValue : modelInfo.getMetrics()) {
SchemaElement metric = new SchemaElement();
BeanUtils.copyProperties(metricValue, metric);
metrics.add(metric);
}
return metrics;
}
private String getEntityPrimaryName(EntityInfo entityInfo) {
return entityInfo.getDataSetInfo().getPrimaryKey();
}
}

2
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
View File

@@ -70,7 +70,7 @@ public class MetricServiceImplTest {
DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
TagMetaService tagMetaService = Mockito.mock(TagMetaService.class);
MetaDiscoveryService metaDiscoveryService = Mockito.mock(MetaDiscoveryService.class);
RetrieveService metaDiscoveryService = Mockito.mock(RetrieveService.class);
return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper,
collectService, dataSetService, eventPublisher, dimensionService,
tagMetaService, metaDiscoveryService);

2
launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java
View File

@@ -84,7 +84,7 @@ public class Text2SQLEval extends BaseTest {
AgentConfig agentConfig = new AgentConfig();
agentConfig.getTools().add(getLLMQueryTool());
agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
agent.setLlmConfig(getLLMConfig(LLMType.DEEPSEEK));
agent.setLlmConfig(getLLMConfig(LLMType.GPT));
MultiTurnConfig multiTurnConfig = new MultiTurnConfig();
multiTurnConfig.setEnableMultiTurn(enableMultiturn);
agent.setMultiTurnConfig(multiTurnConfig);

10
launchers/standalone/src/test/java/com/tencent/supersonic/headless/MetaDiscoveryTest.java
View File

@@ -4,7 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.QueryDataType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
import com.tencent.supersonic.headless.server.service.RetrieveService;
import org.junit.Assert;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -15,7 +15,7 @@ import java.util.Collections;
public class MetaDiscoveryTest extends BaseTest {
@Autowired
protected MetaDiscoveryService metaDiscoveryService;
protected RetrieveService metaDiscoveryService;
@Test
public void testGetMapMeta() throws Exception {
@@ -24,7 +24,7 @@ public class MetaDiscoveryTest extends BaseTest {
queryMapReq.setTopN(10);
queryMapReq.setUser(User.getFakeUser());
queryMapReq.setDataSetNames(Collections.singletonList("超音数数据集"));
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
Assertions.assertNotNull(mapMeta);
Assertions.assertNotEquals(0, mapMeta.getDataSetMapInfo().get("超音数数据集").getMapFields());
@@ -39,7 +39,7 @@ public class MetaDiscoveryTest extends BaseTest {
queryMapReq.setUser(User.getFakeUser());
queryMapReq.setDataSetNames(Collections.singletonList("艺人库"));
queryMapReq.setQueryDataType(QueryDataType.TAG);
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
Assert.assertNotNull(mapMeta);
}
@@ -51,7 +51,7 @@ public class MetaDiscoveryTest extends BaseTest {
queryMapReq.setUser(User.getFakeUser());
queryMapReq.setDataSetNames(Collections.singletonList("超音数"));
queryMapReq.setQueryDataType(QueryDataType.METRIC);
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
Assert.assertNotNull(mapMeta);
}
}