mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-11 03:58:14 +00:00
(improvement)(headless)Remove redundant SemanticService and SearchService.
This commit is contained in:
@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
|
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
|
||||||
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
|
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
@@ -40,7 +40,7 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private List<SchemaElement> getDimensions(Long metricId, Long dataSetId) {
|
private List<SchemaElement> getDimensions(Long metricId, Long dataSetId) {
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
|
||||||
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(dataSetId);
|
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(dataSetId);
|
||||||
List<Long> drillDownDimensions = Lists.newArrayList();
|
List<Long> drillDownDimensions = Lists.newArrayList();
|
||||||
Set<SchemaElement> metricElements = dataSetSchema.getMetrics();
|
Set<SchemaElement> metricElements = dataSetSchema.getMetrics();
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.EntityInfo;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
||||||
import com.tencent.supersonic.headless.chat.query.QueryManager;
|
import com.tencent.supersonic.headless.chat.query.QueryManager;
|
||||||
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
|
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -30,7 +30,7 @@ public class EntityInfoProcessor implements ParseResultProcessor {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
//1. set entity info
|
//1. set entity info
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
|
||||||
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(parseInfo.getDataSetId());
|
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(parseInfo.getDataSetId());
|
||||||
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, chatParseContext.getUser());
|
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, chatParseContext.getUser());
|
||||||
if (QueryManager.isTagQuery(queryMode)
|
if (QueryManager.isTagQuery(queryMode)
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
|
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
|
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
|
||||||
import com.tencent.supersonic.headless.server.service.ChatQueryService;
|
import com.tencent.supersonic.headless.server.service.ChatQueryService;
|
||||||
import com.tencent.supersonic.headless.server.service.SearchService;
|
import com.tencent.supersonic.headless.server.service.RetrieveService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -47,7 +47,7 @@ public class ChatServiceImpl implements ChatService {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private ChatQueryService chatQueryService;
|
private ChatQueryService chatQueryService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private SearchService searchService;
|
private RetrieveService searchService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private SimilarQueryManager similarQueryManager;
|
private SimilarQueryManager similarQueryManager;
|
||||||
private List<ChatParser> chatParsers = ComponentFactory.getChatParsers();
|
private List<ChatParser> chatParsers = ComponentFactory.getChatParsers();
|
||||||
|
|||||||
@@ -33,10 +33,9 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
|||||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||||
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
|
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@@ -56,18 +55,17 @@ public class ConfigServiceImpl implements ConfigService {
|
|||||||
private final ChatConfigHelper chatConfigHelper;
|
private final ChatConfigHelper chatConfigHelper;
|
||||||
private final DimensionService dimensionService;
|
private final DimensionService dimensionService;
|
||||||
private final MetricService metricService;
|
private final MetricService metricService;
|
||||||
@Autowired
|
private final SemanticLayerService semanticLayerService;
|
||||||
private SemanticService semanticService;
|
|
||||||
|
|
||||||
|
|
||||||
public ConfigServiceImpl(ChatConfigRepository chatConfigRepository,
|
public ConfigServiceImpl(ChatConfigRepository chatConfigRepository,
|
||||||
ChatConfigHelper chatConfigHelper,
|
ChatConfigHelper chatConfigHelper, DimensionService dimensionService,
|
||||||
DimensionService dimensionService,
|
MetricService metricService, SemanticLayerService semanticLayerService) {
|
||||||
MetricService metricService) {
|
|
||||||
this.chatConfigRepository = chatConfigRepository;
|
this.chatConfigRepository = chatConfigRepository;
|
||||||
this.chatConfigHelper = chatConfigHelper;
|
this.chatConfigHelper = chatConfigHelper;
|
||||||
this.dimensionService = dimensionService;
|
this.dimensionService = dimensionService;
|
||||||
this.metricService = metricService;
|
this.metricService = metricService;
|
||||||
|
this.semanticLayerService = semanticLayerService;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -208,7 +206,7 @@ public class ConfigServiceImpl implements ConfigService {
|
|||||||
}
|
}
|
||||||
BeanUtils.copyProperties(chatConfigResp, chatConfigRich);
|
BeanUtils.copyProperties(chatConfigResp, chatConfigRich);
|
||||||
|
|
||||||
DataSetSchema dataSetSchema = semanticService.getDataSetSchema(modelId);
|
DataSetSchema dataSetSchema = semanticLayerService.getDataSetSchema(modelId);
|
||||||
if (dataSetSchema == null) {
|
if (dataSetSchema == null) {
|
||||||
return chatConfigRich;
|
return chatConfigRich;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
|||||||
import com.tencent.supersonic.headless.chat.ChatContext;
|
import com.tencent.supersonic.headless.chat.ChatContext;
|
||||||
import com.tencent.supersonic.headless.chat.QueryContext;
|
import com.tencent.supersonic.headless.chat.QueryContext;
|
||||||
import com.tencent.supersonic.headless.chat.query.SemanticQuery;
|
import com.tencent.supersonic.headless.chat.query.SemanticQuery;
|
||||||
import com.tencent.supersonic.headless.server.service.impl.SemanticService;
|
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.commons.lang3.tuple.Pair;
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
@@ -83,7 +83,7 @@ public class ParseInfoProcessor implements ResultProcessor {
|
|||||||
log.error("set dimensionFilter error :", e);
|
log.error("set dimensionFilter error :", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticLayerService.class).getSemanticSchema();
|
||||||
if (Objects.isNull(semanticSchema)) {
|
if (Objects.isNull(semanticSchema)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -195,7 +195,7 @@ public class ParseInfoProcessor implements ResultProcessor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected Map<String, SchemaElement> getNameToElement(Long dataSetId) {
|
protected Map<String, SchemaElement> getNameToElement(Long dataSetId) {
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticLayerService.class).getSemanticSchema();
|
||||||
List<SchemaElement> dimensions = semanticSchema.getDimensions(dataSetId);
|
List<SchemaElement> dimensions = semanticSchema.getDimensions(dataSetId);
|
||||||
List<SchemaElement> metrics = semanticSchema.getMetrics(dataSetId);
|
List<SchemaElement> metrics = semanticSchema.getMetrics(dataSetId);
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
|
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
|
||||||
import com.tencent.supersonic.headless.server.service.ChatQueryService;
|
import com.tencent.supersonic.headless.server.service.ChatQueryService;
|
||||||
import com.tencent.supersonic.headless.server.service.SearchService;
|
import com.tencent.supersonic.headless.server.service.RetrieveService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
@@ -22,8 +22,9 @@ public class ChatQueryApiController {
|
|||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private ChatQueryService chatQueryService;
|
private ChatQueryService chatQueryService;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private SearchService searchService;
|
private RetrieveService searchService;
|
||||||
|
|
||||||
@PostMapping("/chat/search")
|
@PostMapping("/chat/search")
|
||||||
public Object search(@RequestBody QueryReq queryReq,
|
public Object search(@RequestBody QueryReq queryReq,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.rest.api;
|
|||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
||||||
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
|
import com.tencent.supersonic.headless.server.service.RetrieveService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
@@ -20,14 +20,14 @@ import javax.servlet.http.HttpServletResponse;
|
|||||||
public class MetaDiscoveryApiController {
|
public class MetaDiscoveryApiController {
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private MetaDiscoveryService metaDiscovery;
|
private RetrieveService metaDiscovery;
|
||||||
|
|
||||||
@PostMapping("map")
|
@PostMapping("map")
|
||||||
public Object map(@RequestBody QueryMapReq queryMapReq,
|
public Object map(@RequestBody QueryMapReq queryMapReq,
|
||||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||||
User user = UserHolder.findUser(request, response);
|
User user = UserHolder.findUser(request, response);
|
||||||
queryMapReq.setUser(user);
|
queryMapReq.setUser(user);
|
||||||
return metaDiscovery.getMapMeta(queryMapReq);
|
return metaDiscovery.map(queryMapReq);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
package com.tencent.supersonic.headless.server.service;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
|
||||||
|
|
||||||
public interface MetaDiscoveryService {
|
|
||||||
|
|
||||||
MapInfoResp getMapMeta(QueryMapReq queryMapReq);
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,16 +1,16 @@
|
|||||||
package com.tencent.supersonic.headless.server.service;
|
package com.tencent.supersonic.headless.server.service;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
|
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface RetrieveService {
|
||||||
|
|
||||||
/**
|
MapInfoResp map(QueryMapReq queryMapReq);
|
||||||
* search service
|
|
||||||
*/
|
|
||||||
public interface SearchService {
|
|
||||||
|
|
||||||
List<SearchResult> search(QueryReq queryCtx);
|
List<SearchResult> search(QueryReq queryCtx);
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -2,6 +2,8 @@ package com.tencent.supersonic.headless.server.service;
|
|||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
|
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
||||||
@@ -13,19 +15,24 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
|||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This interface abstracts functionalities provided by a semantic layer.
|
||||||
|
*/
|
||||||
public interface SemanticLayerService {
|
public interface SemanticLayerService {
|
||||||
|
|
||||||
SemanticSchema getSemanticSchema();
|
SemanticSchema getSemanticSchema();
|
||||||
|
|
||||||
DataSetSchema getDataSetSchema(Long id);
|
DataSetSchema getDataSetSchema(Long id);
|
||||||
|
|
||||||
SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;
|
List<DataSetSchema> getDataSetSchema();
|
||||||
|
|
||||||
//List<SemanticQueryResp> queryByReqs(List<SemanticQueryReq> queryReqs, User user) throws Exception;
|
SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;
|
||||||
|
|
||||||
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
|
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
|
||||||
|
|
||||||
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
|
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
|
||||||
|
|
||||||
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
|
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
|
||||||
|
|
||||||
|
EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -93,14 +93,12 @@ import java.util.stream.Collectors;
|
|||||||
public class ChatQueryServiceImpl implements ChatQueryService {
|
public class ChatQueryServiceImpl implements ChatQueryService {
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private SemanticService semanticService;
|
private SemanticLayerService semanticLayerService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private ChatContextService chatContextService;
|
private ChatContextService chatContextService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private KnowledgeBaseService knowledgeBaseService;
|
private KnowledgeBaseService knowledgeBaseService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private SemanticLayerService queryService;
|
|
||||||
@Autowired
|
|
||||||
private DataSetService dataSetService;
|
private DataSetService dataSetService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private WorkflowService workflowService;
|
private WorkflowService workflowService;
|
||||||
@@ -137,7 +135,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
|
|
||||||
public QueryContext buildQueryContext(QueryReq queryReq) {
|
public QueryContext buildQueryContext(QueryReq queryReq) {
|
||||||
|
|
||||||
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
|
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
|
||||||
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds();
|
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds();
|
||||||
QueryContext queryCtx = QueryContext.builder()
|
QueryContext queryCtx = QueryContext.builder()
|
||||||
.queryFilters(queryReq.getQueryFilters())
|
.queryFilters(queryReq.getQueryFilters())
|
||||||
@@ -184,7 +182,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
|
|
||||||
private QueryResult doExecution(SemanticQueryReq semanticQueryReq,
|
private QueryResult doExecution(SemanticQueryReq semanticQueryReq,
|
||||||
SemanticParseInfo parseInfo, User user) throws Exception {
|
SemanticParseInfo parseInfo, User user) throws Exception {
|
||||||
SemanticQueryResp queryResp = queryService.queryByReq(semanticQueryReq, user);
|
SemanticQueryResp queryResp = semanticLayerService.queryByReq(semanticQueryReq, user);
|
||||||
QueryResult queryResult = new QueryResult();
|
QueryResult queryResult = new QueryResult();
|
||||||
if (queryResp != null) {
|
if (queryResp != null) {
|
||||||
queryResult.setQueryAuthorization(queryResp.getQueryAuthorization());
|
queryResult.setQueryAuthorization(queryResp.getQueryAuthorization());
|
||||||
@@ -214,7 +212,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
@Override
|
@Override
|
||||||
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws Exception {
|
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws Exception {
|
||||||
SemanticParseInfo parseInfo = getSemanticParseInfo(queryData);
|
SemanticParseInfo parseInfo = getSemanticParseInfo(queryData);
|
||||||
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
|
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
|
||||||
|
|
||||||
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
|
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
|
||||||
semanticQuery.setParseInfo(parseInfo);
|
semanticQuery.setParseInfo(parseInfo);
|
||||||
@@ -239,7 +237,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
|
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
|
||||||
ExplainSqlReq<Object> explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq)
|
ExplainSqlReq<Object> explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq)
|
||||||
.queryTypeEnum(QueryMethod.SQL).build();
|
.queryTypeEnum(QueryMethod.SQL).build();
|
||||||
ExplainResp explain = queryService.explain(explainSqlReq, user);
|
ExplainResp explain = semanticLayerService.explain(explainSqlReq, user);
|
||||||
if (StringUtils.isNotBlank(explain.getSql())) {
|
if (StringUtils.isNotBlank(explain.getSql())) {
|
||||||
parseInfo.getSqlInfo().setQuerySQL(explain.getSql());
|
parseInfo.getSqlInfo().setQuerySQL(explain.getSql());
|
||||||
}
|
}
|
||||||
@@ -258,7 +256,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
QueryResult queryResult = doExecution(semanticQueryReq, semanticQuery.getParseInfo(), user);
|
QueryResult queryResult = doExecution(semanticQueryReq, semanticQuery.getParseInfo(), user);
|
||||||
queryResult.setChatContext(semanticQuery.getParseInfo());
|
queryResult.setChatContext(semanticQuery.getParseInfo());
|
||||||
DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(parseInfo.getDataSetId());
|
DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(parseInfo.getDataSetId());
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
|
||||||
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
|
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
|
||||||
queryResult.setEntityInfo(entityInfo);
|
queryResult.setEntityInfo(entityInfo);
|
||||||
return queryResult;
|
return queryResult;
|
||||||
@@ -324,7 +322,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
|
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
|
||||||
DataSetSchema dataSetSchema =
|
DataSetSchema dataSetSchema =
|
||||||
semanticService.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
|
semanticService.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
|
||||||
return semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
|
return semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
|
||||||
@@ -520,7 +518,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
@Override
|
@Override
|
||||||
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
|
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
|
||||||
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
|
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
|
||||||
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
|
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
|
||||||
SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID());
|
SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID());
|
||||||
Set<Long> detectDataSetIds = new HashSet<>();
|
Set<Long> detectDataSetIds = new HashSet<>();
|
||||||
@@ -589,7 +587,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
List<String> groups = new ArrayList<>();
|
List<String> groups = new ArrayList<>();
|
||||||
groups.add(dimensionValueReq.getBizName());
|
groups.add(dimensionValueReq.getBizName());
|
||||||
queryStructReq.setGroups(groups);
|
queryStructReq.setGroups(groups);
|
||||||
return queryService.queryByReq(queryStructReq, user);
|
return semanticLayerService.queryByReq(queryStructReq, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void correct(QuerySqlReq querySqlReq, User user) {
|
public void correct(QuerySqlReq querySqlReq, User user) {
|
||||||
@@ -605,7 +603,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
|
|
||||||
private SemanticParseInfo correctSqlReq(QuerySqlReq querySqlReq, User user) {
|
private SemanticParseInfo correctSqlReq(QuerySqlReq querySqlReq, User user) {
|
||||||
QueryContext queryCtx = new QueryContext();
|
QueryContext queryCtx = new QueryContext();
|
||||||
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
|
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
|
||||||
queryCtx.setSemanticSchema(semanticSchema);
|
queryCtx.setSemanticSchema(semanticSchema);
|
||||||
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
||||||
SqlInfo sqlInfo = new SqlInfo();
|
SqlInfo sqlInfo = new SqlInfo();
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
|||||||
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
|
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
|
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.builder.WordBuilderFactory;
|
import com.tencent.supersonic.headless.chat.knowledge.builder.WordBuilderFactory;
|
||||||
import com.tencent.supersonic.headless.server.service.SchemaService;
|
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -24,7 +24,7 @@ import java.util.stream.Collectors;
|
|||||||
public class DictWordService {
|
public class DictWordService {
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private SchemaService schemaService;
|
private SemanticLayerService semanticLayerService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private KnowledgeBaseService knowledgeBaseService;
|
private KnowledgeBaseService knowledgeBaseService;
|
||||||
|
|
||||||
@@ -49,7 +49,7 @@ public class DictWordService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public List<DictWord> getAllDictWords() {
|
public List<DictWord> getAllDictWords() {
|
||||||
SemanticSchema semanticSchema = new SemanticSchema(schemaService.getDataSetSchema());
|
SemanticSchema semanticSchema = new SemanticSchema(semanticLayerService.getDataSetSchema());
|
||||||
|
|
||||||
List<DictWord> words = new ArrayList<>();
|
List<DictWord> words = new ArrayList<>();
|
||||||
|
|
||||||
|
|||||||
@@ -1,192 +0,0 @@
|
|||||||
package com.tencent.supersonic.headless.server.service.impl;
|
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
|
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
|
|
||||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
|
||||||
import com.tencent.supersonic.headless.server.service.ChatQueryService;
|
|
||||||
import com.tencent.supersonic.headless.server.service.DataSetService;
|
|
||||||
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
|
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
|
||||||
import org.springframework.beans.BeanUtils;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private DataSetService dataSetService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ChatQueryService chatQueryService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private SemanticService semanticService;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MapInfoResp getMapMeta(QueryMapReq queryMapReq) {
|
|
||||||
|
|
||||||
QueryReq queryReq = new QueryReq();
|
|
||||||
BeanUtils.copyProperties(queryMapReq, queryReq);
|
|
||||||
List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
|
|
||||||
|
|
||||||
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
|
|
||||||
queryReq.setDataSetIds(dataSetIds);
|
|
||||||
MapResp mapResp = chatQueryService.performMapping(queryReq);
|
|
||||||
dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet());
|
|
||||||
return convert(mapResp, queryMapReq.getTopN(), dataSetIds);
|
|
||||||
}
|
|
||||||
|
|
||||||
private MapInfoResp convert(MapResp mapResp, Integer topN, Set<Long> dataSetIds) {
|
|
||||||
MapInfoResp mapInfoResp = new MapInfoResp();
|
|
||||||
if (Objects.isNull(mapResp)) {
|
|
||||||
return mapInfoResp;
|
|
||||||
}
|
|
||||||
BeanUtils.copyProperties(mapResp, mapInfoResp);
|
|
||||||
MetaFilter metaFilter = new MetaFilter();
|
|
||||||
metaFilter.setIds(new ArrayList<>(dataSetIds));
|
|
||||||
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
|
|
||||||
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
|
|
||||||
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
|
|
||||||
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
|
|
||||||
mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap));
|
|
||||||
return mapInfoResp;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
|
|
||||||
Map<Long, DataSetResp> dataSetMap,
|
|
||||||
Integer topN) {
|
|
||||||
Map<String, DataSetMapInfo> map = new HashMap<>();
|
|
||||||
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
|
|
||||||
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
|
|
||||||
for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) {
|
|
||||||
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
|
|
||||||
if (dataSetResp == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
DataSetMapInfo dataSetMapInfo = new DataSetMapInfo();
|
|
||||||
dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList()));
|
|
||||||
dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList()));
|
|
||||||
dataSetMapInfo.setName(dataSetResp.getName());
|
|
||||||
dataSetMapInfo.setDescription(dataSetResp.getDescription());
|
|
||||||
map.put(dataSetMapInfo.getName(), dataSetMapInfo);
|
|
||||||
}
|
|
||||||
return map;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
|
|
||||||
Map<Long, DataSetResp> dataSetMap) {
|
|
||||||
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
|
|
||||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
|
|
||||||
List<SchemaElementMatch> values = entry.getValue().stream()
|
|
||||||
.filter(schemaElementMatch ->
|
|
||||||
!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
|
|
||||||
result.put(entry.getKey(), values);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN,
|
|
||||||
SchemaMapInfo mapInfo,
|
|
||||||
Map<Long, DataSetResp> dataSetMap) {
|
|
||||||
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
|
|
||||||
if (0 == topN) {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
|
|
||||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
|
|
||||||
Long dataSetId = entry.getKey();
|
|
||||||
List<SchemaElementMatch> values = entry.getValue();
|
|
||||||
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
|
|
||||||
if (dataSetResp == null || CollectionUtils.isEmpty(values)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String dataSetName = dataSetResp.getName();
|
|
||||||
//topN dimensions
|
|
||||||
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId)
|
|
||||||
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
|
|
||||||
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
|
|
||||||
|
|
||||||
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
|
|
||||||
dimensions.add(timeDimensionMatch);
|
|
||||||
|
|
||||||
//topN metrics
|
|
||||||
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId)
|
|
||||||
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
|
|
||||||
.limit(topN).map(mergeFunction()).collect(Collectors.toSet());
|
|
||||||
|
|
||||||
dimensions.addAll(metrics);
|
|
||||||
result.put(dataSetId, new ArrayList<>(dimensions));
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
|
|
||||||
Map<Long, DataSetResp> dataSetNameMap) {
|
|
||||||
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
|
|
||||||
Map<Long, List<SchemaElementMatch>> dataSetElementMatches = mapInfo.getDataSetElementMatches();
|
|
||||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
|
|
||||||
DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey());
|
|
||||||
if (dataSetResp == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
List<SchemaElementMatch> terms = entry.getValue().stream().filter(schemaElementMatch
|
|
||||||
-> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
termMap.put(dataSetResp.getName(), terms);
|
|
||||||
}
|
|
||||||
return termMap;
|
|
||||||
}
|
|
||||||
|
|
||||||
/***
|
|
||||||
* get time dimension SchemaElementMatch
|
|
||||||
* @param dataSetId
|
|
||||||
* @param dataSetName
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
|
|
||||||
SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName)
|
|
||||||
.type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();
|
|
||||||
|
|
||||||
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
|
|
||||||
.detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName())
|
|
||||||
.similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
|
|
||||||
|
|
||||||
return timeDimensionMatch;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
|
|
||||||
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
|
|
||||||
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
|
|
||||||
.detectWord(schemaElement.getName()).build();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -61,7 +61,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter;
|
|||||||
import com.tencent.supersonic.headless.server.service.CollectService;
|
import com.tencent.supersonic.headless.server.service.CollectService;
|
||||||
import com.tencent.supersonic.headless.server.service.DataSetService;
|
import com.tencent.supersonic.headless.server.service.DataSetService;
|
||||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||||
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
|
import com.tencent.supersonic.headless.server.service.RetrieveService;
|
||||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||||
import com.tencent.supersonic.headless.server.service.TagMetaService;
|
import com.tencent.supersonic.headless.server.service.TagMetaService;
|
||||||
@@ -110,7 +110,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
|
|
||||||
private TagMetaService tagMetaService;
|
private TagMetaService tagMetaService;
|
||||||
|
|
||||||
private MetaDiscoveryService metaDiscoveryService;
|
private RetrieveService metaDiscoveryService;
|
||||||
|
|
||||||
public MetricServiceImpl(MetricRepository metricRepository,
|
public MetricServiceImpl(MetricRepository metricRepository,
|
||||||
ModelService modelService,
|
ModelService modelService,
|
||||||
@@ -120,7 +120,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
ApplicationEventPublisher eventPublisher,
|
ApplicationEventPublisher eventPublisher,
|
||||||
DimensionService dimensionService,
|
DimensionService dimensionService,
|
||||||
TagMetaService tagMetaService,
|
TagMetaService tagMetaService,
|
||||||
@Lazy MetaDiscoveryService metaDiscoveryService) {
|
@Lazy RetrieveService metaDiscoveryService) {
|
||||||
this.metricRepository = metricRepository;
|
this.metricRepository = metricRepository;
|
||||||
this.modelService = modelService;
|
this.modelService = modelService;
|
||||||
this.aliasGenerateHelper = aliasGenerateHelper;
|
this.aliasGenerateHelper = aliasGenerateHelper;
|
||||||
@@ -298,7 +298,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
queryMapReq.setQueryText(pageMetricReq.getKey());
|
queryMapReq.setQueryText(pageMetricReq.getKey());
|
||||||
queryMapReq.setUser(user);
|
queryMapReq.setUser(user);
|
||||||
queryMapReq.setMapModeEnum(MapModeEnum.LOOSE);
|
queryMapReq.setMapModeEnum(MapModeEnum.LOOSE);
|
||||||
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
|
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
|
||||||
Map<String, DataSetMapInfo> dataSetMapInfoMap = mapMeta.getDataSetMapInfo();
|
Map<String, DataSetMapInfo> dataSetMapInfoMap = mapMeta.getDataSetMapInfo();
|
||||||
if (CollectionUtils.isEmpty(dataSetMapInfoMap)) {
|
if (CollectionUtils.isEmpty(dataSetMapInfoMap)) {
|
||||||
return metricRespPageInfo;
|
return metricRespPageInfo;
|
||||||
|
|||||||
@@ -3,71 +3,104 @@ package com.tencent.supersonic.headless.server.service.impl;
|
|||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
|
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
|
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
|
||||||
import com.tencent.supersonic.headless.chat.mapper.MatchText;
|
|
||||||
import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType;
|
|
||||||
import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy;
|
|
||||||
import com.tencent.supersonic.headless.chat.QueryContext;
|
import com.tencent.supersonic.headless.chat.QueryContext;
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.DataSetInfoStat;
|
import com.tencent.supersonic.headless.chat.knowledge.DataSetInfoStat;
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
|
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
|
import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
|
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
|
||||||
|
import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
|
import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
|
||||||
import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
|
import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
|
||||||
|
import com.tencent.supersonic.headless.chat.mapper.MatchText;
|
||||||
|
import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType;
|
||||||
|
import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy;
|
||||||
|
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||||
import com.tencent.supersonic.headless.server.service.ChatContextService;
|
import com.tencent.supersonic.headless.server.service.ChatContextService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.ChatQueryService;
|
||||||
import com.tencent.supersonic.headless.server.service.DataSetService;
|
import com.tencent.supersonic.headless.server.service.DataSetService;
|
||||||
import com.tencent.supersonic.headless.server.service.SearchService;
|
import com.tencent.supersonic.headless.server.service.RetrieveService;
|
||||||
|
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* search service impl
|
|
||||||
*/
|
|
||||||
@Service
|
@Service
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class SearchServiceImpl implements SearchService {
|
public class RetrieveServiceImpl implements RetrieveService {
|
||||||
|
|
||||||
private static final int RESULT_SIZE = 10;
|
private static final int RESULT_SIZE = 10;
|
||||||
@Autowired
|
|
||||||
private SemanticService semanticService;
|
|
||||||
@Autowired
|
|
||||||
private SearchMatchStrategy searchMatchStrategy;
|
|
||||||
@Autowired
|
|
||||||
private ChatContextService chatContextService;
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeBaseService knowledgeBaseService;
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private DataSetService dataSetService;
|
private DataSetService dataSetService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private ChatQueryService chatQueryService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private ChatContextService chatContextService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private SemanticLayerService semanticLayerService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private KnowledgeBaseService knowledgeBaseService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private SearchMatchStrategy searchMatchStrategy;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MapInfoResp map(QueryMapReq queryMapReq) {
|
||||||
|
|
||||||
|
QueryReq queryReq = new QueryReq();
|
||||||
|
BeanUtils.copyProperties(queryMapReq, queryReq);
|
||||||
|
List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
|
||||||
|
|
||||||
|
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
|
||||||
|
queryReq.setDataSetIds(dataSetIds);
|
||||||
|
MapResp mapResp = chatQueryService.performMapping(queryReq);
|
||||||
|
dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet());
|
||||||
|
return convert(mapResp, queryMapReq.getTopN(), dataSetIds);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<SearchResult> search(QueryReq queryReq) {
|
public List<SearchResult> search(QueryReq queryReq) {
|
||||||
|
|
||||||
String queryText = queryReq.getQueryText();
|
String queryText = queryReq.getQueryText();
|
||||||
// 1.get meta info
|
// 1.get meta info
|
||||||
SemanticSchema semanticSchemaDb = semanticService.getSemanticSchema();
|
SemanticSchema semanticSchemaDb = semanticLayerService.getSemanticSchema();
|
||||||
List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
|
List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
|
||||||
final Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
|
final Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
|
||||||
Map<Long, List<Long>> modelIdToDataSetIds =
|
Map<Long, List<Long>> modelIdToDataSetIds =
|
||||||
@@ -87,7 +120,7 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue()));
|
regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue()));
|
||||||
|
|
||||||
// 3.get the most matching data
|
// 3.get the most matching data
|
||||||
Optional<Entry<MatchText, List<HanlpMapResult>>> mostSimilarSearchResult = regTextMap.entrySet()
|
Optional<Map.Entry<MatchText, List<HanlpMapResult>>> mostSimilarSearchResult = regTextMap.entrySet()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
|
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
|
||||||
.reduce((entry1, entry2) ->
|
.reduce((entry1, entry2) ->
|
||||||
@@ -127,7 +160,7 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private List<Long> getPossibleDataSets(QueryReq queryCtx, List<S2Term> originals,
|
private List<Long> getPossibleDataSets(QueryReq queryCtx, List<S2Term> originals,
|
||||||
DataSetInfoStat dataSetInfoStat, Set<Long> dataSetIds) {
|
DataSetInfoStat dataSetInfoStat, Set<Long> dataSetIds) {
|
||||||
if (CollectionUtils.isNotEmpty(dataSetIds)) {
|
if (CollectionUtils.isNotEmpty(dataSetIds)) {
|
||||||
return new ArrayList<>(dataSetIds);
|
return new ArrayList<>(dataSetIds);
|
||||||
}
|
}
|
||||||
@@ -152,13 +185,13 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private Set<SearchResult> searchDimensionValue(List<SchemaElement> metricsDb,
|
private Set<SearchResult> searchDimensionValue(List<SchemaElement> metricsDb,
|
||||||
Map<Long, String> modelToName,
|
Map<Long, String> modelToName,
|
||||||
long metricModelCount,
|
long metricModelCount,
|
||||||
boolean existMetricAndDimension,
|
boolean existMetricAndDimension,
|
||||||
MatchText matchText,
|
MatchText matchText,
|
||||||
Map<String, String> natureToNameMap,
|
Map<String, String> natureToNameMap,
|
||||||
Map.Entry<String, String> natureToNameEntry,
|
Map.Entry<String, String> natureToNameEntry,
|
||||||
QueryFilters queryFilters) {
|
QueryFilters queryFilters) {
|
||||||
|
|
||||||
Set<SearchResult> searchResults = new LinkedHashSet();
|
Set<SearchResult> searchResults = new LinkedHashSet();
|
||||||
String nature = natureToNameEntry.getKey();
|
String nature = natureToNameEntry.getKey();
|
||||||
@@ -247,7 +280,7 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
private Map<String, String> getNatureToNameMap(Map.Entry<MatchText, List<HanlpMapResult>> recommendTextListEntry,
|
private Map<String, String> getNatureToNameMap(Map.Entry<MatchText, List<HanlpMapResult>> recommendTextListEntry,
|
||||||
Set<Long> possibleModels) {
|
Set<Long> possibleModels) {
|
||||||
List<HanlpMapResult> recommendValues = recommendTextListEntry.getValue();
|
List<HanlpMapResult> recommendValues = recommendTextListEntry.getValue();
|
||||||
return recommendValues.stream()
|
return recommendValues.stream()
|
||||||
.flatMap(entry -> entry.getNatures().stream()
|
.flatMap(entry -> entry.getNatures().stream()
|
||||||
@@ -269,7 +302,8 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private boolean searchMetricAndDimension(Set<Long> possibleDataSets, Map<Long, String> modelToName,
|
private boolean searchMetricAndDimension(Set<Long> possibleDataSets, Map<Long, String> modelToName,
|
||||||
Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry, Set<SearchResult> searchResults) {
|
Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry,
|
||||||
|
Set<SearchResult> searchResults) {
|
||||||
boolean existMetric = false;
|
boolean existMetric = false;
|
||||||
log.info("searchMetricAndDimension searchTextEntry:{}", searchTextEntry);
|
log.info("searchMetricAndDimension searchTextEntry:{}", searchTextEntry);
|
||||||
MatchText matchText = searchTextEntry.getKey();
|
MatchText matchText = searchTextEntry.getKey();
|
||||||
@@ -317,4 +351,134 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
}
|
}
|
||||||
return possibleDataSets.contains(entry.getModel());
|
return possibleDataSets.contains(entry.getModel());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private MapInfoResp convert(MapResp mapResp, Integer topN, Set<Long> dataSetIds) {
|
||||||
|
MapInfoResp mapInfoResp = new MapInfoResp();
|
||||||
|
if (Objects.isNull(mapResp)) {
|
||||||
|
return mapInfoResp;
|
||||||
|
}
|
||||||
|
BeanUtils.copyProperties(mapResp, mapInfoResp);
|
||||||
|
MetaFilter metaFilter = new MetaFilter();
|
||||||
|
metaFilter.setIds(new ArrayList<>(dataSetIds));
|
||||||
|
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
|
||||||
|
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
|
||||||
|
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
|
||||||
|
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
|
||||||
|
mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap));
|
||||||
|
return mapInfoResp;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
|
||||||
|
Map<Long, DataSetResp> dataSetMap,
|
||||||
|
Integer topN) {
|
||||||
|
Map<String, DataSetMapInfo> map = new HashMap<>();
|
||||||
|
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
|
||||||
|
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
|
||||||
|
for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) {
|
||||||
|
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
|
||||||
|
if (dataSetResp == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
DataSetMapInfo dataSetMapInfo = new DataSetMapInfo();
|
||||||
|
dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList()));
|
||||||
|
dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList()));
|
||||||
|
dataSetMapInfo.setName(dataSetResp.getName());
|
||||||
|
dataSetMapInfo.setDescription(dataSetResp.getDescription());
|
||||||
|
map.put(dataSetMapInfo.getName(), dataSetMapInfo);
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
|
||||||
|
Map<Long, DataSetResp> dataSetMap) {
|
||||||
|
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
|
||||||
|
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
|
||||||
|
List<SchemaElementMatch> values = entry.getValue().stream()
|
||||||
|
.filter(schemaElementMatch ->
|
||||||
|
!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
|
||||||
|
result.put(entry.getKey(), values);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN,
|
||||||
|
SchemaMapInfo mapInfo,
|
||||||
|
Map<Long, DataSetResp> dataSetMap) {
|
||||||
|
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
|
||||||
|
if (0 == topN) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
|
||||||
|
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
|
||||||
|
Long dataSetId = entry.getKey();
|
||||||
|
List<SchemaElementMatch> values = entry.getValue();
|
||||||
|
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
|
||||||
|
if (dataSetResp == null || CollectionUtils.isEmpty(values)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String dataSetName = dataSetResp.getName();
|
||||||
|
//topN dimensions
|
||||||
|
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId)
|
||||||
|
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
|
||||||
|
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
|
||||||
|
|
||||||
|
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
|
||||||
|
dimensions.add(timeDimensionMatch);
|
||||||
|
|
||||||
|
//topN metrics
|
||||||
|
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId)
|
||||||
|
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
|
||||||
|
.limit(topN).map(mergeFunction()).collect(Collectors.toSet());
|
||||||
|
|
||||||
|
dimensions.addAll(metrics);
|
||||||
|
result.put(dataSetId, new ArrayList<>(dimensions));
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
|
||||||
|
Map<Long, DataSetResp> dataSetNameMap) {
|
||||||
|
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
|
||||||
|
Map<Long, List<SchemaElementMatch>> dataSetElementMatches = mapInfo.getDataSetElementMatches();
|
||||||
|
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
|
||||||
|
DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey());
|
||||||
|
if (dataSetResp == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
List<SchemaElementMatch> terms = entry.getValue().stream().filter(schemaElementMatch
|
||||||
|
-> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
termMap.put(dataSetResp.getName(), terms);
|
||||||
|
}
|
||||||
|
return termMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
/***
|
||||||
|
* get time dimension SchemaElementMatch
|
||||||
|
* @param dataSetId
|
||||||
|
* @param dataSetName
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
|
||||||
|
SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName)
|
||||||
|
.type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();
|
||||||
|
|
||||||
|
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
|
||||||
|
.detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName())
|
||||||
|
.similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
|
||||||
|
|
||||||
|
return timeDimensionMatch;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
|
||||||
|
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
|
||||||
|
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
|
||||||
|
.detectWord(schemaElement.getName()).build();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -3,15 +3,27 @@ package com.tencent.supersonic.headless.server.service.impl;
|
|||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.google.common.collect.Sets;
|
import com.google.common.collect.Sets;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.DataInfo;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.DataSetInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
||||||
import com.tencent.supersonic.headless.api.pojo.Dim;
|
import com.tencent.supersonic.headless.api.pojo.Dim;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.QueryParam;
|
import com.tencent.supersonic.headless.api.pojo.QueryParam;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
|
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||||
@@ -23,6 +35,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||||
|
import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
|
||||||
import com.tencent.supersonic.headless.core.cache.QueryCache;
|
import com.tencent.supersonic.headless.core.cache.QueryCache;
|
||||||
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
||||||
import com.tencent.supersonic.headless.core.translator.SemanticTranslator;
|
import com.tencent.supersonic.headless.core.translator.SemanticTranslator;
|
||||||
@@ -41,16 +54,23 @@ import com.tencent.supersonic.headless.server.utils.StatUtils;
|
|||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.LocalDate;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class SemanticLayerServiceImpl implements SemanticLayerService {
|
public class S2SemanticLayerService implements SemanticLayerService {
|
||||||
|
|
||||||
private StatUtils statUtils;
|
private StatUtils statUtils;
|
||||||
private final QueryUtils queryUtils;
|
private final QueryUtils queryUtils;
|
||||||
@@ -61,7 +81,7 @@ public class SemanticLayerServiceImpl implements SemanticLayerService {
|
|||||||
private final SchemaService schemaService;
|
private final SchemaService schemaService;
|
||||||
private final SemanticTranslator semanticTranslator;
|
private final SemanticTranslator semanticTranslator;
|
||||||
|
|
||||||
public SemanticLayerServiceImpl(
|
public S2SemanticLayerService(
|
||||||
StatUtils statUtils,
|
StatUtils statUtils,
|
||||||
QueryUtils queryUtils,
|
QueryUtils queryUtils,
|
||||||
QueryReqConverter queryReqConverter,
|
QueryReqConverter queryReqConverter,
|
||||||
@@ -88,6 +108,11 @@ public class SemanticLayerServiceImpl implements SemanticLayerService {
|
|||||||
return schemaService.getDataSetSchema(id);
|
return schemaService.getDataSetSchema(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DataSetSchema> getDataSetSchema() {
|
||||||
|
return schemaService.getDataSetSchema();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@S2DataPermission
|
@S2DataPermission
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
@@ -261,4 +286,177 @@ public class SemanticLayerServiceImpl implements SemanticLayerService {
|
|||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
|
||||||
|
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
|
||||||
|
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
|
||||||
|
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) {
|
||||||
|
entityInfo.setMetrics(null);
|
||||||
|
entityInfo.setDimensions(null);
|
||||||
|
return entityInfo;
|
||||||
|
}
|
||||||
|
String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey();
|
||||||
|
if (StringUtils.isNotBlank(primaryKey)) {
|
||||||
|
String entityId = "";
|
||||||
|
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
|
||||||
|
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
|
||||||
|
.equals(primaryKey)) {
|
||||||
|
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
|
||||||
|
entityId = chatFilter.getValue().toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entityInfo.setEntityId(entityId);
|
||||||
|
try {
|
||||||
|
fillEntityInfoValue(entityInfo, dataSetSchema, user);
|
||||||
|
return entityInfo;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("setMainModel error", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) {
|
||||||
|
|
||||||
|
EntityInfo entityInfo = new EntityInfo();
|
||||||
|
if (dataSetSchema == null) {
|
||||||
|
return entityInfo;
|
||||||
|
}
|
||||||
|
Long dataSetId = dataSetSchema.getDataSet().getDataSet();
|
||||||
|
DataSetInfo dataSetInfo = new DataSetInfo();
|
||||||
|
dataSetInfo.setItemId(dataSetId.intValue());
|
||||||
|
dataSetInfo.setName(dataSetSchema.getDataSet().getName());
|
||||||
|
dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias());
|
||||||
|
dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName());
|
||||||
|
if (Objects.nonNull(dataSetSchema.getEntity())) {
|
||||||
|
dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName());
|
||||||
|
}
|
||||||
|
entityInfo.setDataSetInfo(dataSetInfo);
|
||||||
|
TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig();
|
||||||
|
if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) {
|
||||||
|
return entityInfo;
|
||||||
|
}
|
||||||
|
List<DataInfo> dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
|
||||||
|
.map(id -> {
|
||||||
|
SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id);
|
||||||
|
if (element == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
|
||||||
|
}).filter(Objects::nonNull).collect(Collectors.toList());
|
||||||
|
List<DataInfo> metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
|
||||||
|
.map(id -> {
|
||||||
|
SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id);
|
||||||
|
if (element == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
|
||||||
|
}).filter(Objects::nonNull).collect(Collectors.toList());
|
||||||
|
entityInfo.setDimensions(dimensions);
|
||||||
|
entityInfo.setMetrics(metrics);
|
||||||
|
return entityInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
|
||||||
|
SemanticQueryResp queryResultWithColumns =
|
||||||
|
getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user);
|
||||||
|
if (queryResultWithColumns != null) {
|
||||||
|
if (!org.springframework.util.CollectionUtils.isEmpty(queryResultWithColumns.getResultList())
|
||||||
|
&& queryResultWithColumns.getResultList().size() > 0) {
|
||||||
|
Map<String, Object> result = queryResultWithColumns.getResultList().get(0);
|
||||||
|
for (Map.Entry<String, Object> entry : result.entrySet()) {
|
||||||
|
String entryKey = getEntryKey(entry);
|
||||||
|
if (entry.getValue() == null || entryKey == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
|
||||||
|
.forEach(i -> i.setValue(entry.getValue().toString()));
|
||||||
|
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
|
||||||
|
.forEach(i -> i.setValue(entry.getValue().toString()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
|
||||||
|
DataSetSchema dataSetSchema, User user) {
|
||||||
|
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
||||||
|
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
|
||||||
|
semanticParseInfo.setQueryType(QueryType.DETAIL);
|
||||||
|
semanticParseInfo.setMetrics(getMetrics(entityInfo));
|
||||||
|
semanticParseInfo.setDimensions(getDimensions(entityInfo));
|
||||||
|
DateConf dateInfo = new DateConf();
|
||||||
|
int unit = 1;
|
||||||
|
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
|
||||||
|
if (Objects.nonNull(timeDefaultConfig)) {
|
||||||
|
unit = timeDefaultConfig.getUnit();
|
||||||
|
String date = LocalDate.now().plusDays(-unit).toString();
|
||||||
|
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
|
||||||
|
dateInfo.setStartDate(date);
|
||||||
|
dateInfo.setEndDate(date);
|
||||||
|
} else {
|
||||||
|
dateInfo.setUnit(unit);
|
||||||
|
dateInfo.setDateMode(DateConf.DateMode.RECENT);
|
||||||
|
}
|
||||||
|
semanticParseInfo.setDateInfo(dateInfo);
|
||||||
|
|
||||||
|
//add filter
|
||||||
|
QueryFilter chatFilter = getQueryFilter(entityInfo);
|
||||||
|
Set<QueryFilter> chatFilters = new LinkedHashSet();
|
||||||
|
chatFilters.add(chatFilter);
|
||||||
|
semanticParseInfo.setDimensionFilters(chatFilters);
|
||||||
|
|
||||||
|
SemanticQueryResp queryResultWithColumns = null;
|
||||||
|
try {
|
||||||
|
QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo);
|
||||||
|
queryResultWithColumns = queryByReq(queryStructReq, user);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("setMainModel queryByStruct error, e:", e);
|
||||||
|
}
|
||||||
|
return queryResultWithColumns;
|
||||||
|
}
|
||||||
|
|
||||||
|
private QueryFilter getQueryFilter(EntityInfo entityInfo) {
|
||||||
|
QueryFilter chatFilter = new QueryFilter();
|
||||||
|
chatFilter.setValue(entityInfo.getEntityId());
|
||||||
|
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
|
||||||
|
chatFilter.setBizName(getEntityPrimaryName(entityInfo));
|
||||||
|
return chatFilter;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Set<SchemaElement> getDimensions(EntityInfo modelInfo) {
|
||||||
|
Set<SchemaElement> dimensions = new LinkedHashSet();
|
||||||
|
for (DataInfo mainEntityDimension : modelInfo.getDimensions()) {
|
||||||
|
SchemaElement dimension = new SchemaElement();
|
||||||
|
dimension.setBizName(mainEntityDimension.getBizName());
|
||||||
|
dimensions.add(dimension);
|
||||||
|
}
|
||||||
|
return dimensions;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getEntryKey(Map.Entry<String, Object> entry) {
|
||||||
|
// metric parser special handle, TODO delete
|
||||||
|
String entryKey = entry.getKey();
|
||||||
|
if (entryKey.contains("__")) {
|
||||||
|
entryKey = entryKey.split("__")[1];
|
||||||
|
}
|
||||||
|
return entryKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Set<SchemaElement> getMetrics(EntityInfo modelInfo) {
|
||||||
|
Set<SchemaElement> metrics = new LinkedHashSet();
|
||||||
|
for (DataInfo metricValue : modelInfo.getMetrics()) {
|
||||||
|
SchemaElement metric = new SchemaElement();
|
||||||
|
BeanUtils.copyProperties(metricValue, metric);
|
||||||
|
metrics.add(metric);
|
||||||
|
}
|
||||||
|
return metrics;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getEntityPrimaryName(EntityInfo entityInfo) {
|
||||||
|
return entityInfo.getDataSetInfo().getPrimaryKey();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -1,229 +0,0 @@
|
|||||||
package com.tencent.supersonic.headless.server.service.impl;
|
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.DataInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.DataSetInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
|
||||||
import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
|
|
||||||
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
|
|
||||||
import com.tencent.supersonic.headless.server.service.SchemaService;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.springframework.beans.BeanUtils;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.time.LocalDate;
|
|
||||||
import java.util.LinkedHashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
@Slf4j
|
|
||||||
public class SemanticService {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private SchemaService schemaService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private SemanticLayerService queryService;
|
|
||||||
|
|
||||||
public SemanticSchema getSemanticSchema() {
|
|
||||||
return new SemanticSchema(schemaService.getDataSetSchema());
|
|
||||||
}
|
|
||||||
|
|
||||||
public DataSetSchema getDataSetSchema(Long id) {
|
|
||||||
return schemaService.getDataSetSchema(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
|
|
||||||
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
|
|
||||||
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
|
|
||||||
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) {
|
|
||||||
entityInfo.setMetrics(null);
|
|
||||||
entityInfo.setDimensions(null);
|
|
||||||
return entityInfo;
|
|
||||||
}
|
|
||||||
String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey();
|
|
||||||
if (StringUtils.isNotBlank(primaryKey)) {
|
|
||||||
String entityId = "";
|
|
||||||
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
|
|
||||||
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
|
|
||||||
.equals(primaryKey)) {
|
|
||||||
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
|
|
||||||
entityId = chatFilter.getValue().toString();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
entityInfo.setEntityId(entityId);
|
|
||||||
try {
|
|
||||||
fillEntityInfoValue(entityInfo, dataSetSchema, user);
|
|
||||||
return entityInfo;
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("setMainModel error", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) {
|
|
||||||
|
|
||||||
EntityInfo entityInfo = new EntityInfo();
|
|
||||||
if (dataSetSchema == null) {
|
|
||||||
return entityInfo;
|
|
||||||
}
|
|
||||||
Long dataSetId = dataSetSchema.getDataSet().getDataSet();
|
|
||||||
DataSetInfo dataSetInfo = new DataSetInfo();
|
|
||||||
dataSetInfo.setItemId(dataSetId.intValue());
|
|
||||||
dataSetInfo.setName(dataSetSchema.getDataSet().getName());
|
|
||||||
dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias());
|
|
||||||
dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName());
|
|
||||||
if (Objects.nonNull(dataSetSchema.getEntity())) {
|
|
||||||
dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName());
|
|
||||||
}
|
|
||||||
entityInfo.setDataSetInfo(dataSetInfo);
|
|
||||||
TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig();
|
|
||||||
if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) {
|
|
||||||
return entityInfo;
|
|
||||||
}
|
|
||||||
List<DataInfo> dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
|
|
||||||
.map(id -> {
|
|
||||||
SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id);
|
|
||||||
if (element == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
|
|
||||||
}).filter(Objects::nonNull).collect(Collectors.toList());
|
|
||||||
List<DataInfo> metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
|
|
||||||
.map(id -> {
|
|
||||||
SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id);
|
|
||||||
if (element == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
|
|
||||||
}).filter(Objects::nonNull).collect(Collectors.toList());
|
|
||||||
entityInfo.setDimensions(dimensions);
|
|
||||||
entityInfo.setMetrics(metrics);
|
|
||||||
return entityInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
|
|
||||||
SemanticQueryResp queryResultWithColumns =
|
|
||||||
getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user);
|
|
||||||
if (queryResultWithColumns != null) {
|
|
||||||
if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList())
|
|
||||||
&& queryResultWithColumns.getResultList().size() > 0) {
|
|
||||||
Map<String, Object> result = queryResultWithColumns.getResultList().get(0);
|
|
||||||
for (Map.Entry<String, Object> entry : result.entrySet()) {
|
|
||||||
String entryKey = getEntryKey(entry);
|
|
||||||
if (entry.getValue() == null || entryKey == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
|
|
||||||
.forEach(i -> i.setValue(entry.getValue().toString()));
|
|
||||||
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
|
|
||||||
.forEach(i -> i.setValue(entry.getValue().toString()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
|
|
||||||
DataSetSchema dataSetSchema, User user) {
|
|
||||||
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
|
||||||
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
|
|
||||||
semanticParseInfo.setQueryType(QueryType.DETAIL);
|
|
||||||
semanticParseInfo.setMetrics(getMetrics(entityInfo));
|
|
||||||
semanticParseInfo.setDimensions(getDimensions(entityInfo));
|
|
||||||
DateConf dateInfo = new DateConf();
|
|
||||||
int unit = 1;
|
|
||||||
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
|
|
||||||
if (Objects.nonNull(timeDefaultConfig)) {
|
|
||||||
unit = timeDefaultConfig.getUnit();
|
|
||||||
String date = LocalDate.now().plusDays(-unit).toString();
|
|
||||||
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
|
|
||||||
dateInfo.setStartDate(date);
|
|
||||||
dateInfo.setEndDate(date);
|
|
||||||
} else {
|
|
||||||
dateInfo.setUnit(unit);
|
|
||||||
dateInfo.setDateMode(DateConf.DateMode.RECENT);
|
|
||||||
}
|
|
||||||
semanticParseInfo.setDateInfo(dateInfo);
|
|
||||||
|
|
||||||
//add filter
|
|
||||||
QueryFilter chatFilter = getQueryFilter(entityInfo);
|
|
||||||
Set<QueryFilter> chatFilters = new LinkedHashSet();
|
|
||||||
chatFilters.add(chatFilter);
|
|
||||||
semanticParseInfo.setDimensionFilters(chatFilters);
|
|
||||||
|
|
||||||
SemanticQueryResp queryResultWithColumns = null;
|
|
||||||
try {
|
|
||||||
QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo);
|
|
||||||
queryResultWithColumns = queryService.queryByReq(queryStructReq, user);
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("setMainModel queryByStruct error, e:", e);
|
|
||||||
}
|
|
||||||
return queryResultWithColumns;
|
|
||||||
}
|
|
||||||
|
|
||||||
private QueryFilter getQueryFilter(EntityInfo entityInfo) {
|
|
||||||
QueryFilter chatFilter = new QueryFilter();
|
|
||||||
chatFilter.setValue(entityInfo.getEntityId());
|
|
||||||
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
|
|
||||||
chatFilter.setBizName(getEntityPrimaryName(entityInfo));
|
|
||||||
return chatFilter;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<SchemaElement> getDimensions(EntityInfo modelInfo) {
|
|
||||||
Set<SchemaElement> dimensions = new LinkedHashSet();
|
|
||||||
for (DataInfo mainEntityDimension : modelInfo.getDimensions()) {
|
|
||||||
SchemaElement dimension = new SchemaElement();
|
|
||||||
dimension.setBizName(mainEntityDimension.getBizName());
|
|
||||||
dimensions.add(dimension);
|
|
||||||
}
|
|
||||||
return dimensions;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getEntryKey(Map.Entry<String, Object> entry) {
|
|
||||||
// metric parser special handle, TODO delete
|
|
||||||
String entryKey = entry.getKey();
|
|
||||||
if (entryKey.contains("__")) {
|
|
||||||
entryKey = entryKey.split("__")[1];
|
|
||||||
}
|
|
||||||
return entryKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<SchemaElement> getMetrics(EntityInfo modelInfo) {
|
|
||||||
Set<SchemaElement> metrics = new LinkedHashSet();
|
|
||||||
for (DataInfo metricValue : modelInfo.getMetrics()) {
|
|
||||||
SchemaElement metric = new SchemaElement();
|
|
||||||
BeanUtils.copyProperties(metricValue, metric);
|
|
||||||
metrics.add(metric);
|
|
||||||
}
|
|
||||||
return metrics;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getEntityPrimaryName(EntityInfo entityInfo) {
|
|
||||||
return entityInfo.getDataSetInfo().getPrimaryKey();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -70,7 +70,7 @@ public class MetricServiceImplTest {
|
|||||||
DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
|
DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
|
||||||
DimensionService dimensionService = Mockito.mock(DimensionService.class);
|
DimensionService dimensionService = Mockito.mock(DimensionService.class);
|
||||||
TagMetaService tagMetaService = Mockito.mock(TagMetaService.class);
|
TagMetaService tagMetaService = Mockito.mock(TagMetaService.class);
|
||||||
MetaDiscoveryService metaDiscoveryService = Mockito.mock(MetaDiscoveryService.class);
|
RetrieveService metaDiscoveryService = Mockito.mock(RetrieveService.class);
|
||||||
return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper,
|
return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper,
|
||||||
collectService, dataSetService, eventPublisher, dimensionService,
|
collectService, dataSetService, eventPublisher, dimensionService,
|
||||||
tagMetaService, metaDiscoveryService);
|
tagMetaService, metaDiscoveryService);
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ public class Text2SQLEval extends BaseTest {
|
|||||||
AgentConfig agentConfig = new AgentConfig();
|
AgentConfig agentConfig = new AgentConfig();
|
||||||
agentConfig.getTools().add(getLLMQueryTool());
|
agentConfig.getTools().add(getLLMQueryTool());
|
||||||
agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
|
agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
|
||||||
agent.setLlmConfig(getLLMConfig(LLMType.DEEPSEEK));
|
agent.setLlmConfig(getLLMConfig(LLMType.GPT));
|
||||||
MultiTurnConfig multiTurnConfig = new MultiTurnConfig();
|
MultiTurnConfig multiTurnConfig = new MultiTurnConfig();
|
||||||
multiTurnConfig.setEnableMultiTurn(enableMultiturn);
|
multiTurnConfig.setEnableMultiTurn(enableMultiturn);
|
||||||
agent.setMultiTurnConfig(multiTurnConfig);
|
agent.setMultiTurnConfig(multiTurnConfig);
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
|||||||
import com.tencent.supersonic.headless.api.pojo.QueryDataType;
|
import com.tencent.supersonic.headless.api.pojo.QueryDataType;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
|
||||||
import com.tencent.supersonic.headless.server.service.MetaDiscoveryService;
|
import com.tencent.supersonic.headless.server.service.RetrieveService;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.jupiter.api.Assertions;
|
import org.junit.jupiter.api.Assertions;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
@@ -15,7 +15,7 @@ import java.util.Collections;
|
|||||||
public class MetaDiscoveryTest extends BaseTest {
|
public class MetaDiscoveryTest extends BaseTest {
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
protected MetaDiscoveryService metaDiscoveryService;
|
protected RetrieveService metaDiscoveryService;
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGetMapMeta() throws Exception {
|
public void testGetMapMeta() throws Exception {
|
||||||
@@ -24,7 +24,7 @@ public class MetaDiscoveryTest extends BaseTest {
|
|||||||
queryMapReq.setTopN(10);
|
queryMapReq.setTopN(10);
|
||||||
queryMapReq.setUser(User.getFakeUser());
|
queryMapReq.setUser(User.getFakeUser());
|
||||||
queryMapReq.setDataSetNames(Collections.singletonList("超音数数据集"));
|
queryMapReq.setDataSetNames(Collections.singletonList("超音数数据集"));
|
||||||
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
|
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
|
||||||
|
|
||||||
Assertions.assertNotNull(mapMeta);
|
Assertions.assertNotNull(mapMeta);
|
||||||
Assertions.assertNotEquals(0, mapMeta.getDataSetMapInfo().get("超音数数据集").getMapFields());
|
Assertions.assertNotEquals(0, mapMeta.getDataSetMapInfo().get("超音数数据集").getMapFields());
|
||||||
@@ -39,7 +39,7 @@ public class MetaDiscoveryTest extends BaseTest {
|
|||||||
queryMapReq.setUser(User.getFakeUser());
|
queryMapReq.setUser(User.getFakeUser());
|
||||||
queryMapReq.setDataSetNames(Collections.singletonList("艺人库"));
|
queryMapReq.setDataSetNames(Collections.singletonList("艺人库"));
|
||||||
queryMapReq.setQueryDataType(QueryDataType.TAG);
|
queryMapReq.setQueryDataType(QueryDataType.TAG);
|
||||||
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
|
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
|
||||||
Assert.assertNotNull(mapMeta);
|
Assert.assertNotNull(mapMeta);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,7 +51,7 @@ public class MetaDiscoveryTest extends BaseTest {
|
|||||||
queryMapReq.setUser(User.getFakeUser());
|
queryMapReq.setUser(User.getFakeUser());
|
||||||
queryMapReq.setDataSetNames(Collections.singletonList("超音数"));
|
queryMapReq.setDataSetNames(Collections.singletonList("超音数"));
|
||||||
queryMapReq.setQueryDataType(QueryDataType.METRIC);
|
queryMapReq.setQueryDataType(QueryDataType.METRIC);
|
||||||
MapInfoResp mapMeta = metaDiscoveryService.getMapMeta(queryMapReq);
|
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
|
||||||
Assert.assertNotNull(mapMeta);
|
Assert.assertNotNull(mapMeta);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user