topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Refactor service package of Headless-Server module (#1228)

Browse Source 
Co-authored-by: lxwcodemonkey
This commit is contained in:
LXW
2024-06-25 23:10:36 +08:00
committed by GitHub
parent e9dfb30ccf
commit 015f1e5204
46 changed files with 397 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -46,7 +46,7 @@ public class DimValueAspect {
@Autowired
private DimensionService dimensionService;
@Around("execution(* com.tencent.supersonic.headless.server.web.service.SemanticLayerService.queryByReq(..))")
@Around("execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/ChatQueryApiController.java
View File

@@ -24,14 +24,14 @@ public class ChatQueryApiController {
private ChatQueryService chatQueryService;
@Autowired
private RetrieveService searchService;
private RetrieveService retrieveService;
@PostMapping("/chat/search")
public Object search(@RequestBody QueryReq queryReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
queryReq.setUser(UserHolder.findUser(request, response));
return searchService.search(queryReq);
return retrieveService.retrieve(queryReq);
}
@PostMapping("/chat/map")

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/DataSetQueryApiController.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetaDiscoveryApiController.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.facade.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.server.facade.service.RetrieveService;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
@@ -20,14 +20,14 @@ import javax.servlet.http.HttpServletResponse;
public class MetaDiscoveryApiController {
@Autowired
private RetrieveService metaDiscovery;
private ChatQueryService chatQueryService;
@PostMapping("map")
public Object map(@RequestBody QueryMapReq queryMapReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
queryMapReq.setUser(user);
return metaDiscovery.map(queryMapReq);
return chatQueryService.map(queryMapReq);
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetricQueryApiController.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.server.web.service.DownloadService;
import com.tencent.supersonic.headless.server.web.service.MetricService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/SqlQueryApiController.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QuerySqlsReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/TagQueryApiController.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.facade.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

7
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/ChatQueryService.java
View File

@@ -1,14 +1,15 @@
package com.tencent.supersonic.headless.server.facade.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SqlEvaluation;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
@@ -20,6 +21,8 @@ public interface ChatQueryService {
MapResp performMapping(QueryReq queryReq);
MapInfoResp map(QueryMapReq queryMapReq);
ParseResp performParsing(QueryReq queryReq);
QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception;
@@ -28,8 +31,6 @@ public interface ChatQueryService {
QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws Exception;
EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user);
Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception;
void correct(QuerySqlReq querySqlReq, User user);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/RetrieveService.java
View File

@@ -1,16 +1,12 @@
package com.tencent.supersonic.headless.server.facade.service;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import java.util.List;
public interface RetrieveService {
MapInfoResp map(QueryMapReq queryMapReq);
List<SearchResult> search(QueryReq queryCtx);
List<SearchResult> retrieve(QueryReq queryCtx);
}

15
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/SemanticLayerService.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java
View File

@@ -1,16 +1,14 @@
package com.tencent.supersonic.headless.server.web.service;
package com.tencent.supersonic.headless.server.facade.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.util.List;
@@ -20,19 +18,16 @@ import java.util.List;
*/
public interface SemanticLayerService {
SemanticSchema getSemanticSchema();
DataSetSchema getDataSetSchema(Long id);
List<DataSetSchema> getDataSetSchema();
SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user);
List<ItemResp> getDomainDataSetTree();
}

179
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.server.facade.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
@@ -17,6 +18,9 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
@@ -30,11 +34,15 @@ import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
@@ -45,6 +53,7 @@ import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector;
import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.chat.knowledge.SearchService;
import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.chat.query.QueryManager;
@@ -53,10 +62,12 @@ import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery;
import com.tencent.supersonic.headless.chat.ChatContext;
import com.tencent.supersonic.headless.chat.QueryContext;
import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.web.service.ChatContextService;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import com.tencent.supersonic.headless.server.web.service.WorkflowService;
import com.tencent.supersonic.headless.server.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
@@ -80,21 +91,24 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@Slf4j
public class ChatQueryServiceImpl implements ChatQueryService {
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired
private SchemaService schemaService;
@Autowired
private ChatContextService chatContextService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@@ -116,6 +130,20 @@ public class ChatQueryServiceImpl implements ChatQueryService {
return mapResp;
}
@Override
public MapInfoResp map(QueryMapReq queryMapReq) {
QueryReq queryReq = new QueryReq();
BeanUtils.copyProperties(queryMapReq, queryReq);
List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
queryReq.setDataSetIds(dataSetIds);
MapResp mapResp = performMapping(queryReq);
dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet());
return convert(mapResp, queryMapReq.getTopN(), dataSetIds);
}
@Override
public ParseResp performParsing(QueryReq queryReq) {
ParseResp parseResult = new ParseResp(queryReq.getChatId(), queryReq.getQueryText());
@@ -135,7 +163,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
public QueryContext buildQueryContext(QueryReq queryReq) {
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds();
QueryContext queryCtx = QueryContext.builder()
.queryFilters(queryReq.getQueryFilters())
@@ -212,7 +240,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Override
public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws Exception {
SemanticParseInfo parseInfo = getSemanticParseInfo(queryData);
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
semanticQuery.setParseInfo(parseInfo);
@@ -321,14 +349,6 @@ public class ChatQueryServiceImpl implements ChatQueryService {
parseInfo.getSqlInfo().setCorrectS2SQL(correctorSql);
}
@Override
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dataSetSchema =
semanticService.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
return semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
}
private void updateDateInfo(QueryDataReq queryData, SemanticParseInfo parseInfo,
Map<String, Map<String, String>> filedNameToValueMap,
List<FieldExpression> fieldExpressionList,
@@ -519,8 +539,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Override
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
SchemaElement schemaElement = semanticSchema.getDimension(dimensionValueReq.getElementID());
Set<Long> detectDataSetIds = new HashSet<>();
detectDataSetIds.add(schemaElement.getDataSet());
@@ -604,7 +623,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
private SemanticParseInfo correctSqlReq(QuerySqlReq querySqlReq, User user) {
QueryContext queryCtx = new QueryContext();
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
queryCtx.setSemanticSchema(semanticSchema);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
SqlInfo sqlInfo = new SqlInfo();
@@ -629,4 +648,134 @@ public class ChatQueryServiceImpl implements ChatQueryService {
return semanticParseInfo;
}
private MapInfoResp convert(MapResp mapResp, Integer topN, Set<Long> dataSetIds) {
MapInfoResp mapInfoResp = new MapInfoResp();
if (Objects.isNull(mapResp)) {
return mapInfoResp;
}
BeanUtils.copyProperties(mapResp, mapInfoResp);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(new ArrayList<>(dataSetIds));
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap));
return mapInfoResp;
}
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap,
Integer topN) {
Map<String, DataSetMapInfo> map = new HashMap<>();
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) {
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null) {
continue;
}
if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) {
continue;
}
DataSetMapInfo dataSetMapInfo = new DataSetMapInfo();
dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setName(dataSetResp.getName());
dataSetMapInfo.setDescription(dataSetResp.getDescription());
map.put(dataSetMapInfo.getName(), dataSetMapInfo);
}
return map;
}
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values = entry.getValue().stream()
.filter(schemaElementMatch ->
!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
result.put(entry.getKey(), values);
}
}
return result;
}
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN,
SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
if (0 == topN) {
return result;
}
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
Long dataSetId = entry.getKey();
List<SchemaElementMatch> values = entry.getValue();
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null || CollectionUtils.isEmpty(values)) {
continue;
}
String dataSetName = dataSetResp.getName();
//topN dimensions
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
dimensions.add(timeDimensionMatch);
//topN metrics
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN).map(mergeFunction()).collect(Collectors.toSet());
dimensions.addAll(metrics);
result.put(dataSetId, new ArrayList<>(dimensions));
}
return result;
}
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetNameMap) {
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
Map<Long, List<SchemaElementMatch>> dataSetElementMatches = mapInfo.getDataSetElementMatches();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey());
if (dataSetResp == null) {
continue;
}
List<SchemaElementMatch> terms = entry.getValue().stream().filter(schemaElementMatch
-> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
termMap.put(dataSetResp.getName(), terms);
}
return termMap;
}
/***
* get time dimension SchemaElementMatch
* @param dataSetId
* @param dataSetName
* @return
*/
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName)
.type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
.detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName())
.similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
return timeDimensionMatch;
}
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
.detectWord(schemaElement.getName()).build();
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/FlightServiceImpl.java
View File

@@ -21,7 +21,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.facade.service.FlightService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.utils.FlightUtils;
import java.nio.charset.StandardCharsets;
import java.sql.ResultSet;

173
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/RetrieveServiceImpl.java
View File

@@ -3,21 +3,12 @@ package com.tencent.supersonic.headless.server.facade.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import com.tencent.supersonic.headless.chat.QueryContext;
@@ -25,18 +16,15 @@ import com.tencent.supersonic.headless.chat.knowledge.DataSetInfoStat;
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.chat.mapper.MatchText;
import com.tencent.supersonic.headless.chat.mapper.ModelWithSemanticType;
import com.tencent.supersonic.headless.chat.mapper.SearchMatchStrategy;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.web.service.ChatContextService;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.facade.service.RetrieveService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.web.service.ChatContextService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
@@ -45,7 +33,6 @@ import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -54,7 +41,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@@ -66,41 +52,23 @@ public class RetrieveServiceImpl implements RetrieveService {
@Autowired
private DataSetService dataSetService;
@Autowired
private ChatQueryService chatQueryService;
@Autowired
private ChatContextService chatContextService;
@Autowired
private SemanticLayerService semanticLayerService;
private SchemaService schemaService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private SearchMatchStrategy searchMatchStrategy;
@Override
public MapInfoResp map(QueryMapReq queryMapReq) {
QueryReq queryReq = new QueryReq();
BeanUtils.copyProperties(queryMapReq, queryReq);
List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
queryReq.setDataSetIds(dataSetIds);
MapResp mapResp = chatQueryService.performMapping(queryReq);
dataSetIds.retainAll(mapResp.getMapInfo().getDataSetElementMatches().keySet());
return convert(mapResp, queryMapReq.getTopN(), dataSetIds);
}
@Override
public List<SearchResult> search(QueryReq queryReq) {
public List<SearchResult> retrieve(QueryReq queryReq) {
String queryText = queryReq.getQueryText();
// 1.get meta info
SemanticSchema semanticSchemaDb = semanticLayerService.getSemanticSchema();
SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema();
List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
final Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
Map<Long, List<Long>> modelIdToDataSetIds =
@@ -352,133 +320,4 @@ public class RetrieveServiceImpl implements RetrieveService {
return possibleDataSets.contains(entry.getModel());
}
private MapInfoResp convert(MapResp mapResp, Integer topN, Set<Long> dataSetIds) {
MapInfoResp mapInfoResp = new MapInfoResp();
if (Objects.isNull(mapResp)) {
return mapInfoResp;
}
BeanUtils.copyProperties(mapResp, mapInfoResp);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(new ArrayList<>(dataSetIds));
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap));
return mapInfoResp;
}
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap,
Integer topN) {
Map<String, DataSetMapInfo> map = new HashMap<>();
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
for (Long dataSetId : mapInfo.getDataSetElementMatches().keySet()) {
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null) {
continue;
}
if (CollectionUtils.isEmpty(mapFields.get(dataSetId))) {
continue;
}
DataSetMapInfo dataSetMapInfo = new DataSetMapInfo();
dataSetMapInfo.setMapFields(mapFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setTopFields(topFields.getOrDefault(dataSetId, Lists.newArrayList()));
dataSetMapInfo.setName(dataSetResp.getName());
dataSetMapInfo.setDescription(dataSetResp.getDescription());
map.put(dataSetMapInfo.getName(), dataSetMapInfo);
}
return map;
}
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values = entry.getValue().stream()
.filter(schemaElementMatch ->
!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
result.put(entry.getKey(), values);
}
}
return result;
}
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN,
SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
if (0 == topN) {
return result;
}
SemanticSchema semanticSchema = semanticLayerService.getSemanticSchema();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
Long dataSetId = entry.getKey();
List<SchemaElementMatch> values = entry.getValue();
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
if (dataSetResp == null || CollectionUtils.isEmpty(values)) {
continue;
}
String dataSetName = dataSetResp.getName();
//topN dimensions
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
dimensions.add(timeDimensionMatch);
//topN metrics
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN).map(mergeFunction()).collect(Collectors.toSet());
dimensions.addAll(metrics);
result.put(dataSetId, new ArrayList<>(dimensions));
}
return result;
}
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetNameMap) {
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
Map<Long, List<SchemaElementMatch>> dataSetElementMatches = mapInfo.getDataSetElementMatches();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey());
if (dataSetResp == null) {
continue;
}
List<SchemaElementMatch> terms = entry.getValue().stream().filter(schemaElementMatch
-> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
termMap.put(dataSetResp.getName(), terms);
}
return termMap;
}
/***
* get time dimension SchemaElementMatch
* @param dataSetId
* @param dataSetName
* @return
*/
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
SchemaElement element = SchemaElement.builder().dataSet(dataSetId).dataSetName(dataSetName)
.type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
.detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName())
.similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
return timeDimensionMatch;
}
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
.detectWord(schemaElement.getName()).build();
}
}

45
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/S2SemanticLayerService.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.web.service.impl;
package com.tencent.supersonic.headless.server.facade.service.impl;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -17,12 +17,10 @@ import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
@@ -32,26 +30,25 @@ import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.headless.core.cache.QueryCache;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.SemanticTranslator;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.web.service.CatalogService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -76,7 +73,6 @@ public class S2SemanticLayerService implements SemanticLayerService {
private StatUtils statUtils;
private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter;
private final CatalogService catalog;
private final SemanticSchemaManager semanticSchemaManager;
private final DataSetService dataSetService;
private final SchemaService schemaService;
@@ -86,7 +82,6 @@ public class S2SemanticLayerService implements SemanticLayerService {
StatUtils statUtils,
QueryUtils queryUtils,
QueryReqConverter queryReqConverter,
CatalogService catalog,
SemanticSchemaManager semanticSchemaManager,
DataSetService dataSetService,
SchemaService schemaService,
@@ -94,26 +89,16 @@ public class S2SemanticLayerService implements SemanticLayerService {
this.statUtils = statUtils;
this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter;
this.catalog = catalog;
this.semanticSchemaManager = semanticSchemaManager;
this.dataSetService = dataSetService;
this.schemaService = schemaService;
this.semanticTranslator = semanticTranslator;
}
public SemanticSchema getSemanticSchema() {
return new SemanticSchema(schemaService.getDataSetSchema());
}
public DataSetSchema getDataSetSchema(Long id) {
return schemaService.getDataSetSchema(id);
}
@Override
public List<DataSetSchema> getDataSetSchema() {
return schemaService.getDataSetSchema();
}
@Override
@S2DataPermission
@SneakyThrows
@@ -163,7 +148,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
querySqlReq.setDataSetId(dataSetId);
}
SchemaFilterReq filter = buildSchemaFilterReq(querySqlReq);
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, semanticSchemaResp);
queryStatement.setModelIds(querySqlReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
@@ -191,7 +176,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq) {
SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq);
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
QueryStatement queryStatement = new QueryStatement();
QueryParam queryParam = new QueryParam();
queryReqConverter.convert(queryStructReq, queryParam);
@@ -243,12 +228,6 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryByReq(querySqlReq, user);
}
@Override
@SneakyThrows
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
return catalog.getStatInfo(itemUseReq);
}
@Override
public <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception {
T queryReq = explainSqlReq.getQueryReq();
@@ -264,10 +243,14 @@ public class S2SemanticLayerService implements SemanticLayerService {
return ExplainResp.builder().sql(sql).sourceId(sorceId).build();
}
public List<ItemResp> getDomainDataSetTree() {
return schemaService.getDomainDataSetTree();
}
private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
QuerySqlReq querySqlReq = new QuerySqlReq();
List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(),
List<ModelResp> modelResps = schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getDimensionBizName(),
queryDimValueReq.getModelId());
ModelResp modelResp = modelResps.get(0);
String sql = String.format("select distinct %s from %s", dimensionResp.getName(), modelResp.getName());

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -29,8 +29,13 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.web.service.CatalogService;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -41,20 +46,16 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j
@Service
public class SemanticSchemaManager {
private final CatalogService catalog;
private final SchemaService schemaService;
public SemanticSchemaManager(CatalogService catalog) {
this.catalog = catalog;
public SemanticSchemaManager(SchemaService schemaService) {
this.schemaService = schemaService;
}
public SemanticModel getSemanticModel(SemanticSchemaResp semanticSchemaResp) {
@@ -64,7 +65,7 @@ public class SemanticSchemaManager {
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
Map<Long, String> modelIdName = new HashMap<>();
catalog.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls,
schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls,
dataModelYamlTpls, metricYamlTpls, modelIdName);
DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp();
semanticModel.setDatabase(DatabaseConverter.convert(databaseResp));

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java
View File

@@ -1,13 +1,13 @@
package com.tencent.supersonic.headless.server.processor;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
@@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.chat.ChatContext;
import com.tencent.supersonic.headless.chat.QueryContext;
import com.tencent.supersonic.headless.chat.query.SemanticQuery;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -83,7 +83,7 @@ public class ParseInfoProcessor implements ResultProcessor {
log.error("set dimensionFilter error :", e);
}
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticLayerService.class).getSemanticSchema();
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
if (Objects.isNull(semanticSchema)) {
return;
}
@@ -195,7 +195,7 @@ public class ParseInfoProcessor implements ResultProcessor {
}
protected Map<String, SchemaElement> getNameToElement(Long dataSetId) {
SemanticSchema semanticSchema = ContextUtils.getBean(SemanticLayerService.class).getSemanticSchema();
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
List<SchemaElement> dimensions = semanticSchema.getDimensions(dataSetId);
List<SchemaElement> metrics = semanticSchema.getMetrics(dataSetId);

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java
View File

@@ -13,7 +13,7 @@ import com.tencent.supersonic.headless.chat.QueryContext;
import com.tencent.supersonic.headless.chat.query.QueryManager;
import com.tencent.supersonic.headless.chat.query.SemanticQuery;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
View File

@@ -38,7 +38,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
import com.tencent.supersonic.headless.server.web.service.MetricService;
import com.tencent.supersonic.headless.server.web.service.ModelService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.web.service.TagMetaService;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -1,5 +1,7 @@
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
@@ -7,8 +9,6 @@ import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
@@ -19,11 +19,10 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.web.service.CatalogService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -62,23 +61,22 @@ public class QueryStructUtils {
private final DateModeUtils dateModeUtils;
private final SqlFilterUtils sqlFilterUtils;
private final CatalogService catalog;
private final SchemaService schemaService;
private String variablePrefix = "'${";
public QueryStructUtils(
DateModeUtils dateModeUtils,
SqlFilterUtils sqlFilterUtils, @Lazy CatalogService catalog) {
public QueryStructUtils(DateModeUtils dateModeUtils,
SqlFilterUtils sqlFilterUtils, SchemaService schemaService) {
this.dateModeUtils = dateModeUtils;
this.sqlFilterUtils = sqlFilterUtils;
this.catalog = catalog;
this.schemaService = schemaService;
}
private List<Long> getDimensionIds(QueryStructReq queryStructReq) {
List<Long> dimensionIds = new ArrayList<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDataSetId(queryStructReq.getDataSetId());
List<DimensionResp> dimensions = catalog.getDimensions(metaFilter);
List<DimensionResp> dimensions = schemaService.getDimensions(metaFilter);
Map<String, List<DimensionResp>> pair = dimensions.stream()
.collect(Collectors.groupingBy(DimensionResp::getBizName));
for (String group : queryStructReq.getGroups()) {
@@ -100,7 +98,7 @@ public class QueryStructUtils {
List<Long> metricIds = new ArrayList<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDataSetId(queryStructCmd.getDataSetId());
List<MetricResp> metrics = catalog.getMetrics(metaFilter);
List<MetricResp> metrics = schemaService.getMetrics(metaFilter);
Map<String, List<MetricResp>> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName));
for (Aggregator agg : queryStructCmd.getAggregators()) {
if (pair.containsKey(agg.getColumn())) {
@@ -177,7 +175,7 @@ public class QueryStructUtils {
public ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) {
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
List<Long> metricIds = getMetricIds(queryStructCmd);
ItemDateResp dateDate = catalog.getItemDate(
ItemDateResp dateDate = schemaService.getItemDate(
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.name()),
new ItemDateFilter(metricIds, TypeEnums.METRIC.name()));
return dateDate;

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/rest/CollectController.java
View File

@@ -33,7 +33,7 @@ public class CollectController {
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return collectService.createCollectionIndicators(user, collectDO);
return collectService.collect(user, collectDO);
}
@Deprecated
@@ -42,7 +42,7 @@ public class CollectController {
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return collectService.deleteCollectionIndicators(user, id);
return collectService.unCollect(user, id);
}
@PostMapping("/deleteCollectionIndicators")
@@ -50,7 +50,7 @@ public class CollectController {
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return collectService.deleteCollectionIndicators(user, collectDO);
return collectService.unCollect(user, collectDO);
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/rest/DimensionController.java
View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;

36
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/rest/SchemaController.java
View File

@@ -1,23 +1,13 @@
package com.tencent.supersonic.headless.server.web.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@@ -33,11 +23,6 @@ public class SchemaController {
@Autowired
private SchemaService schemaService;
@PostMapping
public List<DataSetSchemaResp> fetchDataSetSchema(@RequestBody DataSetFilterReq filter) {
return schemaService.fetchDataSetSchema(filter);
}
@GetMapping("/domain/list")
public List<DomainResp> getDomainList(HttpServletRequest request,
HttpServletResponse response) {
@@ -54,25 +39,4 @@ public class SchemaController {
return schemaService.getModelList(user, AuthType.valueOf(authType), domainId);
}
@PostMapping("/dimension/page")
public PageInfo<DimensionResp> queryDimension(@RequestBody PageDimensionReq pageDimensionCmd,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.queryDimension(pageDimensionCmd, user);
}
@PostMapping("/metric/page")
public PageInfo<MetricResp> queryMetric(@RequestBody PageMetricReq pageMetricCmd,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.queryMetric(pageMetricCmd, user);
}
@PostMapping("/schemaItem/list")
public List querySchemaItem(@RequestBody SchemaItemQueryReq schemaItemQueryReq) {
return schemaService.querySchemaItem(schemaItemQueryReq);
}
}

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/CatalogService.java
View File

@@ -1,46 +0,0 @@
package com.tencent.supersonic.headless.server.web.service;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
public interface CatalogService {
DimensionResp getDimension(String bizName, Long modelId);
DimensionResp getDimension(Long id);
List<DimensionResp> getDimensions(MetaFilter metaFilter);
List<MetricResp> getMetrics(MetaFilter metaFilter);
MetricResp getMetric(Long id);
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
List<ModelResp> getModelList(List<Long> modelIds);
void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList,
List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName);
SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) throws ExecutionException;
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/CollectService.java
View File

@@ -11,13 +11,13 @@ import java.util.List;
public interface CollectService {
Boolean createCollectionIndicators(User user, CollectDO collectDO);
Boolean collect(User user, CollectDO collectDO);
Boolean deleteCollectionIndicators(User user, Long id);
Boolean unCollect(User user, Long id);
Boolean deleteCollectionIndicators(User user, CollectDO collectDO);
Boolean unCollect(User user, CollectDO collectDO);
List<CollectDO> getCollectList(String username);
List<CollectDO> getCollectionList(String username);
List<CollectDO> getCollectList(String username, TypeEnums typeEnums);
List<CollectDO> getCollectionList(String username, TypeEnums typeEnums);
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/DataSetService.java
View File

@@ -26,8 +26,6 @@ public interface DataSetService {
Map<Long, List<Long>> getModelIdToDataSetIds();
List<DataSetResp> getDataSets(User user);
List<DataSetResp> getDataSets(String dataSetName, User user);
List<DataSetResp> getDataSets(List<String> dataSetNames, User user);

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/DownloadService.java
View File

@@ -10,5 +10,7 @@ public interface DownloadService {
void downloadByStruct(DownloadMetricReq downloadStructReq,
User user, HttpServletResponse response) throws Exception;
void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) throws Exception;
void batchDownload(BatchDownloadReq batchDownloadReq, User user,
HttpServletResponse response) throws Exception;
}

36
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/SchemaService.java
View File

@@ -1,16 +1,13 @@
package com.tencent.supersonic.headless.server.web.service;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
@@ -19,33 +16,46 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
public interface SchemaService {
List<DataSetSchemaResp> fetchDataSetSchema(DataSetFilterReq filter);
DataSetSchema getDataSetSchema(Long dataSetId);
List<DataSetSchema> getDataSetSchema();
SemanticSchema getSemanticSchema();
SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq);
List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds);
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq, User user);
List<DimensionResp> getDimensions(MetaFilter metaFilter);
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
DimensionResp getDimension(String bizName, Long modelId);
List querySchemaItem(SchemaItemQueryReq schemaItemQueryReq);
List<MetricResp> getMetrics(MetaFilter metaFilter);
List<DomainResp> getDomainList(User user);
List<ModelResp> getModelList(User user, AuthType authType, Long domainId);
SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq);
List<ModelResp> getModelList(List<Long> modelIds);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) throws ExecutionException;
List<ItemResp> getDomainDataSetTree();
void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList,
List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName);
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
}

144
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/CatalogImpl.java
View File

@@ -1,144 +0,0 @@
package com.tencent.supersonic.headless.server.web.service.impl;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.server.manager.MetricYamlManager;
import com.tencent.supersonic.headless.server.manager.ModelYamlManager;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.web.service.CatalogService;
import com.tencent.supersonic.headless.server.web.service.DatabaseService;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
import com.tencent.supersonic.headless.server.web.service.MetricService;
import com.tencent.supersonic.headless.server.web.service.ModelRelaService;
import com.tencent.supersonic.headless.server.web.service.ModelService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@Slf4j
@Component
public class CatalogImpl implements CatalogService {
private final DatabaseService databaseService;
private final ModelService modelService;
private final DimensionService dimensionService;
private final MetricService metricService;
private final ModelRelaService modelRelaService;
private final DataSetService dataSetService;
private final SchemaService schemaService;
public CatalogImpl(DatabaseService databaseService, SchemaService schemaService,
ModelService modelService, DimensionService dimensionService, DataSetService dataSetService,
MetricService metricService, ModelRelaService modelRelaService) {
this.databaseService = databaseService;
this.modelService = modelService;
this.dimensionService = dimensionService;
this.dataSetService = dataSetService;
this.metricService = metricService;
this.modelRelaService = modelRelaService;
this.schemaService = schemaService;
}
@Override
public DimensionResp getDimension(String bizName, Long modelId) {
return dimensionService.getDimension(bizName, modelId);
}
@Override
public DimensionResp getDimension(Long id) {
return dimensionService.getDimension(id);
}
@Override
public List<DimensionResp> getDimensions(MetaFilter metaFilter) {
return dimensionService.getDimensions(metaFilter);
}
@Override
public List<MetricResp> getMetrics(MetaFilter metaFilter) {
return metricService.getMetrics(metaFilter);
}
@Override
public MetricResp getMetric(Long id) {
return metricService.getMetric(id);
}
@Override
public ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric) {
return modelService.getItemDate(dimension, metric);
}
@Override
public List<ModelResp> getModelList(List<Long> modelIds) {
List<ModelResp> modelRespList = new ArrayList<>();
if (!CollectionUtils.isEmpty(modelIds)) {
modelIds.stream().forEach(m -> {
modelRespList.add(modelService.getModel(m));
});
}
return modelRespList;
}
@Override
public void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList,
List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName) {
List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
if (CollectionUtils.isEmpty(modelResps)) {
return;
}
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
Long databaseId = modelResps.get(0).getDatabaseId();
DatabaseResp databaseResp = databaseService.getDatabase(databaseId);
for (ModelResp modelResp : modelResps) {
modelIdName.put(modelResp.getId(), modelResp.getBizName());
dataModelYamlTplList.add(ModelYamlManager.convert2YamlObj(modelResp, databaseResp));
if (!dimensionYamlMap.containsKey(modelResp.getBizName())) {
dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>());
}
List<DimensionResp> dimensionRespList = dimensionResps.stream()
.filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName()))
.collect(Collectors.toList());
dimensionYamlMap.get(modelResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml(
dimensionRespList));
}
List<MetricResp> metricResps = new ArrayList<>(semanticSchemaResp.getMetrics());
metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(metricResps));
}
@Override
public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) {
return schemaService.fetchSemanticSchema(schemaFilterReq);
}
@Override
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) throws ExecutionException {
return schemaService.getStatInfo(itemUseReq);
}
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/CollectServiceImpl.java
View File

@@ -24,7 +24,7 @@ public class CollectServiceImpl implements CollectService {
private CollectMapper collectMapper;
@Override
public Boolean createCollectionIndicators(User user, CollectDO collectReq) {
public Boolean collect(User user, CollectDO collectReq) {
CollectDO collect = new CollectDO();
collect.setType(Strings.isEmpty(collectReq.getType()) ? type : collectReq.getType());
collect.setUsername(user.getName());
@@ -34,7 +34,7 @@ public class CollectServiceImpl implements CollectService {
}
@Override
public Boolean deleteCollectionIndicators(User user, Long id) {
public Boolean unCollect(User user, Long id) {
QueryWrapper<CollectDO> collectDOQueryWrapper = new QueryWrapper<>();
collectDOQueryWrapper.lambda().eq(CollectDO::getUsername, user.getName());
collectDOQueryWrapper.lambda().eq(CollectDO::getId, id);
@@ -44,7 +44,7 @@ public class CollectServiceImpl implements CollectService {
}
@Override
public Boolean deleteCollectionIndicators(User user, CollectDO collectReq) {
public Boolean unCollect(User user, CollectDO collectReq) {
QueryWrapper<CollectDO> collectDOQueryWrapper = new QueryWrapper<>();
collectDOQueryWrapper.lambda().eq(CollectDO::getUsername, user.getName());
collectDOQueryWrapper.lambda().eq(CollectDO::getCollectId, collectReq.getCollectId());
@@ -54,7 +54,7 @@ public class CollectServiceImpl implements CollectService {
}
@Override
public List<CollectDO> getCollectList(String username) {
public List<CollectDO> getCollectionList(String username) {
QueryWrapper<CollectDO> queryWrapper = new QueryWrapper<>();
if (!StringUtils.isEmpty(username)) {
queryWrapper.lambda().eq(CollectDO::getUsername, username);
@@ -63,7 +63,7 @@ public class CollectServiceImpl implements CollectService {
}
@Override
public List<CollectDO> getCollectList(String username, TypeEnums typeEnums) {
public List<CollectDO> getCollectionList(String username, TypeEnums typeEnums) {
QueryWrapper<CollectDO> queryWrapper = new QueryWrapper<>();
if (!StringUtils.isEmpty(username)) {
queryWrapper.lambda().eq(CollectDO::getUsername, username);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/DataSetServiceImpl.java
View File

@@ -132,12 +132,6 @@ public class DataSetServiceImpl
updateById(dataSetDO);
}
@Override
public List<DataSetResp> getDataSets(User user) {
MetaFilter metaFilter = new MetaFilter();
return getDataSetsByAuth(user, metaFilter);
}
@Override
public List<DataSetResp> getDataSets(String dataSetName, User user) {
MetaFilter metaFilter = new MetaFilter();

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/DictWordService.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.chat.knowledge.builder.WordBuilderFactory;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -24,7 +24,7 @@ import java.util.stream.Collectors;
public class DictWordService {
@Autowired
private SemanticLayerService semanticLayerService;
private SchemaService schemaService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@@ -49,7 +49,7 @@ public class DictWordService {
}
public List<DictWord> getAllDictWords() {
SemanticSchema semanticSchema = new SemanticSchema(semanticLayerService.getDataSetSchema());
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
List<DictWord> words = new ArrayList<>();

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/DownloadServiceImpl.java
View File

@@ -28,7 +28,7 @@ import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
import com.tencent.supersonic.headless.server.web.service.DownloadService;
import com.tencent.supersonic.headless.server.web.service.MetricService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;

33
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/MetricServiceImpl.java
View File

@@ -8,6 +8,7 @@ import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DataEvent;
@@ -18,9 +19,6 @@ import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
@@ -46,10 +44,12 @@ import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.TagItem;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
@@ -58,16 +58,16 @@ import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.web.service.CollectService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
import com.tencent.supersonic.headless.server.facade.service.RetrieveService;
import com.tencent.supersonic.headless.server.web.service.MetricService;
import com.tencent.supersonic.headless.server.web.service.ModelService;
import com.tencent.supersonic.headless.server.web.service.TagMetaService;
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
import com.tencent.supersonic.headless.server.web.service.CollectService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
import com.tencent.supersonic.headless.server.web.service.MetricService;
import com.tencent.supersonic.headless.server.web.service.ModelService;
import com.tencent.supersonic.headless.server.web.service.TagMetaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@@ -75,6 +75,7 @@ import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -110,7 +111,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
private TagMetaService tagMetaService;
private RetrieveService metaDiscoveryService;
private ChatQueryService chatQueryService;
public MetricServiceImpl(MetricRepository metricRepository,
ModelService modelService,
@@ -120,7 +121,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService,
TagMetaService tagMetaService,
@Lazy RetrieveService metaDiscoveryService) {
@Lazy ChatQueryService chatQueryService) {
this.metricRepository = metricRepository;
this.modelService = modelService;
this.aliasGenerateHelper = aliasGenerateHelper;
@@ -129,7 +130,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
this.dataSetService = dataSetService;
this.dimensionService = dimensionService;
this.tagMetaService = tagMetaService;
this.metaDiscoveryService = metaDiscoveryService;
this.chatQueryService = chatQueryService;
}
@Override
@@ -298,7 +299,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
queryMapReq.setQueryText(pageMetricReq.getKey());
queryMapReq.setUser(user);
queryMapReq.setMapModeEnum(MapModeEnum.LOOSE);
MapInfoResp mapMeta = metaDiscoveryService.map(queryMapReq);
MapInfoResp mapMeta = chatQueryService.map(queryMapReq);
Map<String, DataSetMapInfo> dataSetMapInfoMap = mapMeta.getDataSetMapInfo();
if (CollectionUtils.isEmpty(dataSetMapInfoMap)) {
return metricRespPageInfo;
@@ -394,7 +395,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}
private List<Long> getCollectIds(PageMetricReq pageMetricReq, User user) {
List<CollectDO> collectList = collectService.getCollectList(user.getName(), TypeEnums.METRIC);
List<CollectDO> collectList = collectService.getCollectionList(user.getName(), TypeEnums.METRIC);
List<Long> collectIds = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList());
if (pageMetricReq.isHasCollect()) {
if (CollectionUtils.isEmpty(collectIds)) {
@@ -531,7 +532,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
ModelFilter modelFilter = new ModelFilter(false,
Lists.newArrayList(metricDO.getModelId()));
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
List<CollectDO> collectList = collectService.getCollectList(user.getName(), TypeEnums.METRIC);
List<CollectDO> collectList = collectService.getCollectionList(user.getName(), TypeEnums.METRIC);
List<Long> collect = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList());
MetricResp metricResp = MetricConverter.convert2MetricResp(metricDO, modelMap, collect);
fillAdminRes(Lists.newArrayList(metricResp), user);

109
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/SchemaServiceImpl.java
View File

@@ -1,24 +1,22 @@
package com.tencent.supersonic.headless.server.web.service.impl;
import com.github.pagehelper.PageInfo;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.enums.SchemaType;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
@@ -34,10 +32,21 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.TermResp;
import com.tencent.supersonic.headless.server.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.server.manager.MetricYamlManager;
import com.tencent.supersonic.headless.server.manager.ModelYamlManager;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.DatabaseService;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
import com.tencent.supersonic.headless.server.web.service.DomainService;
import com.tencent.supersonic.headless.server.web.service.MetricService;
@@ -46,10 +55,6 @@ import com.tencent.supersonic.headless.server.web.service.ModelService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import com.tencent.supersonic.headless.server.web.service.TagMetaService;
import com.tencent.supersonic.headless.server.web.service.TermService;
import com.tencent.supersonic.headless.server.utils.DataSetSchemaBuilder;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
@@ -91,6 +96,7 @@ public class SchemaServiceImpl implements SchemaService {
private final ModelRelaService modelRelaService;
private final TagMetaService tagService;
private final TermService termService;
private final DatabaseService databaseService;
@Value("${s2.schema.cache.enable:true}")
private boolean schemaCacheEnable;
@@ -101,7 +107,8 @@ public class SchemaServiceImpl implements SchemaService {
DomainService domainService,
DataSetService dataSetService,
ModelRelaService modelRelaService,
StatUtils statUtils, TagMetaService tagService, TermService termService) {
StatUtils statUtils, TagMetaService tagService,
TermService termService, DatabaseService databaseService) {
this.modelService = modelService;
this.dimensionService = dimensionService;
this.metricService = metricService;
@@ -111,10 +118,9 @@ public class SchemaServiceImpl implements SchemaService {
this.statUtils = statUtils;
this.tagService = tagService;
this.termService = termService;
this.databaseService = databaseService;
}
@SneakyThrows
@Override
public List<DataSetSchemaResp> fetchDataSetSchema(DataSetFilterReq filter) {
List<DataSetSchemaResp> dataSetList = Lists.newArrayList();
if (schemaCacheEnable) {
@@ -156,11 +162,6 @@ public class SchemaServiceImpl implements SchemaService {
return null;
}
@Override
public List<DataSetSchema> getDataSetSchema() {
return getDataSetSchema(new ArrayList<>());
}
public List<DataSetSchema> getDataSetSchema(List<Long> ids) {
List<DataSetSchema> domainSchemaList = new ArrayList<>();
@@ -171,6 +172,11 @@ public class SchemaServiceImpl implements SchemaService {
return domainSchemaList;
}
@Override
public SemanticSchema getSemanticSchema() {
return new SemanticSchema(getDataSetSchema(new ArrayList<>()));
}
public List<DataSetSchemaResp> buildDataSetSchema(DataSetFilterReq filter) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
@@ -307,25 +313,18 @@ public class SchemaServiceImpl implements SchemaService {
}
@Override
public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionCmd, User user) {
return dimensionService.queryDimension(pageDimensionCmd);
public DimensionResp getDimension(String bizName, Long modelId) {
return dimensionService.getDimension(bizName, modelId);
}
@Override
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user) {
return metricService.queryMetric(pageMetricReq, user);
public List<DimensionResp> getDimensions(MetaFilter metaFilter) {
return dimensionService.getDimensions(metaFilter);
}
@Override
public List querySchemaItem(SchemaItemQueryReq schemaItemQueryReq) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(schemaItemQueryReq.getIds());
if (TypeEnums.METRIC.equals(schemaItemQueryReq.getType())) {
return metricService.getMetrics(metaFilter);
} else if (TypeEnums.DIMENSION.equals(schemaItemQueryReq.getType())) {
return dimensionService.getDimensions(metaFilter);
}
throw new InvalidArgumentException("暂不支持的类型" + schemaItemQueryReq.getType().name());
public List<MetricResp> getMetrics(MetaFilter metaFilter) {
return metricService.getMetrics(metaFilter);
}
@Override
@@ -338,6 +337,17 @@ public class SchemaServiceImpl implements SchemaService {
return modelService.getModelListWithAuth(user, domainId, authTypeEnum);
}
@Override
public List<ModelResp> getModelList(List<Long> modelIds) {
List<ModelResp> modelRespList = new ArrayList<>();
if (!org.apache.commons.collections.CollectionUtils.isEmpty(modelIds)) {
modelIds.stream().forEach(m -> {
modelRespList.add(modelService.getModel(m));
});
}
return modelRespList;
}
public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) {
SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp();
semanticSchemaResp.setDataSetId(schemaFilterReq.getDataSetId());
@@ -365,7 +375,6 @@ public class SchemaServiceImpl implements SchemaService {
.flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
semanticSchemaResp.setSchemaType(SchemaType.MODEL);
}
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
// add tag info
@@ -424,8 +433,7 @@ public class SchemaServiceImpl implements SchemaService {
itemResp.getChildren().add(dataSet);
}
}
return itemResps.stream().filter(itemResp -> itemResp.getParentId() == 0)
.collect(Collectors.toList());
return itemResps.stream().filter(ItemResp::isRoot).collect(Collectors.toList());
}
private void fillStaticInfo(List<DataSetSchemaResp> dataSetSchemaResps) {
@@ -465,4 +473,39 @@ public class SchemaServiceImpl implements SchemaService {
return modelResp;
}
@Override
public void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList,
List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName) {
List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
if (org.apache.commons.collections.CollectionUtils.isEmpty(modelResps)) {
return;
}
List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
Long databaseId = modelResps.get(0).getDatabaseId();
DatabaseResp databaseResp = databaseService.getDatabase(databaseId);
for (ModelResp modelResp : modelResps) {
modelIdName.put(modelResp.getId(), modelResp.getBizName());
dataModelYamlTplList.add(ModelYamlManager.convert2YamlObj(modelResp, databaseResp));
if (!dimensionYamlMap.containsKey(modelResp.getBizName())) {
dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>());
}
List<DimensionResp> dimensionRespList = dimensionResps.stream()
.filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName()))
.collect(Collectors.toList());
dimensionYamlMap.get(modelResp.getBizName()).addAll(DimensionYamlManager.convert2DimensionYaml(
dimensionRespList));
}
List<MetricResp> metricResps = new ArrayList<>(semanticSchemaResp.getMetrics());
metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(metricResps));
}
@Override
public ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric) {
return modelService.getItemDate(dimension, metric);
}
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/TagMetaServiceImpl.java
View File

@@ -168,7 +168,7 @@ public class TagMetaServiceImpl implements TagMetaService {
TagFilter tagFilter = new TagFilter();
BeanUtils.copyProperties(tagMarketPageReq, tagFilter);
List<CollectDO> collectList = collectService.getCollectList(user.getName());
List<CollectDO> collectList = collectService.getCollectionList(user.getName());
if (tagMarketPageReq.isHasCollect()) {
List<Long> collectIds = collectList.stream()
.filter(collectDO -> SchemaElementType.TAG.name().equalsIgnoreCase(collectDO.getType()))
@@ -303,7 +303,7 @@ public class TagMetaServiceImpl implements TagMetaService {
}
private TagResp fillCollectAndAdminInfo(TagResp tagResp, User user) {
List<Long> collectIds = collectService.getCollectList(user.getName())
List<Long> collectIds = collectService.getCollectionList(user.getName())
.stream().filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
.map(CollectDO::getCollectId).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagResp.getId())) {
@@ -317,7 +317,7 @@ public class TagMetaServiceImpl implements TagMetaService {
}
private TagResp fillCollectAndAdminInfo(List<TagResp> tagRespList, User user) {
List<Long> collectIds = collectService.getCollectList(user.getName())
List<Long> collectIds = collectService.getCollectionList(user.getName())
.stream().filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
.map(CollectDO::getCollectId).collect(Collectors.toList());

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/TagQueryServiceImpl.java
View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.web.service.ModelService;
import com.tencent.supersonic.headless.server.web.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.web.service.TagMetaService;
import com.tencent.supersonic.headless.server.web.service.TagQueryService;
import lombok.extern.slf4j.Slf4j;

10
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
View File

@@ -16,9 +16,11 @@ import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.server.facade.service.RetrieveService;
import com.tencent.supersonic.headless.server.facade.service.ChatQueryService;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.web.service.CollectService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
@@ -27,8 +29,6 @@ import com.tencent.supersonic.headless.server.web.service.ModelService;
import com.tencent.supersonic.headless.server.web.service.TagMetaService;
import com.tencent.supersonic.headless.server.web.service.impl.DataSetServiceImpl;
import com.tencent.supersonic.headless.server.web.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
@@ -77,10 +77,10 @@ public class MetricServiceImplTest {
DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
TagMetaService tagMetaService = Mockito.mock(TagMetaService.class);
RetrieveService metaDiscoveryService = Mockito.mock(RetrieveService.class);
ChatQueryService chatQueryService = Mockito.mock(ChatQueryService.class);
return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper,
collectService, dataSetService, eventPublisher, dimensionService,
tagMetaService, metaDiscoveryService);
tagMetaService, chatQueryService);
}
private MetricReq buildMetricReq() {