topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 21:17:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless)(Chat) Change View to DataSet (#782)

Browse Source 
* (improvement)(Headless)(Chat) Change view to dataSet



---------

Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
LXW
2024-03-04 11:48:41 +08:00
committed by GitHub
parent b29e429271
commit a41da3f5fe
184 changed files with 1628 additions and 1532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
View File

@@ -37,10 +37,10 @@ public class MetricDrillDownChecker {
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] objects = joinPoint.getArgs();
QueryStatement queryStatement = (QueryStatement) objects[0];
if (queryStatement.getViewQueryParam() == null) {
if (queryStatement.getDataSetQueryParam() == null) {
return joinPoint.proceed();
}
checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getViewQueryParam().getSql());
checkQuery(queryStatement.getSemanticSchemaResp(), queryStatement.getDataSetQueryParam().getSql());
return joinPoint.proceed();
}

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -30,13 +30,13 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
@@ -84,7 +84,7 @@ public class S2DataPermissionAspect {
@Autowired
private SchemaService schemaService;
@Autowired
private ViewService viewService;
private DataSetService dataSetService;
@Autowired
private AuthService authService;
@@ -104,7 +104,7 @@ public class S2DataPermissionAspect {
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
List<Long> modelIds = getModelsInView(queryReq);
List<Long> modelIds = getModelsInDataSet(queryReq);
// determine whether admin of the model
if (doModelAdmin(user, modelIds)) {
@@ -129,21 +129,21 @@ public class S2DataPermissionAspect {
// fetch data permission meta information
SchemaFilterReq filter = new SchemaFilterReq();
filter.setModelIds(querySqlReq.getModelIds());
filter.setViewId(querySqlReq.getViewId());
filter.setDataSetId(querySqlReq.getDataSetId());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
List<Long> modelIdInView = semanticSchemaResp.getModelResps().stream()
List<Long> modelIdInDataSet = semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, semanticSchemaResp);
log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege);
log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInView, sensitiveResReq);
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInDataSet, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView);
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq);
@@ -157,7 +157,7 @@ public class S2DataPermissionAspect {
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource);
return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource);
}
// if the column has no permission, hit *
@@ -166,7 +166,7 @@ public class S2DataPermissionAspect {
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply);
addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
@@ -228,23 +228,23 @@ public class S2DataPermissionAspect {
// fetch data permission meta information
SchemaFilterReq filter = new SchemaFilterReq();
filter.setModelIds(queryStructReq.getModelIds());
filter.setViewId(queryStructReq.getViewId());
filter.setDataSetId(queryStructReq.getDataSetId());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
List<Long> modelIdInView = semanticSchemaResp.getModelResps().stream()
List<Long> modelIdInDataSet = semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelIdInView, res4Privilege);
log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInView, sensitiveResReq);
log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInDataSet, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
modelIdInView, sensitiveResReq);
modelIdInDataSet, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView);
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
@@ -258,7 +258,7 @@ public class S2DataPermissionAspect {
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIdInView, authorizedResource);
return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource);
}
// if the column has no permission, hit *
@@ -266,7 +266,7 @@ public class S2DataPermissionAspect {
.collect(Collectors.toSet());
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIdInView, queryResultAfterDesensitization, authorizedResource, need2Apply);
addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
@@ -557,11 +557,11 @@ public class S2DataPermissionAspect {
}
}
private List<Long> getModelsInView(SemanticQueryReq queryReq) {
private List<Long> getModelsInDataSet(SemanticQueryReq queryReq) {
List<Long> modelIds = queryReq.getModelIds();
if (queryReq.getViewId() != null) {
ViewResp viewResp = viewService.getView(queryReq.getViewId());
modelIds = viewResp.getAllModels();
if (queryReq.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(queryReq.getDataSetId());
modelIds = dataSetResp.getAllModels();
}
return modelIds;
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ViewDO.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java
View File

@@ -7,8 +7,8 @@ import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_view")
public class ViewDO {
@TableName("s2_data_set")
public class DataSetDO {
@TableId(type = IdType.AUTO)
private Long id;
@@ -25,7 +25,7 @@ public class ViewDO {
private String alias;
private String viewDetail;
private String dataSetDetail;
private Date createdAt;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ViewDOMapper.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ViewDOMapper extends BaseMapper<ViewDO> {
public interface DataSetDOMapper extends BaseMapper<DataSetDO> {
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java
View File

@@ -48,9 +48,9 @@ public class StatRepositoryImpl implements StatRepository {
statInfos.stream().forEach(stat -> {
String dimensions = stat.getDimensions();
String metrics = stat.getMetrics();
if (Objects.nonNull(stat.getViewId())) {
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getViewId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getViewId());
if (Objects.nonNull(stat.getDataSetId())) {
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getDataSetId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getDataSetId());
}
});
map.forEach((k, v) -> {
@@ -69,13 +69,13 @@ public class StatRepositoryImpl implements StatRepository {
return statMapper.getStatInfo(itemUseCommend);
}
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type, Long viewId) {
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type, Long dataSetId) {
if (Strings.isNotEmpty(dimensions)) {
try {
List<String> dimensionList = mapper.readValue(dimensions, new TypeReference<List<String>>() {
});
dimensionList.stream().forEach(dimension -> {
String key = viewId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension;
String key = dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension;
if (map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
View File

@@ -22,7 +22,7 @@ public class MetaFilter {
private Long domainId;
private Long viewId;
private Long dataSetId;
private Integer sensitiveLevel;
@@ -51,7 +51,7 @@ public class MetaFilter {
&& Objects.equal(bizName, that.bizName) && Objects.equal(
createdBy, that.createdBy) && Objects.equal(modelIds, that.modelIds)
&& Objects.equal(domainId, that.domainId) && Objects.equal(
viewId, that.viewId) && Objects.equal(sensitiveLevel, that.sensitiveLevel)
dataSetId, that.dataSetId) && Objects.equal(sensitiveLevel, that.sensitiveLevel)
&& Objects.equal(status, that.status) && Objects.equal(key,
that.key) && Objects.equal(ids, that.ids) && Objects.equal(
fieldsDepend, that.fieldsDepend);
@@ -59,7 +59,7 @@ public class MetaFilter {
@Override
public int hashCode() {
return Objects.hashCode(id, name, bizName, createdBy, modelIds, domainId, viewId, sensitiveLevel, status, key,
ids, fieldsDepend);
return Objects.hashCode(id, name, bizName, createdBy, modelIds, domainId,
dataSetId, sensitiveLevel, status, key, ids, fieldsDepend);
}
}

36
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ViewController.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java
View File

@@ -2,10 +2,10 @@ package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@@ -21,43 +21,43 @@ import javax.servlet.http.HttpServletResponse;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/view")
public class ViewController {
@RequestMapping("/api/semantic/dataSet")
public class DataSetController {
@Autowired
private ViewService viewService;
private DataSetService dataSetService;
@PostMapping
public ViewResp save(@RequestBody ViewReq viewReq,
HttpServletRequest request, HttpServletResponse response) {
public DataSetResp save(@RequestBody DataSetReq dataSetReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return viewService.save(viewReq, user);
return dataSetService.save(dataSetReq, user);
}
@PutMapping
public ViewResp update(@RequestBody ViewReq viewReq,
HttpServletRequest request, HttpServletResponse response) {
public DataSetResp update(@RequestBody DataSetReq dataSetReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return viewService.update(viewReq, user);
return dataSetService.update(dataSetReq, user);
}
@GetMapping("/{id}")
public ViewResp getView(@PathVariable("id") Long id) {
return viewService.getView(id);
public DataSetResp getDataSet(@PathVariable("id") Long id) {
return dataSetService.getDataSet(id);
}
@GetMapping("/getViewList")
public List<ViewResp> getViewList(@RequestParam("domainId") Long domainId) {
@GetMapping("/getDataSetList")
public List<DataSetResp> getDataSetList(@RequestParam("domainId") Long domainId) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDomainId(domainId);
return viewService.getViewList(metaFilter);
return dataSetService.getDataSetList(metaFilter);
}
@DeleteMapping("/{id}")
public Boolean delete(@PathVariable("id") Long id,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
viewService.delete(id, user);
dataSetService.delete(id, user);
return true;
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/SchemaController.java
View File

@@ -7,12 +7,12 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.server.service.SchemaService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
@@ -34,8 +34,8 @@ public class SchemaController {
private SchemaService schemaService;
@PostMapping
public List<ViewSchemaResp> fetchViewSchema(@RequestBody ViewFilterReq filter) {
return schemaService.fetchViewSchema(filter);
public List<DataSetSchemaResp> fetchDataSetSchema(@RequestBody DataSetFilterReq filter) {
return schemaService.fetchDataSetSchema(filter);
}
@GetMapping("/domain/list")

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/ViewQueryApiController.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/DataSetQueryApiController.java
View File

@@ -2,12 +2,10 @@ package com.tencent.supersonic.headless.server.rest.api;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.ViewService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
@@ -15,22 +13,25 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class ViewQueryApiController {
public class DataSetQueryApiController {
@Autowired
private ViewService viewService;
private DataSetService dataSetService;
@Autowired
private QueryService queryService;
@PostMapping("/view")
public Object queryByView(@RequestBody QueryViewReq queryViewReq,
@PostMapping("/dataSet")
public Object queryByDataSet(@RequestBody QueryDataSetReq queryDataSetReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
SemanticQueryReq queryReq = viewService.convert(queryViewReq);
SemanticQueryReq queryReq = dataSetService.convert(queryDataSetReq);
return queryService.queryByReq(queryReq, user);
}

32
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DataSetService.java Normal file
View File

@@ -0,0 +1,32 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;
import java.util.Map;
public interface DataSetService {
DataSetResp save(DataSetReq dataSetReq, User user);
DataSetResp update(DataSetReq dataSetReq, User user);
DataSetResp getDataSet(Long id);
List<DataSetResp> getDataSetList(MetaFilter metaFilter);
void delete(Long id, User user);
Map<Long, List<Long>> getModelIdToDataSetIds(List<Long> dataSetIds);
List<DataSetResp> getDataSets(User user);
List<DataSetResp> getDataSetsInheritAuth(User user, Long domainId);
SemanticQueryReq convert(QueryDataSetReq queryDataSetReq);
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/KnowledgeService.java
View File

@@ -12,9 +12,9 @@ public interface KnowledgeService {
List<S2Term> getTerms(String text);
List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> viewIds);
List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> dataSetIds);
List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> viewIds);
List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> dataSetIds);
void updateSemanticKnowledge(List<DictWord> natures);

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetaEmbeddingService.java
View File

@@ -6,6 +6,6 @@ import java.util.List;
public interface MetaEmbeddingService {
List<RetrieveQueryResult> retrieveQuery(List<Long> viewIds, RetrieveQuery retrieveQuery, int num);
List<RetrieveQueryResult> retrieveQuery(List<Long> dataSetIds, RetrieveQuery retrieveQuery, int num);
}

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
@@ -17,15 +17,15 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import java.util.List;
import java.util.concurrent.ExecutionException;
public interface SchemaService {
List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter);
List<DataSetSchemaResp> fetchDataSetSchema(DataSetFilterReq filter);
List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds);
@@ -39,11 +39,11 @@ public interface SchemaService {
List<ModelResp> getModelList(User user, AuthType authType, Long domainId);
List<ViewResp> getViewList(Long domainId);
List<DataSetResp> getDataSetList(Long domainId);
SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) throws ExecutionException;
List<ItemResp> getDomainViewTree();
List<ItemResp> getDomainDataSetTree();
}

32
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java
View File

@@ -1,32 +0,0 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import java.util.List;
import java.util.Map;
public interface ViewService {
ViewResp save(ViewReq viewReq, User user);
ViewResp update(ViewReq viewReq, User user);
ViewResp getView(Long id);
List<ViewResp> getViewList(MetaFilter metaFilter);
void delete(Long id, User user);
Map<Long, List<Long>> getModelIdToViewIds(List<Long> viewIds);
List<ViewResp> getViews(User user);
List<ViewResp> getViewsInheritAuth(User user, Long domainId);
SemanticQueryReq convert(QueryViewReq queryViewReq);
}

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CanvasServiceImpl.java
View File

@@ -24,7 +24,7 @@ import java.util.List;
@Service
public class CanvasServiceImpl {
private CanvasRepository viewInfoRepository;
private CanvasRepository canvasRepository;
private ModelService modelService;
@@ -32,16 +32,16 @@ public class CanvasServiceImpl {
private MetricService metricService;
public CanvasServiceImpl(CanvasRepository viewInfoRepository, ModelService modelService,
public CanvasServiceImpl(CanvasRepository canvasRepository, ModelService modelService,
MetricService metricService, DimensionService dimensionService) {
this.viewInfoRepository = viewInfoRepository;
this.canvasRepository = canvasRepository;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.modelService = modelService;
}
public List<CanvasDO> getCanvasList(Long domainId) {
return viewInfoRepository.getCanvasList(domainId);
return canvasRepository.getCanvasList(domainId);
}
public List<CanvasSchemaResp> getCanvasSchema(Long domainId, User user) {
@@ -70,20 +70,20 @@ public class CanvasServiceImpl {
viewInfoDO.setCreatedBy(user.getName());
viewInfoDO.setUpdatedAt(new Date());
viewInfoDO.setUpdatedBy(user.getName());
viewInfoRepository.createCanvas(viewInfoDO);
canvasRepository.createCanvas(viewInfoDO);
return viewInfoDO;
}
Long id = canvasReq.getId();
CanvasDO viewInfoDO = viewInfoRepository.getCanvasById(id);
CanvasDO viewInfoDO = canvasRepository.getCanvasById(id);
BeanUtils.copyProperties(canvasReq, viewInfoDO);
viewInfoDO.setUpdatedAt(new Date());
viewInfoDO.setUpdatedBy(user.getName());
viewInfoRepository.updateCanvas(viewInfoDO);
canvasRepository.updateCanvas(viewInfoDO);
return viewInfoDO;
}
public void deleteCanvas(Long id) {
viewInfoRepository.deleteCanvas(id);
canvasRepository.deleteCanvas(id);
}
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/CatalogImpl.java
View File

@@ -25,7 +25,7 @@ import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Component;
@@ -45,16 +45,16 @@ public class CatalogImpl implements Catalog {
private final DimensionService dimensionService;
private final MetricService metricService;
private final ModelRelaService modelRelaService;
private final ViewService viewService;
private final DataSetService dataSetService;
private final SchemaService schemaService;
public CatalogImpl(DatabaseService databaseService, SchemaService schemaService,
ModelService modelService, DimensionService dimensionService, ViewService viewService,
ModelService modelService, DimensionService dimensionService, DataSetService dataSetService,
MetricService metricService, ModelRelaService modelRelaService) {
this.databaseService = databaseService;
this.modelService = modelService;
this.dimensionService = dimensionService;
this.viewService = viewService;
this.dataSetService = dataSetService;
this.metricService = metricService;
this.modelRelaService = modelRelaService;
this.schemaService = schemaService;

186
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetImpl.java
View File

@@ -13,23 +13,23 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.ViewDetail;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO;
import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
import com.tencent.supersonic.headless.server.persistence.mapper.DataSetDOMapper;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
@@ -50,10 +50,10 @@ import java.util.function.Function;
import java.util.stream.Collectors;
@Service
public class ViewServiceImpl
extends ServiceImpl<ViewDOMapper, ViewDO> implements ViewService {
public class DataSetImpl
extends ServiceImpl<DataSetDOMapper, DataSetDO> implements DataSetService {
protected final Cache<MetaFilter, List<ViewResp>> viewSchemaCache =
protected final Cache<MetaFilter, List<DataSetResp>> dataSetSchemaCache =
CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build();
@Autowired
@@ -68,154 +68,156 @@ public class ViewServiceImpl
private MetricService metricService;
@Override
public ViewResp save(ViewReq viewReq, User user) {
viewReq.createdBy(user.getName());
ViewDO viewDO = convert(viewReq);
viewDO.setStatus(StatusEnum.ONLINE.getCode());
ViewResp viewResp = convert(viewDO);
conflictCheck(viewResp);
save(viewDO);
return viewResp;
public DataSetResp save(DataSetReq dataSetReq, User user) {
dataSetReq.createdBy(user.getName());
DataSetDO dataSetDO = convert(dataSetReq);
dataSetDO.setStatus(StatusEnum.ONLINE.getCode());
DataSetResp dataSetResp = convert(dataSetDO);
conflictCheck(dataSetResp);
save(dataSetDO);
return dataSetResp;
}
@Override
public ViewResp update(ViewReq viewReq, User user) {
viewReq.updatedBy(user.getName());
ViewDO viewDO = convert(viewReq);
ViewResp viewResp = convert(viewDO);
conflictCheck(viewResp);
updateById(viewDO);
return viewResp;
public DataSetResp update(DataSetReq dataSetReq, User user) {
dataSetReq.updatedBy(user.getName());
DataSetDO dataSetDO = convert(dataSetReq);
DataSetResp dataSetResp = convert(dataSetDO);
conflictCheck(dataSetResp);
updateById(dataSetDO);
return dataSetResp;
}
@Override
public ViewResp getView(Long id) {
ViewDO viewDO = getById(id);
return convert(viewDO);
public DataSetResp getDataSet(Long id) {
DataSetDO dataSetDO = getById(id);
return convert(dataSetDO);
}
@Override
public List<ViewResp> getViewList(MetaFilter metaFilter) {
QueryWrapper<ViewDO> wrapper = new QueryWrapper<>();
public List<DataSetResp> getDataSetList(MetaFilter metaFilter) {
QueryWrapper<DataSetDO> wrapper = new QueryWrapper<>();
if (metaFilter.getDomainId() != null) {
wrapper.lambda().eq(ViewDO::getDomainId, metaFilter.getDomainId());
wrapper.lambda().eq(DataSetDO::getDomainId, metaFilter.getDomainId());
}
if (!CollectionUtils.isEmpty(metaFilter.getIds())) {
wrapper.lambda().in(ViewDO::getId, metaFilter.getIds());
wrapper.lambda().in(DataSetDO::getId, metaFilter.getIds());
}
if (metaFilter.getStatus() != null) {
wrapper.lambda().eq(ViewDO::getStatus, metaFilter.getStatus());
wrapper.lambda().eq(DataSetDO::getStatus, metaFilter.getStatus());
}
wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode());
wrapper.lambda().ne(DataSetDO::getStatus, StatusEnum.DELETED.getCode());
return list(wrapper).stream().map(this::convert).collect(Collectors.toList());
}
@Override
public void delete(Long id, User user) {
ViewDO viewDO = getById(id);
viewDO.setStatus(StatusEnum.DELETED.getCode());
viewDO.setUpdatedBy(user.getName());
viewDO.setUpdatedAt(new Date());
updateById(viewDO);
DataSetDO dataSetDO = getById(id);
dataSetDO.setStatus(StatusEnum.DELETED.getCode());
dataSetDO.setUpdatedBy(user.getName());
dataSetDO.setUpdatedAt(new Date());
updateById(dataSetDO);
}
@Override
public List<ViewResp> getViews(User user) {
List<ViewResp> viewResps = getViewList(new MetaFilter());
return getViewFilterByAuth(viewResps, user);
public List<DataSetResp> getDataSets(User user) {
List<DataSetResp> dataSetResps = getDataSetList(new MetaFilter());
return getDataSetFilterByAuth(dataSetResps, user);
}
@Override
public List<ViewResp> getViewsInheritAuth(User user, Long domainId) {
List<ViewResp> viewResps = getViewList(new MetaFilter());
List<ViewResp> inheritAuthFormDomain = getViewFilterByDomainAuth(viewResps, user);
Set<ViewResp> viewRespSet = new HashSet<>(inheritAuthFormDomain);
List<ViewResp> viewFilterByAuth = getViewFilterByAuth(viewResps, user);
viewRespSet.addAll(viewFilterByAuth);
public List<DataSetResp> getDataSetsInheritAuth(User user, Long domainId) {
List<DataSetResp> dataSetResps = getDataSetList(new MetaFilter());
List<DataSetResp> inheritAuthFormDomain = getDataSetFilterByDomainAuth(dataSetResps, user);
Set<DataSetResp> dataSetRespSet = new HashSet<>(inheritAuthFormDomain);
List<DataSetResp> dataSetFilterByAuth = getDataSetFilterByAuth(dataSetResps, user);
dataSetRespSet.addAll(dataSetFilterByAuth);
if (domainId != null && domainId > 0) {
viewRespSet = viewRespSet.stream().filter(modelResp ->
dataSetRespSet = dataSetRespSet.stream().filter(modelResp ->
modelResp.getDomainId().equals(domainId)).collect(Collectors.toSet());
}
return viewRespSet.stream().sorted(Comparator.comparingLong(ViewResp::getId))
return dataSetRespSet.stream().sorted(Comparator.comparingLong(DataSetResp::getId))
.collect(Collectors.toList());
}
private List<ViewResp> getViewFilterByAuth(List<ViewResp> viewResps, User user) {
return viewResps.stream()
.filter(viewResp -> checkAdminPermission(user, viewResp))
private List<DataSetResp> getDataSetFilterByAuth(List<DataSetResp> dataSetResps, User user) {
return dataSetResps.stream()
.filter(dataSetResp -> checkAdminPermission(user, dataSetResp))
.collect(Collectors.toList());
}
private List<ViewResp> getViewFilterByDomainAuth(List<ViewResp> viewResps, User user) {
private List<DataSetResp> getDataSetFilterByDomainAuth(List<DataSetResp> dataSetResps, User user) {
Set<DomainResp> domainResps = domainService.getDomainAuthSet(user, AuthType.ADMIN);
if (CollectionUtils.isEmpty(domainResps)) {
return Lists.newArrayList();
}
Set<Long> domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toSet());
return viewResps.stream().filter(viewResp ->
domainIds.contains(viewResp.getDomainId())).collect(Collectors.toList());
return dataSetResps.stream().filter(dataSetResp ->
domainIds.contains(dataSetResp.getDomainId())).collect(Collectors.toList());
}
private ViewResp convert(ViewDO viewDO) {
ViewResp viewResp = new ViewResp();
BeanMapper.mapper(viewDO, viewResp);
viewResp.setViewDetail(JSONObject.parseObject(viewDO.getViewDetail(), ViewDetail.class));
if (viewDO.getQueryConfig() != null) {
viewResp.setQueryConfig(JSONObject.parseObject(viewDO.getQueryConfig(), QueryConfig.class));
private DataSetResp convert(DataSetDO dataSetDO) {
DataSetResp dataSetResp = new DataSetResp();
BeanMapper.mapper(dataSetDO, dataSetResp);
dataSetResp.setDataSetDetail(JSONObject.parseObject(dataSetDO.getDataSetDetail(), DataSetDetail.class));
if (dataSetDO.getQueryConfig() != null) {
dataSetResp.setQueryConfig(JSONObject.parseObject(dataSetDO.getQueryConfig(), QueryConfig.class));
}
viewResp.setAdmins(StringUtils.isBlank(viewDO.getAdmin())
? Lists.newArrayList() : Arrays.asList(viewDO.getAdmin().split(",")));
viewResp.setAdminOrgs(StringUtils.isBlank(viewDO.getAdminOrg())
? Lists.newArrayList() : Arrays.asList(viewDO.getAdminOrg().split(",")));
viewResp.setTypeEnum(TypeEnums.VIEW);
return viewResp;
dataSetResp.setAdmins(StringUtils.isBlank(dataSetDO.getAdmin())
? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdmin().split(",")));
dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg())
? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdminOrg().split(",")));
dataSetResp.setTypeEnum(TypeEnums.DATASET);
return dataSetResp;
}
private ViewDO convert(ViewReq viewReq) {
ViewDO viewDO = new ViewDO();
BeanMapper.mapper(viewReq, viewDO);
viewDO.setViewDetail(JSONObject.toJSONString(viewReq.getViewDetail()));
viewDO.setQueryConfig(JSONObject.toJSONString(viewReq.getQueryConfig()));
return viewDO;
private DataSetDO convert(DataSetReq dataSetReq) {
DataSetDO dataSetDO = new DataSetDO();
BeanMapper.mapper(dataSetReq, dataSetDO);
dataSetDO.setDataSetDetail(JSONObject.toJSONString(dataSetReq.getDataSetDetail()));
dataSetDO.setQueryConfig(JSONObject.toJSONString(dataSetReq.getQueryConfig()));
return dataSetDO;
}
public SemanticQueryReq convert(QueryViewReq queryViewReq) {
public SemanticQueryReq convert(QueryDataSetReq queryDataSetReq) {
SemanticQueryReq queryReq = new QueryStructReq();
if (StringUtils.isNotBlank(queryViewReq.getSql())) {
if (StringUtils.isNotBlank(queryDataSetReq.getSql())) {
queryReq = new QuerySqlReq();
}
BeanUtils.copyProperties(queryViewReq, queryReq);
BeanUtils.copyProperties(queryDataSetReq, queryReq);
return queryReq;
}
public static boolean checkAdminPermission(User user, ViewResp viewResp) {
List<String> admins = viewResp.getAdmins();
public static boolean checkAdminPermission(User user, DataSetResp dataSetResp) {
List<String> admins = dataSetResp.getAdmins();
if (user.isSuperAdmin()) {
return true;
}
String userName = user.getName();
return admins.contains(userName) || viewResp.getCreatedBy().equals(userName);
return admins.contains(userName) || dataSetResp.getCreatedBy().equals(userName);
}
@Override
public Map<Long, List<Long>> getModelIdToViewIds(List<Long> viewIds) {
public Map<Long, List<Long>> getModelIdToDataSetIds(List<Long> dataSetIds) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
metaFilter.setIds(viewIds);
List<ViewResp> viewList = viewSchemaCache.getIfPresent(metaFilter);
if (CollectionUtils.isEmpty(viewList)) {
viewList = getViewList(metaFilter);
viewSchemaCache.put(metaFilter, viewList);
metaFilter.setIds(dataSetIds);
List<DataSetResp> dataSetList = dataSetSchemaCache.getIfPresent(metaFilter);
if (CollectionUtils.isEmpty(dataSetList)) {
dataSetList = getDataSetList(metaFilter);
dataSetSchemaCache.put(metaFilter, dataSetList);
}
return viewList.stream()
return dataSetList.stream()
.flatMap(
viewResp -> viewResp.getAllModels().stream().map(modelId -> Pair.of(modelId, viewResp.getId())))
.collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList())));
dataSetResp -> dataSetResp.getAllModels().stream().map(modelId ->
Pair.of(modelId, dataSetResp.getId())))
.collect(Collectors.groupingBy(Pair::getLeft,
Collectors.mapping(Pair::getRight, Collectors.toList())));
}
private void conflictCheck(ViewResp viewResp) {
List<Long> allDimensionIds = viewResp.getAllDimensions();
List<Long> allMetricIds = viewResp.getAllMetrics();
private void conflictCheck(DataSetResp dataSetResp) {
List<Long> allDimensionIds = dataSetResp.getAllDimensions();
List<Long> allMetricIds = dataSetResp.getAllMetrics();
MetaFilter metaFilter = new MetaFilter();
if (!CollectionUtils.isEmpty(allDimensionIds)) {
metaFilter.setIds(allDimensionIds);

29
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View File

@@ -21,29 +21,23 @@ import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -51,6 +45,13 @@ import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@Slf4j
public class DimensionServiceImpl implements DimensionService {
@@ -66,7 +67,7 @@ public class DimensionServiceImpl implements DimensionService {
private ModelRelaService modelRelaService;
private ViewService viewService;
private DataSetService dataSetService;
@Autowired
private ApplicationEventPublisher eventPublisher;
@@ -77,13 +78,13 @@ public class DimensionServiceImpl implements DimensionService {
ChatGptHelper chatGptHelper,
DatabaseService databaseService,
ModelRelaService modelRelaService,
ViewService viewService) {
DataSetService dataSetService) {
this.modelService = modelService;
this.dimensionRepository = dimensionRepository;
this.chatGptHelper = chatGptHelper;
this.databaseService = databaseService;
this.modelRelaService = modelRelaService;
this.viewService = viewService;
this.dataSetService = dataSetService;
}
@Override
@@ -232,9 +233,9 @@ public class DimensionServiceImpl implements DimensionService {
if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
return filterByField(dimensionResps, metaFilter.getFieldsDepend());
}
if (metaFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(metaFilter.getViewId());
return DimensionConverter.filterByView(dimensionResps, viewResp);
if (metaFilter.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(metaFilter.getDataSetId());
return DimensionConverter.filterByDataSet(dimensionResps, dataSetResp);
}
return dimensionResps;
}

20
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DomainServiceImpl.java
View File

@@ -10,12 +10,12 @@ import com.tencent.supersonic.headless.api.pojo.request.DomainReq;
import com.tencent.supersonic.headless.api.pojo.request.DomainUpdateReq;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
import com.tencent.supersonic.headless.server.persistence.repository.DomainRepository;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.DomainConvert;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.util.Sets;
@@ -43,17 +43,17 @@ public class DomainServiceImpl implements DomainService {
private final DomainRepository domainRepository;
private final ModelService modelService;
private final UserService userService;
private final ViewService viewService;
private final DataSetService dataSetService;
public DomainServiceImpl(DomainRepository domainRepository,
@Lazy ModelService modelService,
UserService userService,
@Lazy ViewService viewService) {
@Lazy DataSetService dataSetService) {
this.domainRepository = domainRepository;
this.modelService = modelService;
this.userService = userService;
this.viewService = viewService;
this.dataSetService = dataSetService;
}
@Override
@@ -104,9 +104,9 @@ public class DomainServiceImpl implements DomainService {
List<Long> domainIds = modelResps.stream().map(ModelResp::getDomainId).collect(Collectors.toList());
domainWithAuthAll.addAll(getParentDomain(domainIds));
}
List<ViewResp> viewResps = viewService.getViews(user);
if (!CollectionUtils.isEmpty(viewResps)) {
List<Long> domainIds = viewResps.stream().map(ViewResp::getDomainId).collect(Collectors.toList());
List<DataSetResp> dataSetResps = dataSetService.getDataSets(user);
if (!CollectionUtils.isEmpty(dataSetResps)) {
List<Long> domainIds = dataSetResps.stream().map(DataSetResp::getDomainId).collect(Collectors.toList());
domainWithAuthAll.addAll(getParentDomain(domainIds));
}
return new ArrayList<>(domainWithAuthAll).stream()
@@ -125,7 +125,7 @@ public class DomainServiceImpl implements DomainService {
}
if (authTypeEnum.equals(AuthType.VISIBLE)) {
domainWithAuth = domainResps.stream()
.filter(domainResp -> checkViewerPermission(orgIds, user, domainResp))
.filter(domainResp -> checkDataSeterPermission(orgIds, user, domainResp))
.collect(Collectors.toList());
}
List<Long> domainIds = domainWithAuth.stream().map(DomainResp::getId)
@@ -253,7 +253,7 @@ public class DomainServiceImpl implements DomainService {
return false;
}
private boolean checkViewerPermission(Set<String> orgIds, User user, DomainResp domainDesc) {
private boolean checkDataSeterPermission(Set<String> orgIds, User user, DomainResp domainDesc) {
List<String> admins = domainDesc.getAdmins();
List<String> viewers = domainDesc.getViewers();
List<String> adminOrgs = domainDesc.getAdminOrgs();

28
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.server.service.KnowledgeService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@@ -21,10 +21,10 @@ import java.util.stream.Collectors;
@Slf4j
public class KnowledgeServiceImpl implements KnowledgeService {
private final ViewService viewService;
private final DataSetService dataSetService;
public KnowledgeServiceImpl(ViewService viewService) {
this.viewService = viewService;
public KnowledgeServiceImpl(DataSetService dataSetService) {
this.dataSetService = dataSetService;
}
@Override
@@ -69,25 +69,25 @@ public class KnowledgeServiceImpl implements KnowledgeService {
@Override
public List<S2Term> getTerms(String text) {
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>());
return HanlpHelper.getTerms(text, modelIdToViewIds);
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>());
return HanlpHelper.getTerms(text, modelIdToDataSetIds);
}
@Override
public List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> viewIds) {
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds));
return prefixSearchByModel(key, limit, modelIdToViewIds);
public List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> dataSetIds) {
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIds));
return prefixSearchByModel(key, limit, modelIdToDataSetIds);
}
public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
Map<Long, List<Long>> modelIdToViewIds) {
return SearchService.prefixSearch(key, limit, modelIdToViewIds);
Map<Long, List<Long>> modelIdToDataSetIds) {
return SearchService.prefixSearch(key, limit, modelIdToDataSetIds);
}
@Override
public List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> viewIds) {
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds));
return suffixSearchByModel(key, limit, modelIdToViewIds.keySet());
public List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> dataSetIds) {
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIds));
return suffixSearchByModel(key, limit, modelIdToDataSetIds.keySet());
}
public List<HanlpMapResult> suffixSearchByModel(String key, int limit, Set<Long> models) {

33
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java
View File

@@ -7,8 +7,14 @@ import com.tencent.supersonic.common.util.embedding.Retrieval;
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.MetaEmbeddingService;
import com.tencent.supersonic.headless.server.service.ViewService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -17,11 +23,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@@ -32,13 +33,13 @@ public class MetaEmbeddingServiceImpl implements MetaEmbeddingService {
private EmbeddingConfig embeddingConfig;
@Autowired
private ViewService viewService;
private DataSetService dataSetService;
@Override
public List<RetrieveQueryResult> retrieveQuery(List<Long> viewIds, RetrieveQuery retrieveQuery, int num) {
// viewIds->modelIds
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(viewIds);
Set<Long> allModels = modelIdToViewIds.keySet();
public List<RetrieveQueryResult> retrieveQuery(List<Long> dataSetIds, RetrieveQuery retrieveQuery, int num) {
// dataSetIds->modelIds
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(dataSetIds);
Set<Long> allModels = modelIdToDataSetIds.keySet();
if (CollectionUtils.isNotEmpty(allModels) && allModels.size() == 1) {
Map<String, String> filterCondition = new HashMap<>();
@@ -69,16 +70,16 @@ public class MetaEmbeddingServiceImpl implements MetaEmbeddingService {
}
return !allModels.contains(modelId);
});
//add viewId
//add dataSetId
retrievals = retrievals.stream().flatMap(retrieval -> {
Long modelId = Retrieval.getLongId(retrieval.getMetadata().get("modelId"));
List<Long> viewIdsByModelId = modelIdToViewIds.get(modelId);
if (!CollectionUtils.isEmpty(viewIdsByModelId)) {
List<Long> dataSetIdsByModelId = modelIdToDataSetIds.get(modelId);
if (!CollectionUtils.isEmpty(dataSetIdsByModelId)) {
Set<Retrieval> result = new HashSet<>();
for (Long viewId : viewIdsByModelId) {
for (Long dataSetId : dataSetIdsByModelId) {
Retrieval retrievalNew = new Retrieval();
BeanUtils.copyProperties(retrieval, retrievalNew);
retrievalNew.getMetadata().putIfAbsent("viewId", viewId + Constants.UNDERLINE);
retrievalNew.getMetadata().putIfAbsent("dataSetId", dataSetId + Constants.UNDERLINE);
result.add(retrievalNew);
}
return result.stream();

27
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -29,10 +29,10 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
@@ -43,13 +43,19 @@ import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -60,11 +66,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
@Slf4j
@@ -80,7 +81,7 @@ public class MetricServiceImpl implements MetricService {
private CollectService collectService;
private ViewService viewService;
private DataSetService dataSetService;
private ApplicationEventPublisher eventPublisher;
@@ -88,7 +89,7 @@ public class MetricServiceImpl implements MetricService {
ModelService modelService,
ChatGptHelper chatGptHelper,
CollectService collectService,
ViewService viewService,
DataSetService dataSetService,
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService) {
this.metricRepository = metricRepository;
@@ -96,7 +97,7 @@ public class MetricServiceImpl implements MetricService {
this.chatGptHelper = chatGptHelper;
this.eventPublisher = eventPublisher;
this.collectService = collectService;
this.viewService = viewService;
this.dataSetService = dataSetService;
this.dimensionService = dimensionService;
}
@@ -232,9 +233,9 @@ public class MetricServiceImpl implements MetricService {
if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
return filterByField(metricResps, metaFilter.getFieldsDepend());
}
if (metaFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(metaFilter.getViewId());
return MetricConverter.filterByView(metricResps, viewResp);
if (metaFilter.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(metaFilter.getDataSetId());
return MetricConverter.filterByDataSet(metricResps, dataSetResp);
}
return metricResps;
}

20
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -25,7 +25,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;
@@ -37,7 +37,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.ModelConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
@@ -76,7 +76,7 @@ public class ModelServiceImpl implements ModelService {
private UserService userService;
private ViewService viewService;
private DataSetService dataSetService;
private DateInfoRepository dateInfoRepository;
@@ -86,7 +86,7 @@ public class ModelServiceImpl implements ModelService {
@Lazy MetricService metricService,
DomainService domainService,
UserService userService,
ViewService viewService,
DataSetService dataSetService,
DateInfoRepository dateInfoRepository) {
this.modelRepository = modelRepository;
this.databaseService = databaseService;
@@ -94,7 +94,7 @@ public class ModelServiceImpl implements ModelService {
this.metricService = metricService;
this.domainService = domainService;
this.userService = userService;
this.viewService = viewService;
this.dataSetService = dataSetService;
this.dateInfoRepository = dateInfoRepository;
}
@@ -145,9 +145,9 @@ public class ModelServiceImpl implements ModelService {
ModelFilter modelFilter = new ModelFilter();
BeanUtils.copyProperties(metaFilter, modelFilter);
List<ModelResp> modelResps = ModelConverter.convertList(modelRepository.getModelList(modelFilter));
if (modelFilter.getViewId() != null) {
ViewResp viewResp = viewService.getView(modelFilter.getViewId());
return modelResps.stream().filter(modelResp -> viewResp.getAllModels().contains(modelResp.getId()))
if (modelFilter.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(modelFilter.getDataSetId());
return modelResps.stream().filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId()))
.collect(Collectors.toList());
}
return modelResps;
@@ -329,7 +329,7 @@ public class ModelServiceImpl implements ModelService {
}
if (authTypeEnum.equals(AuthType.VISIBLE)) {
modelWithAuth = modelResps.stream()
.filter(domainResp -> checkViewerPermission(orgIds, user, domainResp))
.filter(domainResp -> checkDataSeterPermission(orgIds, user, domainResp))
.collect(Collectors.toList());
}
return modelWithAuth;
@@ -456,7 +456,7 @@ public class ModelServiceImpl implements ModelService {
return false;
}
public static boolean checkViewerPermission(Set<String> orgIds, User user, ModelResp modelResp) {
public static boolean checkDataSeterPermission(Set<String> orgIds, User user, ModelResp modelResp) {
if (checkAdminPermission(orgIds, user, modelResp)) {
return true;
}

15
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -51,17 +51,18 @@ import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.utils.TagReqConverter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
@Service
@@ -176,7 +177,7 @@ public class QueryServiceImpl implements QueryService {
queryStatement.setQueryParam(queryParam);
queryStatement.setIsS2SQL(false);
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setViewId(queryStructReq.getViewId());
queryStatement.setDataSetId(queryStructReq.getDataSetId());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
SemanticModel semanticModel = semanticSchemaManager.getSemanticModel(semanticSchemaResp);
queryStatement.setSemanticModel(semanticModel);
@@ -216,7 +217,7 @@ public class QueryServiceImpl implements QueryService {
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
schemaFilterReq.setViewId(semanticQueryReq.getViewId());
schemaFilterReq.setDataSetId(semanticQueryReq.getDataSetId());
schemaFilterReq.setModelIds(semanticQueryReq.getModelIds());
return schemaFilterReq;
}

146
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaItemQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
@@ -30,8 +30,8 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -41,7 +41,7 @@ import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.TagService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.StatUtils;
@@ -68,7 +68,7 @@ public class SchemaServiceImpl implements SchemaService {
protected final Cache<String, List<ItemUseResp>> itemUseCache =
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
protected final Cache<ViewFilterReq, List<ViewSchemaResp>> viewSchemaCache =
protected final Cache<DataSetFilterReq, List<DataSetSchemaResp>> dataSetSchemaCache =
CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build();
protected final Cache<SchemaFilterReq, SemanticSchemaResp> semanticSchemaCache =
@@ -79,7 +79,7 @@ public class SchemaServiceImpl implements SchemaService {
private final DimensionService dimensionService;
private final MetricService metricService;
private final DomainService domainService;
private final ViewService viewService;
private final DataSetService dataSetService;
private final ModelRelaService modelRelaService;
private final TagService tagService;
@@ -87,14 +87,14 @@ public class SchemaServiceImpl implements SchemaService {
DimensionService dimensionService,
MetricService metricService,
DomainService domainService,
ViewService viewService,
DataSetService dataSetService,
ModelRelaService modelRelaService,
StatUtils statUtils, TagService tagService) {
this.modelService = modelService;
this.dimensionService = dimensionService;
this.metricService = metricService;
this.domainService = domainService;
this.viewService = viewService;
this.dataSetService = dataSetService;
this.modelRelaService = modelRelaService;
this.statUtils = statUtils;
this.tagService = tagService;
@@ -102,30 +102,30 @@ public class SchemaServiceImpl implements SchemaService {
@SneakyThrows
@Override
public List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter) {
List<ViewSchemaResp> viewList = viewSchemaCache.getIfPresent(filter);
if (CollectionUtils.isEmpty(viewList)) {
viewList = buildViewSchema(filter);
viewSchemaCache.put(filter, viewList);
public List<DataSetSchemaResp> fetchDataSetSchema(DataSetFilterReq filter) {
List<DataSetSchemaResp> dataSetList = dataSetSchemaCache.getIfPresent(filter);
if (CollectionUtils.isEmpty(dataSetList)) {
dataSetList = buildDataSetSchema(filter);
dataSetSchemaCache.put(filter, dataSetList);
}
return viewList;
return dataSetList;
}
public ViewSchemaResp fetchViewSchema(Long viewId) {
if (viewId == null) {
public DataSetSchemaResp fetchDataSetSchema(Long dataSetId) {
if (dataSetId == null) {
return null;
}
return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null);
return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream().findFirst().orElse(null);
}
public List<ViewSchemaResp> buildViewSchema(ViewFilterReq filter) {
List<ViewSchemaResp> viewSchemaResps = new ArrayList<>();
List<Long> viewIds = filter.getViewIds();
public List<DataSetSchemaResp> buildDataSetSchema(DataSetFilterReq filter) {
List<DataSetSchemaResp> dataSetSchemaResps = new ArrayList<>();
List<Long> dataSetIds = filter.getDataSetIds();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
metaFilter.setIds(viewIds);
List<ViewResp> viewResps = viewService.getViewList(metaFilter);
List<Long> modelIds = viewResps.stream().map(ViewResp::getAllModels)
metaFilter.setIds(dataSetIds);
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(metaFilter);
List<Long> modelIds = dataSetResps.stream().map(DataSetResp::getAllModels)
.flatMap(Collection::stream).collect(Collectors.toList());
metaFilter.setModelIds(modelIds);
metaFilter.setIds(Lists.newArrayList());
@@ -133,26 +133,26 @@ public class SchemaServiceImpl implements SchemaService {
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
metaFilter.setIds(modelIds);
List<ModelResp> modelResps = modelService.getModelList(metaFilter);
Map<Long, ViewResp> viewRespMap = getViewMap(viewResps);
for (Long viewId : viewRespMap.keySet()) {
ViewResp viewResp = viewRespMap.get(viewId);
if (viewResp == null || !StatusEnum.ONLINE.getCode().equals(viewResp.getStatus())) {
Map<Long, DataSetResp> dataSetRespMap = getDataSetMap(dataSetResps);
for (Long dataSetId : dataSetRespMap.keySet()) {
DataSetResp dataSetResp = dataSetRespMap.get(dataSetId);
if (dataSetResp == null || !StatusEnum.ONLINE.getCode().equals(dataSetResp.getStatus())) {
continue;
}
List<MetricSchemaResp> metricSchemaResps = MetricConverter.filterByView(metricResps, viewResp)
List<MetricSchemaResp> metricSchemaResps = MetricConverter.filterByDataSet(metricResps, dataSetResp)
.stream().map(this::convert).collect(Collectors.toList());
List<DimSchemaResp> dimSchemaResps = DimensionConverter.filterByView(dimensionResps, viewResp)
List<DimSchemaResp> dimSchemaResps = DimensionConverter.filterByDataSet(dimensionResps, dataSetResp)
.stream().map(this::convert).collect(Collectors.toList());
ViewSchemaResp viewSchemaResp = new ViewSchemaResp();
BeanUtils.copyProperties(viewResp, viewSchemaResp);
viewSchemaResp.setDimensions(dimSchemaResps);
viewSchemaResp.setMetrics(metricSchemaResps);
viewSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
viewResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));
viewSchemaResps.add(viewSchemaResp);
DataSetSchemaResp dataSetSchemaResp = new DataSetSchemaResp();
BeanUtils.copyProperties(dataSetResp, dataSetSchemaResp);
dataSetSchemaResp.setDimensions(dimSchemaResps);
dataSetSchemaResp.setMetrics(metricSchemaResps);
dataSetSchemaResp.setModelResps(modelResps.stream().filter(modelResp ->
dataSetResp.getAllModels().contains(modelResp.getId())).collect(Collectors.toList()));
dataSetSchemaResps.add(dataSetSchemaResp);
}
fillStaticInfo(viewSchemaResps);
return viewSchemaResps;
fillStaticInfo(dataSetSchemaResps);
return dataSetSchemaResps;
}
public List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds) {
@@ -191,7 +191,7 @@ public class SchemaServiceImpl implements SchemaService {
}
private void fillCnt(List<ViewSchemaResp> viewSchemaResps, List<ItemUseResp> statInfos) {
private void fillCnt(List<DataSetSchemaResp> dataSetSchemaResps, List<ItemUseResp> statInfos) {
Map<String, ItemUseResp> typeIdAndStatPair = statInfos.stream()
.collect(Collectors.toMap(
@@ -199,15 +199,15 @@ public class SchemaServiceImpl implements SchemaService {
itemUseInfo -> itemUseInfo,
(item1, item2) -> item1));
log.debug("typeIdAndStatPair:{}", typeIdAndStatPair);
for (ViewSchemaResp viewSchemaResp : viewSchemaResps) {
fillDimCnt(viewSchemaResp, typeIdAndStatPair);
fillMetricCnt(viewSchemaResp, typeIdAndStatPair);
for (DataSetSchemaResp dataSetSchemaResp : dataSetSchemaResps) {
fillDimCnt(dataSetSchemaResp, typeIdAndStatPair);
fillMetricCnt(dataSetSchemaResp, typeIdAndStatPair);
}
}
private void fillMetricCnt(ViewSchemaResp viewSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<MetricSchemaResp> metrics = viewSchemaResp.getMetrics();
if (CollectionUtils.isEmpty(viewSchemaResp.getMetrics())) {
private void fillMetricCnt(DataSetSchemaResp dataSetSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<MetricSchemaResp> metrics = dataSetSchemaResp.getMetrics();
if (CollectionUtils.isEmpty(dataSetSchemaResp.getMetrics())) {
return;
}
@@ -220,12 +220,12 @@ public class SchemaServiceImpl implements SchemaService {
}
});
}
viewSchemaResp.setMetrics(metrics);
dataSetSchemaResp.setMetrics(metrics);
}
private void fillDimCnt(ViewSchemaResp viewSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<DimSchemaResp> dimensions = viewSchemaResp.getDimensions();
if (CollectionUtils.isEmpty(viewSchemaResp.getDimensions())) {
private void fillDimCnt(DataSetSchemaResp dataSetSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
List<DimSchemaResp> dimensions = dataSetSchemaResp.getDimensions();
if (CollectionUtils.isEmpty(dataSetSchemaResp.getDimensions())) {
return;
}
if (!CollectionUtils.isEmpty(dimensions)) {
@@ -237,7 +237,7 @@ public class SchemaServiceImpl implements SchemaService {
}
});
}
viewSchemaResp.setDimensions(dimensions);
dataSetSchemaResp.setDimensions(dimensions);
}
@Override
@@ -273,20 +273,20 @@ public class SchemaServiceImpl implements SchemaService {
}
@Override
public List<ViewResp> getViewList(Long domainId) {
public List<DataSetResp> getDataSetList(Long domainId) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDomainId(domainId);
return viewService.getViewList(metaFilter);
return dataSetService.getDataSetList(metaFilter);
}
public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) {
SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp();
semanticSchemaResp.setViewId(schemaFilterReq.getViewId());
semanticSchemaResp.setDataSetId(schemaFilterReq.getDataSetId());
semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds());
if (schemaFilterReq.getViewId() != null) {
ViewSchemaResp viewSchemaResp = fetchViewSchema(schemaFilterReq.getViewId());
BeanUtils.copyProperties(viewSchemaResp, semanticSchemaResp);
List<Long> modelIds = viewSchemaResp.getAllModels();
if (schemaFilterReq.getDataSetId() != null) {
DataSetSchemaResp dataSetSchemaResp = fetchDataSetSchema(schemaFilterReq.getDataSetId());
BeanUtils.copyProperties(dataSetSchemaResp, semanticSchemaResp);
List<Long> modelIds = dataSetSchemaResp.getAllModels();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(modelIds);
List<ModelResp> modelList = modelService.getModelList(metaFilter);
@@ -343,7 +343,7 @@ public class SchemaServiceImpl implements SchemaService {
}
@Override
public List<ItemResp> getDomainViewTree() {
public List<ItemResp> getDomainDataSetTree() {
List<DomainResp> domainResps = domainService.getDomainList();
List<ItemResp> itemResps = domainResps.stream().map(domain ->
new ItemResp(domain.getId(), domain.getParentId(), domain.getName(), TypeEnums.DOMAIN))
@@ -357,36 +357,36 @@ public class SchemaServiceImpl implements SchemaService {
}
parentItem.getChildren().add(itemResp);
}
List<ViewResp> viewResps = viewService.getViewList(new MetaFilter());
for (ViewResp viewResp : viewResps) {
ItemResp itemResp = itemRespMap.get(viewResp.getDomainId());
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(new MetaFilter());
for (DataSetResp dataSetResp : dataSetResps) {
ItemResp itemResp = itemRespMap.get(dataSetResp.getDomainId());
if (itemResp != null) {
ItemResp view = new ItemResp(viewResp.getId(), viewResp.getDomainId(),
viewResp.getName(), TypeEnums.VIEW);
itemResp.getChildren().add(view);
ItemResp dataSet = new ItemResp(dataSetResp.getId(), dataSetResp.getDomainId(),
dataSetResp.getName(), TypeEnums.DATASET);
itemResp.getChildren().add(dataSet);
}
}
return itemResps.stream().filter(itemResp -> itemResp.getParentId() == 0)
.collect(Collectors.toList());
}
private void fillStaticInfo(List<ViewSchemaResp> viewSchemaResps) {
List<Long> viewIds = viewSchemaResps.stream()
.map(ViewSchemaResp::getId).collect(Collectors.toList());
private void fillStaticInfo(List<DataSetSchemaResp> dataSetSchemaResps) {
List<Long> dataSetIds = dataSetSchemaResps.stream()
.map(DataSetSchemaResp::getId).collect(Collectors.toList());
ItemUseReq itemUseReq = new ItemUseReq();
itemUseReq.setModelIds(viewIds);
itemUseReq.setModelIds(dataSetIds);
List<ItemUseResp> statInfos = getStatInfo(itemUseReq);
log.debug("statInfos:{}", statInfos);
fillCnt(viewSchemaResps, statInfos);
fillCnt(dataSetSchemaResps, statInfos);
}
private Map<Long, ViewResp> getViewMap(List<ViewResp> viewResps) {
if (CollectionUtils.isEmpty(viewResps)) {
private Map<Long, DataSetResp> getDataSetMap(List<DataSetResp> dataSetResps) {
if (CollectionUtils.isEmpty(dataSetResps)) {
return new HashMap<>();
}
return viewResps.stream().collect(
Collectors.toMap(ViewResp::getId, viewResp -> viewResp));
return dataSetResps.stream().collect(
Collectors.toMap(DataSetResp::getId, dataSetResp -> dataSetResp));
}
private DimSchemaResp convert(DimensionResp dimensionResp) {

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java
View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
@@ -79,10 +79,10 @@ public class DimensionConverter {
return dimensionResp;
}
public static List<DimensionResp> filterByView(List<DimensionResp> dimensionResps, ViewResp viewResp) {
public static List<DimensionResp> filterByDataSet(List<DimensionResp> dimensionResps, DataSetResp dataSetResp) {
return dimensionResps.stream().filter(dimensionResp ->
viewResp.getAllDimensions().contains(dimensionResp.getId())
|| viewResp.getAllIncludeAllModels().contains(dimensionResp.getModelId()))
dataSetResp.getAllDimensions().contains(dimensionResp.getId())
|| dataSetResp.getAllIncludeAllModels().contains(dimensionResp.getModelId()))
.collect(Collectors.toList());
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
View File

@@ -14,7 +14,7 @@ import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import org.springframework.beans.BeanUtils;
@@ -99,10 +99,10 @@ public class MetricConverter {
return metricResp;
}
public static List<MetricResp> filterByView(List<MetricResp> metricResps, ViewResp viewResp) {
public static List<MetricResp> filterByDataSet(List<MetricResp> metricResps, DataSetResp dataSetResp) {
return metricResps.stream().filter(metricResp ->
viewResp.getAllMetrics().contains(metricResp.getId())
|| viewResp.getAllIncludeAllModels().contains(metricResp.getModelId()))
dataSetResp.getAllMetrics().contains(metricResp.getId())
|| dataSetResp.getAllIncludeAllModels().contains(metricResp.getModelId()))
.collect(Collectors.toList());
}

31
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -16,7 +16,6 @@ import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
@@ -27,7 +26,17 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -37,14 +46,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
@@ -103,7 +104,7 @@ public class QueryReqConverter {
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//4.build ParseSqlReq
ViewQueryParam result = new ViewQueryParam();
DataSetQueryParam result = new DataSetQueryParam();
BeanUtils.copyProperties(querySQLReq, result);
result.setTables(tables);
@@ -118,17 +119,17 @@ public class QueryReqConverter {
//6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setViewId(querySQLReq.getViewId());
queryStructReq.setDataSetId(querySQLReq.getDataSetId());
queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryParam queryParam = new QueryParam();
convert(queryStructReq, queryParam);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryParam(queryParam);
queryStatement.setViewQueryParam(result);
queryStatement.setDataSetQueryParam(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setViewId(querySQLReq.getViewId());
queryStatement.setDataSetId(querySQLReq.getDataSetId());
queryStatement.setEnableLimitWrapper(limitWrapper);
return queryStatement;
@@ -225,7 +226,7 @@ public class QueryReqConverter {
public void correctTableName(QuerySqlReq querySqlReq) {
String sql = querySqlReq.getSql();
sql = SqlReplaceHelper.replaceTable(sql,
Constants.TABLE_PREFIX + querySqlReq.getViewId());
Constants.TABLE_PREFIX + querySqlReq.getDataSetId());
querySqlReq.setSql(sql);
}
@@ -239,7 +240,7 @@ public class QueryReqConverter {
}
private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
ViewQueryParam viewQueryParam) {
DataSetQueryParam viewQueryParam) {
String sql = viewQueryParam.getSql();
for (MetricTable metricTable : viewQueryParam.getTables()) {
List<String> measures = new ArrayList<>();

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -77,7 +77,7 @@ public class QueryStructUtils {
private List<Long> getDimensionIds(QueryStructReq queryStructReq) {
List<Long> dimensionIds = new ArrayList<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setViewId(queryStructReq.getViewId());
metaFilter.setDataSetId(queryStructReq.getDataSetId());
List<DimensionResp> dimensions = catalog.getDimensions(metaFilter);
Map<String, List<DimensionResp>> pair = dimensions.stream()
.collect(Collectors.groupingBy(DimensionResp::getBizName));
@@ -99,7 +99,7 @@ public class QueryStructUtils {
private List<Long> getMetricIds(QueryStructReq queryStructCmd) {
List<Long> metricIds = new ArrayList<>();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setViewId(queryStructCmd.getViewId());
metaFilter.setDataSetId(queryStructCmd.getDataSetId());
List<MetricResp> metrics = catalog.getMetrics(metaFilter);
Map<String, List<MetricResp>> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName));
for (Aggregator agg : queryStructCmd.getAggregators()) {

24
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -20,18 +20,18 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@@ -41,17 +41,13 @@ public class StatUtils {
private static final TransmittableThreadLocal<QueryStat> STATS = new TransmittableThreadLocal<>();
private final StatRepository statRepository;
private final SqlFilterUtils sqlFilterUtils;
private final ModelService modelService;
private final ObjectMapper objectMapper = new ObjectMapper();
public StatUtils(StatRepository statRepository,
SqlFilterUtils sqlFilterUtils,
ModelService modelService) {
SqlFilterUtils sqlFilterUtils) {
this.statRepository = statRepository;
this.sqlFilterUtils = sqlFilterUtils;
this.modelService = modelService;
}
public static QueryStat get() {
@@ -112,7 +108,7 @@ public class StatUtils {
try {
queryStatInfo.setTraceId(traceId)
.setViewId(queryTagReq.getViewId())
.setDataSetId(queryTagReq.getDataSetId())
.setUser(user)
.setQueryType(QueryType.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
@@ -150,7 +146,7 @@ public class StatUtils {
try {
queryStatInfo.setTraceId("")
.setUser(userName)
.setViewId(querySqlReq.getViewId())
.setDataSetId(querySqlReq.getDataSetId())
.setQueryType(QueryType.SQL.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQuerySqlCmd(querySqlReq.toString())
@@ -180,7 +176,7 @@ public class StatUtils {
try {
queryStatInfo.setTraceId(traceId)
.setViewId(queryStructReq.getViewId())
.setDataSetId(queryStructReq.getDataSetId())
.setUser(user)
.setQueryType(QueryType.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())

17
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java
View File

@@ -12,11 +12,8 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
@@ -24,6 +21,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Component
@Slf4j
public class TagReqConverter {
@@ -61,7 +62,7 @@ public class TagReqConverter {
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//.build ParseSqlReq
ViewQueryParam result = new ViewQueryParam();
DataSetQueryParam result = new DataSetQueryParam();
BeanUtils.copyProperties(querySqlReq, result);
result.setTables(tables);
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
@@ -72,15 +73,15 @@ public class TagReqConverter {
}
//.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
queryStructReq.setViewId(querySqlReq.getViewId());
queryStructReq.setDataSetId(querySqlReq.getDataSetId());
queryStructReq.setQueryType(QueryType.TAG);
QueryParam queryParam = new QueryParam();
convert(queryTagReq, queryParam);
queryStatement.setQueryParam(queryParam);
queryStatement.setViewQueryParam(result);
queryStatement.setDataSetQueryParam(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setViewId(queryTagReq.getViewId());
queryStatement.setDataSetId(queryTagReq.getDataSetId());
queryStatement.setEnableLimitWrapper(limitWrapper);
}
return queryStatement;

8
headless/server/src/main/resources/mapper/StatMapper.xml
View File

@@ -9,7 +9,7 @@
<id column="id" property="id"/>
<result column="trace_id" property="traceId"/>
<result column="model_id" property="modelId"/>
<result column="view_id" property="viewId"/>
<result column="data_set_id" property="dataSetId"/>
<result column="user" property="user"/>
<result column="created_at" property="createdAt"/>
<result column="query_type" property="queryType"/>
@@ -43,13 +43,13 @@
<insert id="createRecord">
insert into s2_query_stat_info
(
trace_id, model_id, view_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
trace_id, model_id, data_set_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics, select_cols, agg_cols, filter_cols, group_by_cols,
order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key, query_opt_mode
)
values
(
#{traceId}, #{modelId}, #{viewId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
#{traceId}, #{modelId}, #{dataSetId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
#{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate}, #{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols},
#{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey}, #{resultCacheKey}, #{queryOptMode}
)
@@ -61,7 +61,7 @@
from s2_query_stat_info
<where>
<if test="startTime != null">
and start_time >= #{startTime}
and start_time >= #{startTime}String.valueOf(queryFilter.getValue())
</if>
<if test="modelId != null">
and model_id = #{modelId}

4
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
View File

@@ -23,7 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.service.impl.ViewServiceImpl;
import com.tencent.supersonic.headless.server.service.impl.DataSetImpl;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import java.util.HashMap;
import org.junit.jupiter.api.Assertions;
@@ -66,7 +66,7 @@ public class MetricServiceImplTest {
ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class);
CollectService collectService = Mockito.mock(CollectService.class);
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
ViewService viewService = Mockito.mock(ViewServiceImpl.class);
DataSetService viewService = Mockito.mock(DataSetImpl.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
return new MetricServiceImpl(metricRepository, modelService, chatGptHelper, collectService, viewService,
eventPublisher, dimensionService);

2
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/ModelServiceImplTest.java
View File

@@ -75,7 +75,7 @@ class ModelServiceImplTest {
DomainService domainService = Mockito.mock(DomainService.class);
UserService userService = Mockito.mock(UserService.class);
DateInfoRepository dateInfoRepository = Mockito.mock(DateInfoRepository.class);
ViewService viewService = Mockito.mock(ViewService.class);
DataSetService viewService = Mockito.mock(DataSetService.class);
return new ModelServiceImpl(modelRepository, databaseService,
dimensionService, metricService, domainService, userService,
viewService, dateInfoRepository);