[improvement][project] supersonic 0.7.0 version backend update (#24)

* [improvement][project] supersonic 0.7.0 version backend update

* [improvement][project] supersonic 0.7.0 version backend update

* [improvement][project] supersonic 0.7.0 version readme update

---------

Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
SunDean
2023-08-05 22:17:56 +08:00
committed by GitHub
parent 6951eada9d
commit aa0a100a85
184 changed files with 2609 additions and 1238 deletions

View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.api.model.request;
import com.tencent.supersonic.common.pojo.PageBaseReq;
import lombok.Data;
import java.util.List;
@Data
public class PageSchemaItemReq extends PageBaseReq {
@@ -10,7 +11,7 @@ public class PageSchemaItemReq extends PageBaseReq {
private String name;
private String bizName;
private String createdBy;
private Long domainId;
private List<Long> domainIds;
private Integer sensitiveLevel;
private Integer status;
}

View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.semantic.api.model.response;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -10,7 +11,6 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
public class DomainSchemaResp extends DomainResp {
private List<String> entityNames;
private List<MetricSchemaResp> metrics;
private List<DimSchemaResp> dimensions;

View File

@@ -15,5 +15,6 @@ public class MetricReq {
private String where;
private Long limit;
private List<ColumnOrder> order;
private boolean nativeQuery = false;
}

View File

@@ -109,6 +109,7 @@ public class DimensionServiceImpl implements DimensionService {
public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq) {
DimensionFilter dimensionFilter = new DimensionFilter();
BeanUtils.copyProperties(pageDimensionReq, dimensionFilter);
dimensionFilter.setDomainIds(pageDimensionReq.getDomainIds());
PageInfo<DimensionDO> dimensionDOPageInfo = PageHelper.startPage(pageDimensionReq.getCurrent(),
pageDimensionReq.getPageSize())
.doSelectPageInfo(() -> queryDimension(dimensionFilter));

View File

@@ -2,7 +2,9 @@ package com.tencent.supersonic.semantic.model.application;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.request.DomainReq;
@@ -24,12 +26,7 @@ import com.tencent.supersonic.semantic.model.domain.pojo.Domain;
import com.tencent.supersonic.semantic.model.domain.repository.DomainRepository;
import com.tencent.supersonic.semantic.model.domain.utils.DomainConvert;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@@ -47,14 +44,17 @@ public class DomainServiceImpl implements DomainService {
private final MetricService metricService;
private final DimensionService dimensionService;
private final DatasourceService datasourceService;
private final UserService userService;
public DomainServiceImpl(DomainRepository domainRepository, @Lazy MetricService metricService,
@Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService) {
@Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService,
UserService userService) {
this.domainRepository = domainRepository;
this.metricService = metricService;
this.dimensionService = dimensionService;
this.datasourceService = datasourceService;
this.userService = userService;
}
@@ -130,7 +130,7 @@ public class DomainServiceImpl implements DomainService {
@Override
public List<DomainResp> getDomainListForAdmin(String userName) {
List<DomainDO> domainDOS = domainRepository.getDomainList();
List<String> orgIds = Lists.newArrayList();
Set<String> orgIds = Sets.newHashSet();
log.info("orgIds:{},userName:{}", orgIds, userName);
Map<Long, List<MetricResp>> metricDomainMap = metricService.getMetrics().stream()
.collect(Collectors.groupingBy(MetricResp::getDomainId));
@@ -144,7 +144,7 @@ public class DomainServiceImpl implements DomainService {
@Override
public List<DomainResp> getDomainListForViewer(String userName) {
List<DomainDO> domainDOS = domainRepository.getDomainList();
List<String> orgIds = Lists.newArrayList();
Set<String> orgIds = Sets.newHashSet();
log.info("orgIds:{},userName:{}", orgIds, userName);
return convertList(domainDOS, new HashMap<>(), new HashMap<>()).stream()
.filter(domainDesc -> checkViewerPermission(orgIds, userName, domainDesc))
@@ -165,7 +165,7 @@ public class DomainServiceImpl implements DomainService {
return "";
}
Map<Long, String> map = getDomainFullPathMap();
return map.containsKey(domainId) ? map.get(domainId) : "";
return map.getOrDefault(domainId, "");
}
@Override
@@ -201,6 +201,32 @@ public class DomainServiceImpl implements DomainService {
return getDomainList().stream().collect(Collectors.toMap(DomainResp::getId, a -> a, (k1, k2) -> k1));
}
@Override
public Set<DomainResp> getDomainChildren(List<Long> domainIds) {
Set<DomainResp> childDomains = new HashSet<>();
if (CollectionUtils.isEmpty(domainIds)) {
return childDomains;
}
Map<Long, DomainResp> allDomainMap = getDomainMap();
for (Long domainId : domainIds) {
DomainResp domain = allDomainMap.get(domainId);
if (domain != null) {
childDomains.add(domain);
Queue<DomainResp> queue = new LinkedList<>();
queue.add(domain);
while (!queue.isEmpty()) {
DomainResp currentDomain = queue.poll();
for (DomainResp child : allDomainMap.values()) {
if (Objects.equals(child.getParentId(), currentDomain.getId())) {
childDomains.add(child);
queue.add(child);
}
}
}
}
}
return childDomains;
}
public Map<Long, String> getDomainFullPathMap() {
Map<Long, String> domainFullPathMap = new HashMap<>();
@@ -286,13 +312,16 @@ public class DomainServiceImpl implements DomainService {
}
private boolean checkAdminPermission(List<String> orgIds, String userName, DomainResp domainDesc) {
private boolean checkAdminPermission(Set<String> orgIds, String userName, DomainResp domainDesc) {
List<String> admins = domainDesc.getAdmins();
List<String> adminOrgs = domainDesc.getAdminOrgs();
if (admins.contains(userName) || domainDesc.getCreatedBy().equals(userName)) {
return true;
}
if (CollectionUtils.isEmpty(adminOrgs)) {
return false;
}
for (String orgId : orgIds) {
if (adminOrgs.contains(orgId)) {
return true;
@@ -301,7 +330,7 @@ public class DomainServiceImpl implements DomainService {
return false;
}
private boolean checkViewerPermission(List<String> orgIds, String userName, DomainResp domainDesc) {
private boolean checkViewerPermission(Set<String> orgIds, String userName, DomainResp domainDesc) {
if (domainDesc.getIsOpen() == 1) {
return true;
}
@@ -312,6 +341,9 @@ public class DomainServiceImpl implements DomainService {
if (admins.contains(userName) || viewers.contains(userName) || domainDesc.getCreatedBy().equals(userName)) {
return true;
}
if (CollectionUtils.isEmpty(adminOrgs) && CollectionUtils.isEmpty(viewOrgs)) {
return false;
}
for (String orgId : orgIds) {
if (adminOrgs.contains(orgId)) {
return true;

View File

@@ -102,6 +102,9 @@ public class MetricServiceImpl implements MetricService {
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq) {
MetricFilter metricFilter = new MetricFilter();
BeanUtils.copyProperties(pageMetricReq, metricFilter);
Set<DomainResp> domainResps = domainService.getDomainChildren(pageMetricReq.getDomainIds());
List<Long> domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList());
metricFilter.setDomainIds(domainIds);
PageInfo<MetricDO> metricDOPageInfo = PageHelper.startPage(pageMetricReq.getCurrent(),
pageMetricReq.getPageSize())
.doSelectPageInfo(() -> queryMetric(metricFilter));

View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp;
import java.util.List;
import java.util.Map;
import java.util.Set;
public interface DomainService {
@@ -36,6 +37,8 @@ public interface DomainService {
List<DomainResp> getDomainListForViewer(String userName);
Set<DomainResp> getDomainChildren(List<Long> domainId);
List<DomainSchemaResp> fetchDomainSchema(DomainSchemaFilterReq filter, User user);
}

View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.semantic.model.domain.pojo;
import lombok.Data;
import java.util.List;
@Data
@@ -14,7 +15,7 @@ public class MetaFilter {
private String createdBy;
private Long domainId;
private List<Long> domainIds;
private Integer sensitiveLevel;

View File

@@ -4,6 +4,7 @@ package com.tencent.supersonic.semantic.model.infrastructure.mapper;
import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO;
import java.util.List;
import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
import org.apache.ibatis.annotations.Mapper;
@@ -14,4 +15,6 @@ public interface MetricDOCustomMapper {
void batchUpdate(List<MetricDO> metricDOS);
List<MetricDO> query(MetricFilter metricFilter);
}

View File

@@ -7,6 +7,7 @@ import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOCustomMapper;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOMapper;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
@@ -97,8 +98,8 @@ public class DimensionRepositoryImpl implements DimensionRepository {
if (dimensionFilter.getCreatedBy() != null) {
dimensionDOExample.getOredCriteria().get(0).andCreatedByEqualTo(dimensionFilter.getCreatedBy());
}
if (dimensionFilter.getDomainId() != null) {
dimensionDOExample.getOredCriteria().get(0).andDomainIdEqualTo(dimensionFilter.getDomainId());
if (CollectionUtils.isNotEmpty(dimensionFilter.getDomainIds())) {
dimensionDOExample.getOredCriteria().get(0).andDomainIdIn(dimensionFilter.getDomainIds());
}
if (dimensionFilter.getSensitiveLevel() != null) {
dimensionDOExample.getOredCriteria().get(0).andSensitiveLevelEqualTo(dimensionFilter.getSensitiveLevel());

View File

@@ -74,33 +74,7 @@ public class MetricRepositoryImpl implements MetricRepository {
@Override
public List<MetricDO> getMetric(MetricFilter metricFilter) {
MetricDOExample metricDOExample = new MetricDOExample();
metricDOExample.createCriteria();
if (metricFilter.getId() != null) {
metricDOExample.getOredCriteria().get(0).andIdEqualTo(metricFilter.getId());
}
if (metricFilter.getName() != null) {
metricDOExample.getOredCriteria().get(0).andNameLike("%" + metricFilter.getName() + "%");
}
if (metricFilter.getBizName() != null) {
metricDOExample.getOredCriteria().get(0).andBizNameLike("%" + metricFilter.getBizName() + "%");
}
if (metricFilter.getCreatedBy() != null) {
metricDOExample.getOredCriteria().get(0).andCreatedByEqualTo(metricFilter.getCreatedBy());
}
if (metricFilter.getDomainId() != null) {
metricDOExample.getOredCriteria().get(0).andDomainIdEqualTo(metricFilter.getDomainId());
}
if (metricFilter.getSensitiveLevel() != null) {
metricDOExample.getOredCriteria().get(0).andSensitiveLevelEqualTo(metricFilter.getSensitiveLevel());
}
if (metricFilter.getStatus() != null) {
metricDOExample.getOredCriteria().get(0).andStatusEqualTo(metricFilter.getStatus());
}
if (StringUtils.isNotBlank(metricFilter.getType())) {
metricDOExample.getOredCriteria().get(0).andTypeEqualTo(metricFilter.getType());
}
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
return metricDOCustomMapper.query(metricFilter);
}
@Override

View File

@@ -97,4 +97,29 @@
</foreach>
</update>
<select id="query" resultMap="ResultMapWithBLOBs">
select *
from s2_metric
where 1=1
<if test="type != null and type != ''">
and type = #{type}
</if>
<if test="name != null and name != ''">
and ( id like CONCAT('%',#{name , jdbcType=VARCHAR},'%') or
name like CONCAT('%',#{name , jdbcType=VARCHAR},'%') or
biz_name like CONCAT('%',#{name , jdbcType=VARCHAR},'%') or
description like CONCAT('%',#{name , jdbcType=VARCHAR},'%') )
</if>
<if test="sensitiveLevel != null">
and sensitive_level = #{sensitiveLevel}
</if>
<if test="domainIds != null and domainIds.size >0">
and domain_id in
<foreach collection="domainIds" index="index" item="domain" open="(" close=")"
separator=",">
#{domain}
</foreach>
</if>
</select>
</mapper>

View File

@@ -42,6 +42,7 @@ public class QueryParser {
if (!parseSqlReq.getSql().isEmpty()) {
return parser(parseSqlReq);
}
metricReq.setNativeQuery(queryStructReq.getNativeQuery());
return parser(metricReq);
}
@@ -94,7 +95,7 @@ public class QueryParser {
}
public QueryStatement parser(MetricReq metricCommand) {
return parser(metricCommand, true);
return parser(metricCommand, !metricCommand.isNativeQuery());
}
public QueryStatement parser(MetricReq metricCommand, boolean isAgg) {

View File

@@ -2,12 +2,6 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DimensionNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
@@ -15,7 +9,13 @@ import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DimensionNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.IdentifyNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;
import java.util.HashSet;
@@ -33,7 +33,7 @@ import org.springframework.util.CollectionUtils;
@Slf4j
public class SourceRender extends Renderer {
public static TableView renderOne(String alias, List<String> fieldWhere,
public static TableView renderOne(String alias, List<String> fieldWheres,
List<String> reqMetrics, List<String> reqDimensions, String queryWhere, DataSource datasource,
SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception {
@@ -41,11 +41,8 @@ public class SourceRender extends Renderer {
TableView output = new TableView();
List<String> queryMetrics = new ArrayList<>(reqMetrics);
List<String> queryDimensions = new ArrayList<>(reqDimensions);
List<String> fieldWhere = new ArrayList<>(fieldWheres);
if (!fieldWhere.isEmpty()) {
// SqlNode sqlNode = SemanticNode.parse(queryWhere, scope);
// if (addWhere) {
// output.getFilter().add(sqlNode);
// }
Set<String> dimensions = new HashSet<>();
Set<String> metrics = new HashSet<>();
whereDimMetric(fieldWhere, queryMetrics, queryDimensions, datasource, schema, dimensions, metrics);
@@ -69,7 +66,8 @@ public class SourceRender extends Renderer {
}
}
for (String dimension : queryDimensions) {
if(dimension.contains(Constants.DIMENSION_IDENTIFY) && queryDimensions.contains(dimension.split(Constants.DIMENSION_IDENTIFY)[1])){
if (dimension.contains(Constants.DIMENSION_IDENTIFY) && queryDimensions.contains(
dimension.split(Constants.DIMENSION_IDENTIFY)[1])) {
continue;
}
buildDimension(dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension : "",
@@ -213,14 +211,15 @@ public class SourceRender extends Renderer {
continue;
}
String filterField = field;
if(field.contains(Constants.DIMENSION_IDENTIFY)) {
if (field.contains(Constants.DIMENSION_IDENTIFY)) {
filterField = field.split(Constants.DIMENSION_IDENTIFY)[1];
}
addField(filterField,field,datasource,schema,dimensions,metrics);
addField(filterField, field, datasource, schema, dimensions, metrics);
}
}
private static void addField(String field,String oriField,DataSource datasource, SemanticSchema schema, Set<String> dimensions,
private static void addField(String field, String oriField, DataSource datasource, SemanticSchema schema,
Set<String> dimensions,
Set<String> metrics) {
Optional<Dimension> dimension = datasource.getDimensions().stream()
.filter(d -> d.getName().equalsIgnoreCase(field)).findFirst();
@@ -251,9 +250,11 @@ public class SourceRender extends Renderer {
Optional<Metric> datasourceMetric = schema.getMetrics()
.stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst();
if (datasourceMetric.isPresent()) {
Set<String> measures = datasourceMetric.get().getMetricTypeParams().getMeasures().stream().map(m->m.getName()).collect(
Collectors.toSet());
if(datasource.getMeasures().stream().map(m->m.getName()).collect(Collectors.toSet()).containsAll(measures)){
Set<String> measures = datasourceMetric.get().getMetricTypeParams().getMeasures().stream()
.map(m -> m.getName()).collect(
Collectors.toSet());
if (datasource.getMeasures().stream().map(m -> m.getName()).collect(Collectors.toSet())
.containsAll(measures)) {
metrics.add(oriField);
return;
}

View File

@@ -197,9 +197,9 @@ public class CalculateAggConverter implements SemanticConverter {
public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s",
return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getColumn());
f.getFunc().getOperator(),f.getColumn());
} else {
return f.getColumn();
}
@@ -335,9 +335,9 @@ public class CalculateAggConverter implements SemanticConverter {
String aggStr = queryStructCmd.getAggregators().stream().map(f -> {
if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) {
return String.format(
"if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s",
"if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s",
f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(),
f.getColumn());
f.getColumn(),f.getFunc().getOperator(),f.getColumn());
} else {
return f.getColumn();
}

View File

@@ -39,7 +39,7 @@ public class DateUtils {
private String sysDateWeekCol;
public Boolean recentMode(DateConf dateInfo) {
if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT_UNITS == dateInfo.getDateMode()
if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT == dateInfo.getDateMode()
&& DAY.equalsIgnoreCase(dateInfo.getPeriod()) && Objects.nonNull(dateInfo.getUnit())) {
return true;
}
@@ -47,7 +47,7 @@ public class DateUtils {
}
public boolean hasAvailableDataMode(DateConf dateInfo) {
if (Objects.nonNull(dateInfo) && DateConf.DateMode.AVAILABLE_TIME == dateInfo.getDateMode()) {
if (Objects.nonNull(dateInfo) && DateConf.DateMode.AVAILABLE == dateInfo.getDateMode()) {
return true;
}
return false;
@@ -263,16 +263,16 @@ public class DateUtils {
public String getDateWhereStr(DateConf dateInfo, ItemDateResp dateDate) {
String dateStr = "";
switch (dateInfo.getDateMode()) {
case BETWEEN_CONTINUOUS:
case BETWEEN:
dateStr = betweenDateStr(dateDate, dateInfo);
break;
case LIST_DISCRETE:
case LIST:
dateStr = listDateStr(dateDate, dateInfo);
break;
case RECENT_UNITS:
case RECENT:
dateStr = recentDateStr(dateDate, dateInfo);
break;
case AVAILABLE_TIME:
case AVAILABLE:
dateStr = hasDataModeStr(dateDate, dateInfo);
break;
default:

View File

@@ -101,10 +101,10 @@ public class QueryStructUtils {
if (Objects.isNull(dateDate)
|| Strings.isEmpty(dateDate.getStartDate())
&& Strings.isEmpty(dateDate.getEndDate())) {
if (dateInfo.getDateMode().equals(DateMode.LIST_DISCRETE)) {
if (dateInfo.getDateMode().equals(DateMode.LIST)) {
return dateUtils.listDateStr(dateDate, dateInfo);
}
if (dateInfo.getDateMode().equals(DateMode.BETWEEN_CONTINUOUS)) {
if (dateInfo.getDateMode().equals(DateMode.BETWEEN)) {
return dateUtils.betweenDateStr(dateDate, dateInfo);
}
if (dateUtils.hasAvailableDataMode(dateInfo)) {