4 Commits

Author SHA1 Message Date
jipeli
40ea6a9396 (feature)(headless) Add tag query api (#790) 2024-03-06 11:41:47 +08:00
daikon
78d724ea83 [improvement](headless) add queryTag for tagMarket (#772) 2024-02-27 20:34:00 +08:00
lexluo09
eadbdc4e30 Merge pull request #759 from lexluo09/master
(improvement)(project) merge master to dev-0.9
2024-02-26 14:41:23 +08:00
jipeli
b8831317e9 (feature)(headless) Add tag rest api (#733) 2024-02-21 17:45:28 +08:00
32 changed files with 1148 additions and 35 deletions

View File

@@ -19,6 +19,8 @@ public class Aggregator {
private List<String> args; private List<String> args;
private String alias;
public Aggregator() { public Aggregator() {
} }

View File

@@ -4,6 +4,7 @@ public enum TypeEnums {
METRIC, METRIC,
DIMENSION, DIMENSION,
TAG,
DOMAIN, DOMAIN,
ENTITY, ENTITY,
VIEW, VIEW,

View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public class TagDefineParams {
private String expr;
}

View File

@@ -0,0 +1,8 @@
package com.tencent.supersonic.headless.api.pojo.enums;
public enum TagDefineType {
FIELD,
DIMENSION,
Tag
}

View File

@@ -0,0 +1,27 @@
package com.tencent.supersonic.headless.api.pojo.enums;
import java.util.Objects;
public enum TagType {
ATOMIC,
DERIVED;
public static TagType of(String src) {
for (TagType tagType : TagType.values()) {
if (Objects.nonNull(src) && src.equalsIgnoreCase(tagType.name())) {
return tagType;
}
}
return null;
}
public static Boolean isDerived(String src) {
TagType tagType = of(src);
return Objects.nonNull(tagType) && tagType.equals(DERIVED);
}
public static TagType getType(TagDefineType tagDefineType) {
return Objects.nonNull(tagDefineType) && TagDefineType.Tag.equals(tagDefineType) ? TagType.DERIVED
: TagType.ATOMIC;
}
}

View File

@@ -12,6 +12,10 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.JSQLParserException;
@@ -35,11 +39,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings; import org.apache.logging.log4j.util.Strings;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
@Data @Data
@Slf4j @Slf4j
@@ -207,7 +206,8 @@ public class QueryStructReq extends SemanticQueryReq {
} }
sumFunction.setParameters(new ExpressionList(new Column(columnName))); sumFunction.setParameters(new ExpressionList(new Column(columnName)));
SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction); SelectExpressionItem selectExpressionItem = new SelectExpressionItem(sumFunction);
selectExpressionItem.setAlias(new Alias(columnName)); String alias = StringUtils.isNotBlank(aggregator.getAlias()) ? aggregator.getAlias() : columnName;
selectExpressionItem.setAlias(new Alias(alias));
selectItems.add(selectExpressionItem); selectItems.add(selectExpressionItem);
} }
} }

View File

@@ -0,0 +1,64 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Data
@Slf4j
@ToString
public class QueryTagReq extends SemanticQueryReq {
private List<String> groups = new ArrayList<>();
private List<Aggregator> aggregators = new ArrayList<>();
private List<Filter> tagFilters = new ArrayList<>();
private List<Order> orders = new ArrayList<>();
private Long limit = 20L;
private Long offset = 0L;
private String tagFiltersDate;
private DateConf dateInfo;
@Override
public String toCustomizedString() {
StringBuilder stringBuilder = new StringBuilder("{");
stringBuilder.append("\"viewId\":")
.append(viewId);
stringBuilder.append("\"modelIds\":")
.append(modelIds);
stringBuilder.append(",\"groups\":")
.append(groups);
stringBuilder.append(",\"aggregators\":")
.append(aggregators);
stringBuilder.append(",\"orders\":")
.append(orders);
stringBuilder.append(",\"tagFilters\":")
.append(tagFilters);
stringBuilder.append(",\"dateInfo\":")
.append(dateInfo);
stringBuilder.append(",\"params\":")
.append(params);
stringBuilder.append(",\"limit\":")
.append(limit);
stringBuilder.append('}');
return stringBuilder.toString();
}
public List<String> getMetrics() {
List<String> metrics = Lists.newArrayList();
if (!CollectionUtils.isEmpty(this.aggregators)) {
metrics = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList());
}
return metrics;
}
}

View File

@@ -0,0 +1,33 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.TagType;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import lombok.Data;
@Data
public class TagReq extends SchemaItem {
private Long modelId;
private Map<String, Object> ext = new HashMap<>();
private TagDefineType tagDefineType;
private TagDefineParams tagDefineParams;
public String getTypeParamsJson() {
return JSONObject.toJSONString(tagDefineParams);
}
public String getExtJson() {
return Objects.nonNull(ext) && ext.size() > 0 ? JSONObject.toJSONString(ext) : "";
}
public TagType getType() {
return TagType.getType(tagDefineType);
}
}

View File

@@ -22,6 +22,7 @@ public class SemanticSchemaResp {
private SchemaType schemaType; private SchemaType schemaType;
private List<MetricSchemaResp> metrics = Lists.newArrayList(); private List<MetricSchemaResp> metrics = Lists.newArrayList();
private List<DimSchemaResp> dimensions = Lists.newArrayList(); private List<DimSchemaResp> dimensions = Lists.newArrayList();
private List<TagResp> tags = Lists.newArrayList();
private List<ModelRela> modelRelas = Lists.newArrayList(); private List<ModelRela> modelRelas = Lists.newArrayList();
private List<ModelResp> modelResps = Lists.newArrayList(); private List<ModelResp> modelResps = Lists.newArrayList();
private ViewResp viewResp; private ViewResp viewResp;

View File

@@ -0,0 +1,33 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;
import lombok.ToString;
@Data
@ToString(callSuper = true)
public class TagResp extends SchemaItem {
private Long modelId;
private String type;
private Boolean isCollect;
private boolean hasAdminRes;
private Map<String, Object> ext = new HashMap<>();
private TagDefineType tagDefineType = TagDefineType.FIELD;
private TagDefineParams tagDefineParams;
public String getExpr() {
return tagDefineParams.getExpr();
}
}

View File

@@ -13,11 +13,19 @@ import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DimensionService;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings; import org.apache.logging.log4j.util.Strings;
@@ -29,14 +37,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Aspect @Aspect
@Component @Component
@Slf4j @Slf4j
@@ -63,9 +63,17 @@ public class DimValueAspect {
if (queryReq instanceof QuerySqlReq) { if (queryReq instanceof QuerySqlReq) {
return handleSqlDimValue(joinPoint); return handleSqlDimValue(joinPoint);
} }
if (queryReq instanceof QueryTagReq) {
return handleTagValue(joinPoint);
}
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq); throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
} }
public Object handleTagValue(ProceedingJoinPoint joinPoint) throws Throwable {
return (SemanticQueryResp) joinPoint.proceed();
}
private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable { private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] args = joinPoint.getArgs(); Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0]; QueryStructReq queryStructReq = (QueryStructReq) args[0];

View File

@@ -52,6 +52,7 @@ public class ModelYamlManager {
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
} }
dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields()); dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
dataModelYamlTpl.setId(modelResp.getId());
return dataModelYamlTpl; return dataModelYamlTpl;
} }

View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.Field; import com.tencent.supersonic.headless.api.pojo.Field;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
@@ -29,11 +30,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter; import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
@@ -44,11 +40,16 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j @Slf4j
@Service @Service
public class SemanticSchemaManager { public class SemanticSchemaManager {
private final Catalog catalog; private final Catalog catalog;
public SemanticSchemaManager(Catalog catalog) { public SemanticSchemaManager(Catalog catalog) {
@@ -87,6 +88,54 @@ public class SemanticSchemaManager {
return semanticModel; return semanticModel;
} }
public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception {
if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) {
throw new Exception("semanticSchemaResp tag is empty");
}
SemanticModel semanticModel = getSemanticModel(semanticSchemaResp);
//Map<String, List<Dimension>> dimensions = new HashMap<>();
Map<Long, List<TagResp>> tagMap = new HashMap<>();
for (TagResp tagResp : semanticSchemaResp.getTags()) {
if (!tagMap.containsKey(tagResp.getModelId())) {
tagMap.put(tagResp.getModelId(), new ArrayList<>());
}
tagMap.get(tagResp.getModelId()).add(tagResp);
}
if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) {
for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap().entrySet()) {
List<Dimension> dimensions = new ArrayList<>();
List<String> tagNames = new ArrayList<>();
if (tagMap.containsKey(entry.getValue().getId())) {
for (TagResp tagResp : tagMap.get(entry.getValue().getId())) {
tagNames.add(tagResp.getBizName());
Dimension dimension = Dimension.builder().build();
dimension.setType("");
dimension.setExpr(tagResp.getExpr());
dimension.setName(tagResp.getBizName());
dimension.setOwners("");
dimension.setBizName(tagResp.getBizName());
if (Objects.isNull(dimension.getDataType())) {
dimension.setDataType(DataType.UNKNOWN);
}
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
dimension.setDimensionTimeTypeParams(dimensionTimeTypeParams);
dimensions.add(dimension);
}
}
if (semanticModel.getDimensionMap().containsKey(entry.getKey())) {
semanticModel.getDimensionMap().get(entry.getKey()).stream()
.filter(d -> !tagNames.contains(d.getBizName())).forEach(d -> {
dimensions.add(d);
});
}
semanticModel.getDimensionMap().put(entry.getKey(), dimensions);
}
}
// metric ignored
semanticModel.setMetrics(new ArrayList<>());
return semanticModel;
}
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) { public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
return getMetricsByMetricYamlTpl(t); return getMetricsByMetricYamlTpl(t);
} }

View File

@@ -0,0 +1,79 @@
package com.tencent.supersonic.headless.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.util.Date;
import lombok.Data;
@Data
@TableName("s2_tag")
public class TagDO {
@TableId(type = IdType.AUTO)
private Long id;
/**
* 主体域ID
*/
private Long modelId;
/**
* 指标名称
*/
private String name;
/**
* 字段名称
*/
private String bizName;
/**
* 描述
*/
private String description;
/**
* 指标状态,0正常,1下架,2删除
*/
private Integer status;
/**
* 敏感级别
*/
private Integer sensitiveLevel;
/**
* 类型 DERIVED,ATOMIC
*/
private String type;
/**
* 创建时间
*/
private Date createdAt;
/**
* 创建人
*/
private String createdBy;
/**
* 更新时间
*/
private Date updatedAt;
/**
* 更新人
*/
private String updatedBy;
/**
* 类型参数
*/
private String defineType;
private String typeParams;
private String ext;
}

View File

@@ -0,0 +1,11 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TagCustomMapper {
List<TagDO> query(TagFilter tagFilter);
}

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TagMapper extends BaseMapper<TagDO> {
}

View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import java.util.List;
public interface TagRepository {
Long create(TagDO tagDO);
void update(TagDO tagDO);
TagDO getTagById(Long id);
List<TagDO> query(TagFilter tagFilter);
}

View File

@@ -0,0 +1,44 @@
package com.tencent.supersonic.headless.server.persistence.repository.impl;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import com.tencent.supersonic.headless.server.persistence.mapper.TagCustomMapper;
import com.tencent.supersonic.headless.server.persistence.mapper.TagMapper;
import com.tencent.supersonic.headless.server.persistence.repository.TagRepository;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Repository;
@Slf4j
@Repository
public class TagRepositoryImpl implements TagRepository {
private final TagMapper mapper;
private final TagCustomMapper tagCustomMapper;
public TagRepositoryImpl(TagMapper mapper,
TagCustomMapper tagCustomMapper) {
this.mapper = mapper;
this.tagCustomMapper = tagCustomMapper;
}
@Override
public Long create(TagDO tagDO) {
mapper.insert(tagDO);
return tagDO.getId();
}
@Override
public void update(TagDO tagDO) {
mapper.updateById(tagDO);
}
@Override
public TagDO getTagById(Long id) {
return mapper.selectById(id);
}
@Override
public List<TagDO> query(TagFilter tagFilter) {
return tagCustomMapper.query(tagFilter);
}
}

View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.List;
import lombok.Data;
@Data
public class TagFilter extends MetaFilter {
private String type;
private List<Integer> statusList;
}

View File

@@ -0,0 +1,11 @@
package com.tencent.supersonic.headless.server.pojo;
import com.tencent.supersonic.headless.api.pojo.request.PageSchemaItemReq;
import java.util.List;
public class TagFilterPage extends PageSchemaItemReq {
private String type;
private List<Integer> statusList;
}

View File

@@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp; import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
@@ -58,6 +59,14 @@ public class QueryController {
return queryService.queryByReq(querySqlReq, user); return queryService.queryByReq(querySqlReq, user);
} }
@PostMapping("/tag")
public Object queryByTag(@RequestBody QueryTagReq queryTagReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return queryService.queryByReq(queryTagReq, user);
}
@PostMapping("/queryMetricDataById") @PostMapping("/queryMetricDataById")
public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq, public ItemQueryResultResp queryMetricDataById(@Valid @RequestBody QueryItemReq queryApiReq,
HttpServletRequest request) throws Exception { HttpServletRequest request) throws Exception {

View File

@@ -0,0 +1,69 @@
package com.tencent.supersonic.headless.server.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.TagReq;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
import com.tencent.supersonic.headless.server.service.TagService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/semantic/tag")
public class TagController {
private final TagService tagService;
public TagController(TagService tagService) {
this.tagService = tagService;
}
@PostMapping("/create")
public TagResp create(@RequestBody TagReq tagReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagService.create(tagReq, user);
}
@PostMapping("/update")
public TagResp update(@RequestBody TagReq tagReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagService.update(tagReq, user);
}
@DeleteMapping("delete/{id}")
public Boolean delete(@PathVariable("id") Long id,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
tagService.delete(id, user);
return true;
}
@GetMapping("getTag/{id}")
public TagResp getTag(@PathVariable("id") Long id,
HttpServletRequest request,
HttpServletResponse response) {
return tagService.getTag(id);
}
@PostMapping("/queryTag")
public PageInfo<TagResp> queryPage(@RequestBody TagFilterPage tagFilterPage,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagService.queryPage(tagFilterPage, user);
}
}

View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.headless.server.service;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.TagReq;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
import java.util.List;
public interface TagService {
TagResp create(TagReq tagReq, User user) throws Exception;
TagResp update(TagReq tagReq, User user) throws Exception;
void delete(Long id, User user) throws Exception;
TagResp getTag(Long id);
List<TagResp> query(TagFilter tagFilter);
PageInfo<TagResp> queryPage(TagFilterPage tagFilterPage, User user);
}

View File

@@ -23,6 +23,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp; import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp;
@@ -58,6 +59,7 @@ import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils; import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.utils.TagReqConverter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
@@ -83,6 +85,7 @@ public class QueryServiceImpl implements QueryService {
private StatUtils statUtils; private StatUtils statUtils;
private final QueryUtils queryUtils; private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter; private final QueryReqConverter queryReqConverter;
private final TagReqConverter tagReqConverter;
private final Catalog catalog; private final Catalog catalog;
private final AppService appService; private final AppService appService;
private final QueryCache queryCache; private final QueryCache queryCache;
@@ -102,7 +105,7 @@ public class QueryServiceImpl implements QueryService {
StatUtils statUtils, StatUtils statUtils,
QueryUtils queryUtils, QueryUtils queryUtils,
QueryReqConverter queryReqConverter, QueryReqConverter queryReqConverter,
Catalog catalog, TagReqConverter tagReqConverter, Catalog catalog,
AppService appService, AppService appService,
QueryCache queryCache, QueryCache queryCache,
SemanticSchemaManager semanticSchemaManager, SemanticSchemaManager semanticSchemaManager,
@@ -114,6 +117,7 @@ public class QueryServiceImpl implements QueryService {
this.statUtils = statUtils; this.statUtils = statUtils;
this.queryUtils = queryUtils; this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter; this.queryReqConverter = queryReqConverter;
this.tagReqConverter = tagReqConverter;
this.catalog = catalog; this.catalog = catalog;
this.appService = appService; this.appService = appService;
this.queryCache = queryCache; this.queryCache = queryCache;
@@ -185,6 +189,9 @@ public class QueryServiceImpl implements QueryService {
if (semanticQueryReq instanceof QueryMultiStructReq) { if (semanticQueryReq instanceof QueryMultiStructReq) {
return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq); return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq);
} }
if (semanticQueryReq instanceof QueryTagReq) {
return buildTagQueryStatement((QueryTagReq) semanticQueryReq);
}
return null; return null;
} }
@@ -220,6 +227,21 @@ public class QueryServiceImpl implements QueryService {
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
} }
private QueryStatement buildTagQueryStatement(QueryTagReq queryTagReq)
throws Exception {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
SchemaFilterReq filter = buildSchemaFilterReq(queryTagReq);
schemaFilterReq.setModelIds(queryTagReq.getModelIds());
SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
QueryStatement queryStatement = tagReqConverter.convert(queryTagReq, semanticSchemaResp);
queryStatement.setModelIds(queryTagReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
SemanticModel semanticModel = semanticSchemaManager.getTagSemanticModel(semanticSchemaResp);
queryStatement.setSemanticModel(semanticModel);
return queryStatement;
}
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
schemaFilterReq.setViewId(semanticQueryReq.getViewId()); schemaFilterReq.setViewId(semanticQueryReq.getViewId());

View File

@@ -29,15 +29,18 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService; import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.TagService;
import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter;
@@ -78,14 +81,15 @@ public class SchemaServiceImpl implements SchemaService {
private final DomainService domainService; private final DomainService domainService;
private final ViewService viewService; private final ViewService viewService;
private final ModelRelaService modelRelaService; private final ModelRelaService modelRelaService;
private final TagService tagService;
public SchemaServiceImpl(ModelService modelService, public SchemaServiceImpl(ModelService modelService,
DimensionService dimensionService, DimensionService dimensionService,
MetricService metricService, MetricService metricService,
DomainService domainService, DomainService domainService,
ViewService viewService, ViewService viewService,
ModelRelaService modelRelaService, ModelRelaService modelRelaService,
StatUtils statUtils) { StatUtils statUtils, TagService tagService) {
this.modelService = modelService; this.modelService = modelService;
this.dimensionService = dimensionService; this.dimensionService = dimensionService;
this.metricService = metricService; this.metricService = metricService;
@@ -93,6 +97,7 @@ public class SchemaServiceImpl implements SchemaService {
this.viewService = viewService; this.viewService = viewService;
this.modelRelaService = modelRelaService; this.modelRelaService = modelRelaService;
this.statUtils = statUtils; this.statUtils = statUtils;
this.tagService = tagService;
} }
@SneakyThrows @SneakyThrows
@@ -301,6 +306,11 @@ public class SchemaServiceImpl implements SchemaService {
.flatMap(Collection::stream).collect(Collectors.toList())); .flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
semanticSchemaResp.setSchemaType(SchemaType.MODEL); semanticSchemaResp.setSchemaType(SchemaType.MODEL);
// add tag info
TagFilter tagFilter = new TagFilter();
tagFilter.setModelIds(schemaFilterReq.getModelIds());
List<TagResp> tagResps = tagService.query(tagFilter);
semanticSchemaResp.setTags(tagResps);
} }
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) {
DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0));

View File

@@ -0,0 +1,246 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.TagReq;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import com.tencent.supersonic.headless.server.persistence.repository.TagRepository;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagService;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class TagServiceImpl implements TagService {
private final TagRepository tagRepository;
private final ModelService modelService;
private final CollectService collectService;
public TagServiceImpl(TagRepository tagRepository, ModelService modelService,
CollectService collectService) {
this.tagRepository = tagRepository;
this.modelService = modelService;
this.collectService = collectService;
}
@Override
public TagResp create(TagReq tagReq, User user) throws Exception {
checkParam(tagReq);
checkExit(tagReq);
TagDO tagDO = convert(tagReq);
tagDO.setCreatedBy(user.getName());
tagDO.setCreatedAt(new Date());
tagDO.setStatus(StatusEnum.ONLINE.getCode());
tagRepository.create(tagDO);
return convert(tagDO);
}
@Override
public TagResp update(TagReq tagReq, User user) throws Exception {
if (Objects.isNull(tagReq.getId()) || tagReq.getId() <= 0) {
throw new RuntimeException("id is empty");
}
TagDO tagDO = tagRepository.getTagById(tagReq.getId());
if (Objects.nonNull(tagDO) && tagDO.getId() > 0) {
if (Objects.nonNull(tagReq.getExt()) && !tagReq.getExt().isEmpty()) {
tagDO.setExt(tagReq.getExtJson());
}
}
if (Objects.nonNull(tagReq.getTagDefineType())) {
tagDO.setDefineType(tagReq.getTagDefineType().name());
}
if (Objects.nonNull(tagReq.getTagDefineParams()) && !StringUtils.isBlank(
tagReq.getTagDefineParams().getExpr())) {
tagDO.setTypeParams(tagReq.getTypeParamsJson());
}
tagDO.setUpdatedBy(user.getName());
tagDO.setUpdatedAt(new Date());
tagRepository.update(tagDO);
return convert(tagDO);
}
@Override
public void delete(Long id, User user) throws Exception {
TagDO tagDO = tagRepository.getTagById(id);
if (Objects.isNull(tagDO)) {
throw new RuntimeException("tag not found");
}
tagDO.setStatus(StatusEnum.DELETED.getCode());
tagDO.setUpdatedBy(user.getName());
tagDO.setUpdatedAt(new Date());
tagRepository.update(tagDO);
}
@Override
public TagResp getTag(Long id) {
return convert(tagRepository.getTagById(id));
}
@Override
public List<TagResp> query(TagFilter tagFilter) {
List<TagDO> tagDOS = tagRepository.query(tagFilter);
if (!CollectionUtils.isEmpty(tagDOS)) {
return tagDOS.stream().map(tagDO -> convert(tagDO)).collect(Collectors.toList());
}
return new ArrayList<>();
}
@Override
public PageInfo<TagResp> queryPage(TagFilterPage tagFilterPage, User user) {
TagFilter tagFilter = new TagFilter();
BeanUtils.copyProperties(tagFilterPage, tagFilter);
List<ModelResp> modelRespList = modelService.getAllModelByDomainIds(tagFilterPage.getDomainIds());
List<Long> modelIds = modelRespList.stream().map(ModelResp::getId).collect(Collectors.toList());
tagFilterPage.getModelIds().addAll(modelIds);
tagFilter.setModelIds(tagFilterPage.getModelIds());
List<CollectDO> collectList = collectService.getCollectList(user.getName())
.stream().filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
.collect(Collectors.toList());
List<Long> collectIds = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList());
if (tagFilterPage.isHasCollect()) {
if (CollectionUtils.isEmpty(collectIds)) {
tagFilter.setIds(Lists.newArrayList(-1L));
} else {
tagFilter.setIds(collectIds);
}
}
PageInfo<TagDO> tagDOPageInfo = PageHelper.startPage(tagFilterPage.getCurrent(),
tagFilterPage.getPageSize())
.doSelectPageInfo(() -> query(tagFilter));
PageInfo<TagResp> pageInfo = new PageInfo<>();
BeanUtils.copyProperties(tagDOPageInfo, pageInfo);
List<TagResp> tagRespList = convertList(tagDOPageInfo.getList(), collectIds);
fillAdminRes(tagRespList, user);
pageInfo.setList(tagRespList);
return pageInfo;
}
private void fillAdminRes(List<TagResp> tagRespList, User user) {
List<ModelResp> modelRespList = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelRespList)) {
return;
}
Set<Long> modelIdSet = modelRespList.stream().map(ModelResp::getId).collect(Collectors.toSet());
for (TagResp tagResp : tagRespList) {
if (modelIdSet.contains(tagResp.getModelId())) {
tagResp.setHasAdminRes(true);
} else {
tagResp.setHasAdminRes(false);
}
}
}
private List<TagResp> convertList(List<TagDO> tagDOList, List<Long> collectIds) {
List<TagResp> tagRespList = new ArrayList<>();
if (CollectionUtils.isNotEmpty(tagDOList)) {
tagDOList.stream().forEach(tagDO -> {
TagResp tagResp = convert(tagDO);
if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagDO.getId())) {
tagResp.setIsCollect(true);
} else {
tagResp.setIsCollect(false);
}
tagRespList.add(tagResp);
});
}
return tagRespList;
}
private void checkExit(TagReq tagReq) {
TagFilter tagFilter = new TagFilter();
tagFilter.setModelIds(Arrays.asList(tagReq.getModelId()));
List<TagResp> tagResps = query(tagFilter);
if (!CollectionUtils.isEmpty(tagResps)) {
Long bizNameSameCount = tagResps.stream().filter(tagResp -> !tagResp.getId().equals(tagReq.getId()))
.filter(tagResp -> tagResp.getBizName().equalsIgnoreCase(tagReq.getBizName())).count();
if (bizNameSameCount > 0) {
throw new RuntimeException(String.format("the bizName %s is exit", tagReq.getBizName()));
}
Long nameSameCount = tagResps.stream().filter(tagResp -> !tagResp.getId().equals(tagReq.getId()))
.filter(tagResp -> tagResp.getName().equalsIgnoreCase(tagReq.getName())).count();
if (nameSameCount > 0) {
throw new RuntimeException(String.format("the name %s is exit", tagReq.getName()));
}
}
}
private void checkParam(TagReq tagReq) {
if (Objects.isNull(tagReq.getModelId()) || tagReq.getModelId() <= 0) {
throw new RuntimeException("the modelId is empty");
}
if (Objects.isNull(tagReq.getBizName()) || tagReq.getBizName().isEmpty() || Objects.isNull(tagReq.getName())
|| tagReq.getName().isEmpty()) {
throw new RuntimeException("the bizName or name is empty");
}
if (Objects.isNull(tagReq.getTagDefineType()) || Objects.isNull(tagReq.getTagDefineParams())
|| StringUtils.isBlank(tagReq.getTagDefineParams().getExpr())) {
throw new InvalidArgumentException("表达式不可为空");
}
if (NameCheckUtils.containsSpecialCharacters(tagReq.getBizName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
}
private TagResp convert(TagDO tagDO) {
TagResp tagResp = new TagResp();
BeanUtils.copyProperties(tagDO, tagResp);
if (Objects.nonNull(tagDO.getExt()) && !tagDO.getExt().isEmpty()) {
Map<String, Object> ext = JSONObject.parseObject(tagDO.getExt(),
Map.class);
tagResp.setExt(ext);
}
tagResp.setTagDefineType(TagDefineType.valueOf(tagDO.getDefineType()));
if (Objects.nonNull(tagDO.getTypeParams()) && !tagDO.getTypeParams().isEmpty()) {
TagDefineParams tagDefineParams = JSONObject.parseObject(tagDO.getTypeParams(),
TagDefineParams.class);
tagResp.setTagDefineParams(tagDefineParams);
}
return tagResp;
}
private TagDO convert(TagReq tagReq) {
TagDO tagDO = new TagDO();
BeanUtils.copyProperties(tagReq, tagDO);
tagDO.setDefineType(tagReq.getTagDefineType().name());
tagDO.setType(tagReq.getType().name());
tagDO.setTypeParams(tagReq.getTypeParamsJson());
tagDO.setExt(tagReq.getExtJson());
return tagDO;
}
}

View File

@@ -16,22 +16,22 @@ import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import com.tencent.supersonic.headless.server.persistence.repository.StatRepository; import com.tencent.supersonic.headless.server.persistence.repository.StatRepository;
import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component @Component
@@ -96,6 +96,48 @@ public class StatUtils {
QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0); QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0);
initStructStatInfo(queryStructCmd, facadeUser); initStructStatInfo(queryStructCmd, facadeUser);
} }
if (semanticQueryReq instanceof QueryTagReq) {
initTagStatInfo((QueryTagReq) semanticQueryReq, facadeUser);
}
}
public void initTagStatInfo(QueryTagReq queryTagReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
String traceId = "";
List<String> dimensions = queryTagReq.getGroups();
List<String> metrics = new ArrayList<>();
queryTagReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn()));
String user = getUserName(facadeUser);
try {
queryStatInfo.setTraceId(traceId)
.setViewId(queryTagReq.getViewId())
.setUser(user)
.setQueryType(QueryType.STRUCT.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQueryStructCmd(queryTagReq.toString())
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryTagReq.toString()))
.setStartTime(System.currentTimeMillis())
.setNativeQuery(CollectionUtils.isEmpty(queryTagReq.getAggregators()))
.setGroupByCols(objectMapper.writeValueAsString(queryTagReq.getGroups()))
.setAggCols(objectMapper.writeValueAsString(queryTagReq.getAggregators()))
.setOrderByCols(objectMapper.writeValueAsString(queryTagReq.getOrders()))
.setFilterCols(objectMapper.writeValueAsString(
sqlFilterUtils.getFiltersCol(queryTagReq.getTagFilters())))
.setUseResultCache(true)
.setUseSqlCache(true)
.setMetrics(objectMapper.writeValueAsString(metrics))
.setDimensions(objectMapper.writeValueAsString(dimensions))
.setQueryOptMode(QueryOptMode.NONE.name());
if (!CollectionUtils.isEmpty(queryTagReq.getModelIds())) {
queryStatInfo.setModelId(queryTagReq.getModelIds().get(0));
}
} catch (JsonProcessingException e) {
e.printStackTrace();
}
StatUtils.set(queryStatInfo);
} }
public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) { public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {

View File

@@ -0,0 +1,97 @@
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class TagReqConverter {
@Value("${query.sql.limitWrapper:true}")
private Boolean limitWrapper;
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private SqlGenerateUtils sqlGenerateUtils;
public QueryStatement convert(QueryTagReq queryTagReq,
SemanticSchemaResp semanticSchemaResp) throws Exception {
QueryStatement queryStatement = new QueryStatement();
// covert to QueryReqConverter
QueryStructReq queryStructReq = new QueryStructReq();
BeanUtils.copyProperties(queryTagReq, queryStructReq);
if (!CollectionUtils.isEmpty(queryTagReq.getTagFilters())) {
queryStructReq.setDimensionFilters(queryTagReq.getTagFilters());
}
QuerySqlReq querySqlReq = queryStructReq.convert();
if (Objects.nonNull(querySqlReq)) {
log.info("convert to QuerySqlReq {}", querySqlReq);
String tableName = SqlSelectHelper.getTableName(querySqlReq.getSql());
MetricTable metricTable = new MetricTable();
metricTable.setMetrics(new ArrayList<>());
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
semanticSchemaResp.getModelResps().get(0).getBizName()));
metricTable.setAggOption(AggOption.NATIVE);
List<String> allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql());
metricTable.setDimensions(allFields);
metricTable.setAlias(tableName.toLowerCase());
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//.build ParseSqlReq
ViewQueryParam result = new ViewQueryParam();
BeanUtils.copyProperties(querySqlReq, result);
result.setTables(tables);
DatabaseResp database = semanticSchemaResp.getDatabaseResp();
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
result.setWithAlias(false);
}
//.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
queryStructReq.setViewId(querySqlReq.getViewId());
queryStructReq.setQueryType(QueryType.TAG);
QueryParam queryParam = new QueryParam();
convert(queryTagReq, queryParam);
queryStatement.setQueryParam(queryParam);
queryStatement.setViewQueryParam(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setViewId(queryTagReq.getViewId());
queryStatement.setEnableLimitWrapper(limitWrapper);
}
return queryStatement;
}
public void convert(QueryTagReq queryTagReq, QueryParam queryParam) {
BeanUtils.copyProperties(queryTagReq, queryParam);
queryParam.setOrders(queryTagReq.getOrders());
queryParam.setMetrics(queryTagReq.getMetrics());
queryParam.setGroups(queryTagReq.getGroups());
queryParam.setDimensionFilters(queryTagReq.getTagFilters());
queryParam.setQueryType(QueryType.TAG);
}
}

View File

@@ -0,0 +1,110 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.tencent.supersonic.headless.server.persistence.mapper.TagCustomMapper">
<resultMap id="BaseResultMap" type="com.tencent.supersonic.headless.server.persistence.dataobject.TagDO">
<id column="id" jdbcType="BIGINT" property="id" />
<result column="model_id" jdbcType="BIGINT" property="modelId" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="biz_name" jdbcType="VARCHAR" property="bizName" />
<result column="description" jdbcType="VARCHAR" property="description" />
<result column="status" jdbcType="INTEGER" property="status" />
<result column="sensitive_level" jdbcType="INTEGER" property="sensitiveLevel" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt" />
<result column="created_by" jdbcType="VARCHAR" property="createdBy" />
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
<result column="define_type" jdbcType="VARCHAR" property="defineType" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.headless.server.persistence.dataobject.TagDO">
<result column="type_params" jdbcType="LONGVARCHAR" property="typeParams" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and
#{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem"
open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, model_id, name, biz_name, description, status, sensitive_level, type, created_at,
created_by, updated_at, updated_by, define_type
</sql>
<sql id="Blob_Column_List">
type_params
</sql>
<select id="query" resultMap="ResultMapWithBLOBs">
select *
from s2_tag
where status != 3
<if test="type != null and type != ''">
and type = #{type}
</if>
<if test="key != null and key != ''">
and ( id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
description like CONCAT('%',#{key , jdbcType=VARCHAR},'%'))
</if>
<if test="id != null">
and id like CONCAT('%',#{id , jdbcType=VARCHAR},'%')
</if>
<if test="name != null and name != '' ">
and name like CONCAT('%',#{name , jdbcType=VARCHAR},'%')
</if>
<if test="bizName != null and bizName != ''">
and biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%')
</if>
<if test="sensitiveLevel != null">
and sensitive_level = #{sensitiveLevel}
</if>
<if test="status != null">
and status = #{status}
</if>
<if test="modelIds != null and modelIds.size >0">
and model_id in
<foreach collection="modelIds" index="index" item="model" open="(" close=")"
separator=",">
#{model}
</foreach>
</if>
<if test="ids != null and ids.size >0">
and id in
<foreach collection="ids" index="index" item="id" open="(" close=")"
separator=",">
#{id}
</foreach>
</if>
<if test="createdBy != null">
and created_by = #{createdBy}
</if>
</select>
</mapper>

View File

@@ -192,4 +192,24 @@ CREATE TABLE s2_view(
alter table s2_plugin change column model `view` varchar(100); alter table s2_plugin change column model `view` varchar(100);
alter table s2_view_info rename to s2_canvas; alter table s2_view_info rename to s2_canvas;
alter table s2_query_stat_info add column `view_id` bigint(20) DEFAULT NULL after `model_id`; alter table s2_query_stat_info add column `view_id` bigint(20) DEFAULT NULL after `model_id`;
--20240221
CREATE TABLE s2_tag(
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL ,
`sensitive_level` INT NOT NULL ,
`type` varchar(50) NOT NULL , -- ATOMIC, DERIVED
`define_type` varchar(50) NOT NULL, -- FIELD, DIMENSION
`type_params` LONGVARCHAR DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
`ext` LONGVARCHAR DEFAULT NULL ,
PRIMARY KEY (`id`)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

View File

@@ -572,4 +572,24 @@ CREATE TABLE IF NOT EXISTS `s2_view` (
query_config VARCHAR(3000), query_config VARCHAR(3000),
`admin` varchar(3000) DEFAULT NULL, `admin` varchar(3000) DEFAULT NULL,
`admin_org` varchar(3000) DEFAULT NULL `admin_org` varchar(3000) DEFAULT NULL
); );
CREATE TABLE IF NOT EXISTS `s2_tag` (
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL ,
`sensitive_level` INT NOT NULL ,
`type` varchar(50) NOT NULL , -- ATOMIC, DERIVED
`define_type` varchar(50) NOT NULL, -- FIELD, DIMENSION
`type_params` LONGVARCHAR DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
`ext` LONGVARCHAR DEFAULT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_tag IS 'tag information';

View File

@@ -497,4 +497,25 @@ CREATE TABLE s2_view
query_config VARCHAR(3000), query_config VARCHAR(3000),
`admin` varchar(3000) DEFAULT NULL, `admin` varchar(3000) DEFAULT NULL,
`admin_org` varchar(3000) DEFAULT NULL `admin_org` varchar(3000) DEFAULT NULL
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `s2_tag`
(
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`model_id` bigint(20) DEFAULT NULL,
`name` varchar(255) NOT NULL COMMENT '名称',
`biz_name` varchar(255) NOT NULL COMMENT '英文名称',
`description` varchar(500) DEFAULT NULL COMMENT '描述',
`status` int(10) NOT NULL COMMENT '状态',
`sensitive_level` int(10) NOT NULL COMMENT '敏感级别',
`type` varchar(50) NOT NULL COMMENT '类型(DERIVED,ATOMIC)',
`define_type` varchar(50) DEFAULT NULL, -- FIELD, DIMENSION
`type_params` text NOT NULL COMMENT '类型参数',
`created_at` datetime NOT NULL COMMENT '创建时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
`updated_at` datetime NULL COMMENT '更新时间',
`updated_by` varchar(100) NULL COMMENT '更新人',
`ext` text DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8 COMMENT ='标签表';