(improvement)(headless) Add API interface to provide data services to other applications (#561)

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-12-21 22:10:29 +08:00
committed by GitHub
parent 7b580b7c94
commit fa38e37be3
40 changed files with 1106 additions and 55 deletions

View File

@@ -28,6 +28,11 @@ public class User {
return new User(1L, "admin", "admin", "admin@email", 1);
}
public static User getAppUser(int appId) {
String name = String.format("app_%s", appId);
return new User(1L, name, name, "", 1);
}
public String getDisplayName() {
return StringUtils.isBlank(displayName) ? name : displayName;
}

View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.common.pojo;
public final class Pair<T, U> {
public final T first;
public final U second;
public Pair(T first, U second) {
this.second = second;
this.first = first;
}
// Because 'pair()' is shorter than 'new Pair<>()'.
// Sometimes this difference might be very significant (especially in a
// 80-ish characters boundary). Sorry diamond operator.
public static <T, U> Pair<T, U> pair(T first, U second) {
return new Pair<>(first, second);
}
@Override
public String toString() {
return "(" + first + ", " + second + ")";
}
}

View File

@@ -1,10 +1,11 @@
package com.tencent.supersonic.common.pojo;
import com.google.common.base.Objects;
import java.util.Date;
import lombok.Data;
import lombok.ToString;
import java.util.Date;
@Data
@ToString
public class RecordInfo {
@@ -42,8 +43,7 @@ public class RecordInfo {
}
RecordInfo that = (RecordInfo) o;
return Objects.equal(createdBy, that.createdBy) && Objects.equal(
updatedBy, that.updatedBy) && Objects.equal(createdAt, that.createdAt)
&& Objects.equal(updatedAt, that.updatedAt);
updatedBy, that.updatedBy);
}
@Override

View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.common.pojo.enums;
public enum ApiItemType {
METRIC,
TAG,
DIMENSION
}

View File

@@ -0,0 +1,71 @@
package com.tencent.supersonic.common.util;
import com.tencent.supersonic.common.pojo.Pair;
import org.apache.commons.codec.binary.Hex;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import static java.lang.Thread.sleep;
public class SignatureUtils {
private static final String ALGORITHM_HMAC_SHA256 = "HmacSHA256";
private static final long TIME_OUT = 60 * 1000 * 30;
public static String generateSignature(String appKey, String appSecret, long timestamp) {
try {
Mac sha256HMAC = Mac.getInstance(ALGORITHM_HMAC_SHA256);
SecretKeySpec secretKey = new SecretKeySpec(appSecret.getBytes(), ALGORITHM_HMAC_SHA256);
sha256HMAC.init(secretKey);
String data = appKey + timestamp;
byte[] hash = sha256HMAC.doFinal(data.getBytes());
return Hex.encodeHexString(hash);
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
throw new RuntimeException("Error generating signature", e);
}
}
public static Pair<Boolean, String> isValidSignature(String appKey, String appSecret,
long timestamp, String signatureToCheck) {
long currentTimeMillis = System.currentTimeMillis();
if (currentTimeMillis < timestamp) {
return new Pair<>(false, "Timestamp is in the future");
}
if (currentTimeMillis - timestamp > TIME_OUT) {
return new Pair<>(false, "Timestamp is too old");
}
String generatedSignature = generateSignature(appKey, appSecret, timestamp);
if (generatedSignature.equals(signatureToCheck)) {
return new Pair<>(true, "Signature is valid");
} else {
return new Pair<>(false, "Invalid signature");
}
}
public static void main(String[] args) throws InterruptedException {
// appkey为申请的接口id
String appKey = "1";
//生成的密钥
String appSecret = "8fb44f17-f37d-4510-bb29-59b0e0b266d0";
long timestamp = System.currentTimeMillis();
System.out.println("timeStamp:" + timestamp);
//生成的签名
String serverSignature = generateSignature(appKey, appSecret, timestamp);
System.out.println("Server Signature: " + serverSignature);
sleep(4000);
//用户需要的入参
Pair<Boolean, String> isValid = isValidSignature(appKey, appSecret, timestamp, serverSignature);
System.out.println("Is Signature Valid? " + isValid.first);
}
}

View File

@@ -0,0 +1,29 @@
package com.tencent.supersonic.headless.api.model.enums;
public enum AppStatusEnum {
INIT(0),
ONLINE(1),
OFFLINE(2),
DELETED(3),
UNKNOWN(4);
private Integer code;
AppStatusEnum(Integer code) {
this.code = code;
}
public Integer getCode() {
return code;
}
public static AppStatusEnum fromCode(Integer code) {
for (AppStatusEnum appStatusEnum : AppStatusEnum.values()) {
if (appStatusEnum.getCode().equals(code)) {
return appStatusEnum;
}
}
return AppStatusEnum.UNKNOWN;
}
}

View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.headless.api.model.pojo;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class AppConfig {
private List<Item> items = Lists.newArrayList();
}

View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.headless.api.model.pojo;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.ApiItemType;
import lombok.Data;
import java.util.List;
@Data
public class Item {
private Long id;
private String name;
private ApiItemType type;
private List<Item> relateItems = Lists.newArrayList();
public String getValue() {
return name;
}
}

View File

@@ -0,0 +1,29 @@
package com.tencent.supersonic.headless.api.model.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.PageBaseReq;
import com.tencent.supersonic.headless.api.model.enums.AppStatusEnum;
import lombok.Data;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
@Data
public class AppQueryReq extends PageBaseReq {
private String name;
private List<AppStatusEnum> appStatus;
private String createdBy;
public List<Integer> getStatus() {
if (CollectionUtils.isEmpty(appStatus)) {
return Lists.newArrayList();
}
return appStatus.stream().map(AppStatusEnum::getCode).collect(Collectors.toList());
}
}

View File

@@ -0,0 +1,35 @@
package com.tencent.supersonic.headless.api.model.request;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.headless.api.model.pojo.AppConfig;
import lombok.Data;
import org.springframework.util.CollectionUtils;
import java.util.Date;
import java.util.List;
@Data
public class AppReq extends RecordInfo {
private Long id;
private String name;
private String description;
private AppConfig config;
private Date endDate;
private Integer qps;
private List<String> owners;
public String getOwner() {
if (CollectionUtils.isEmpty(owners)) {
return "";
}
return String.join(",", owners);
}
}

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.api.model.response;
import lombok.Data;
@Data
public class AppDetailResp extends AppResp {
private String appSecret;
}

View File

@@ -0,0 +1,44 @@
package com.tencent.supersonic.headless.api.model.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.headless.api.model.enums.AppStatusEnum;
import com.tencent.supersonic.headless.api.model.pojo.AppConfig;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
@Data
public class AppResp extends RecordInfo {
private Long id;
private String name;
private String description;
private AppStatusEnum appStatus;
private AppConfig config;
private Date endDate;
private Integer qps;
private List<String> owners;
private boolean hasAdminRes;
public void setOwner(String owner) {
if (StringUtils.isBlank(owner)) {
owners = Lists.newArrayList();
return;
}
owners = Arrays.asList(owner.split(","));
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.api.model.response;
import com.google.common.base.Objects;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.model.pojo.Dim;
import com.tencent.supersonic.headless.api.model.pojo.DrillDownDimension;
@@ -44,10 +43,6 @@ public class ModelResp extends SchemaItem {
private String fullPath;
private Integer dimensionCnt;
private Integer metricCnt;
public boolean openToAll() {
return isOpen != null && isOpen == 1;
}
@@ -74,24 +69,4 @@ public class ModelResp extends SchemaItem {
return modelDetail.getTimeDims();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
if (!super.equals(o)) {
return false;
}
ModelResp that = (ModelResp) o;
return Objects.equal(getId(), that.getId());
}
@Override
public int hashCode() {
return Objects.hashCode(super.hashCode(), getId());
}
}

View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.headless.api.query.pojo;
import com.tencent.supersonic.headless.api.model.pojo.Item;
import com.tencent.supersonic.headless.api.model.response.QueryResultWithSchemaResp;
import lombok.Data;
@Data
public class ApiQuerySingleResult {
private Item item;
private QueryResultWithSchemaResp result;
}

View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.headless.api.query.request;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.headless.api.model.pojo.Item;
import lombok.Data;
@Data
public class QueryApiPreviewReq {
private Item item;
private DateConf dateConf = new DateConf();
}

View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.headless.api.query.request;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.ApiItemType;
import lombok.Data;
import javax.validation.constraints.NotEmpty;
import java.util.List;
@Data
public class QueryApiReq {
@NotEmpty(message = "ids不可为空")
private List<Long> ids;
private ApiItemType type = ApiItemType.METRIC;
private DateConf dateConf = new DateConf();
}

View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.headless.api.query.response;
import com.tencent.supersonic.headless.api.query.pojo.ApiQuerySingleResult;
import lombok.Builder;
import lombok.Data;
import java.util.List;
@Data
@Builder
public class ApiQueryResultResp {
private List<ApiQuerySingleResult> results;
}

View File

@@ -0,0 +1,202 @@
package com.tencent.supersonic.headless.model.application;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.PageUtils;
import com.tencent.supersonic.headless.api.model.enums.AppStatusEnum;
import com.tencent.supersonic.headless.api.model.pojo.AppConfig;
import com.tencent.supersonic.headless.api.model.request.AppQueryReq;
import com.tencent.supersonic.headless.api.model.request.AppReq;
import com.tencent.supersonic.headless.api.model.response.AppDetailResp;
import com.tencent.supersonic.headless.api.model.response.AppResp;
import com.tencent.supersonic.headless.api.model.response.DimensionResp;
import com.tencent.supersonic.headless.api.model.response.MetricResp;
import com.tencent.supersonic.headless.model.domain.AppService;
import com.tencent.supersonic.headless.model.domain.DimensionService;
import com.tencent.supersonic.headless.model.domain.MetricService;
import com.tencent.supersonic.headless.model.domain.dataobject.AppDO;
import com.tencent.supersonic.headless.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.headless.model.infrastructure.mapper.AppMapper;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@Service
public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements AppService {
private AppMapper appMapper;
private MetricService metricService;
private DimensionService dimensionService;
public AppServiceImpl(AppMapper appMapper, MetricService metricService,
DimensionService dimensionService) {
this.appMapper = appMapper;
this.metricService = metricService;
this.dimensionService = dimensionService;
}
@Override
public AppDetailResp save(AppReq app, User user) {
app.createdBy(user.getName());
AppDO appDO = new AppDO();
BeanMapper.mapper(app, appDO);
appDO.setStatus(AppStatusEnum.INIT.getCode());
appDO.setConfig(JSONObject.toJSONString(app.getConfig()));
appDO.setAppSecret(getUniqueId());
appMapper.insert(appDO);
return convertDetail(appDO);
}
@Override
public AppDetailResp update(AppReq app, User user) {
app.updatedBy(user.getName());
AppDO appDO = getById(app.getId());
checkAuth(appDO, user);
BeanMapper.mapper(app, appDO);
appDO.setConfig(JSONObject.toJSONString(app.getConfig()));
appMapper.updateById(appDO);
return convertDetail(appDO);
}
@Override
public void online(Integer id, User user) {
AppDO appDO = getAppDO(id);
checkAuth(appDO, user);
appDO.setStatus(AppStatusEnum.ONLINE.getCode());
appDO.setUpdatedAt(new Date());
appDO.setUpdatedBy(user.getName());
updateById(appDO);
}
@Override
public void offline(Integer id, User user) {
AppDO appDO = getAppDO(id);
checkAuth(appDO, user);
appDO.setStatus(AppStatusEnum.OFFLINE.getCode());
appDO.setUpdatedAt(new Date());
appDO.setUpdatedBy(user.getName());
updateById(appDO);
}
@Override
public void delete(Integer id, User user) {
AppDO appDO = getAppDO(id);
checkAuth(appDO, user);
appDO.setStatus(AppStatusEnum.DELETED.getCode());
appDO.setUpdatedAt(new Date());
appDO.setUpdatedBy(user.getName());
updateById(appDO);
}
@Override
public PageInfo<AppResp> pageApp(AppQueryReq appQueryReq, User user) {
PageInfo<AppDO> appDOPageInfo = PageHelper.startPage(appQueryReq.getCurrent(),
appQueryReq.getPageSize())
.doSelectPageInfo(() -> queryApp(appQueryReq));
PageInfo<AppResp> appPageInfo = PageUtils.pageInfo2PageInfoVo(appDOPageInfo);
Map<Long, MetricResp> metricResps = metricService.getMetrics(new MetaFilter())
.stream().collect(Collectors.toMap(MetricResp::getId, m -> m));
Map<Long, DimensionResp> dimensionResps = dimensionService.getDimensions(new MetaFilter())
.stream().collect(Collectors.toMap(DimensionResp::getId, m -> m));
appPageInfo.setList(appDOPageInfo.getList().stream().map(appDO
-> convert(appDO, dimensionResps, metricResps, user))
.collect(Collectors.toList()));
return appPageInfo;
}
public List<AppDO> queryApp(AppQueryReq appQueryReq) {
QueryWrapper<AppDO> appDOQueryWrapper = new QueryWrapper<>();
appDOQueryWrapper.lambda().ne(AppDO::getStatus, AppStatusEnum.DELETED.getCode());
if (StringUtils.isNotBlank(appQueryReq.getName())) {
appDOQueryWrapper.lambda().like(AppDO::getName, appQueryReq.getName());
}
if (!CollectionUtils.isEmpty(appQueryReq.getStatus())) {
appDOQueryWrapper.lambda().in(AppDO::getStatus, appQueryReq.getStatus());
}
if (StringUtils.isNotBlank(appQueryReq.getCreatedBy())) {
appDOQueryWrapper.lambda().eq(AppDO::getCreatedBy, appQueryReq.getCreatedBy());
}
return list(appDOQueryWrapper);
}
@Override
public AppDetailResp getApp(Integer id, User user) {
AppDO appDO = getAppDO(id);
checkAuth(appDO, user);
return convertDetail(appDO);
}
@Override
public AppDetailResp getApp(Integer id) {
AppDO appDO = getAppDO(id);
return convertDetail(appDO);
}
private AppDO getAppDO(Integer id) {
AppDO appDO = getById(id);
if (appDO == null) {
throw new InvalidArgumentException("该应用不存在");
}
return appDO;
}
private void checkAuth(AppDO appDO, User user) {
if (!hasAuth(appDO, user)) {
throw new InvalidPermissionException("您不是该应用的负责人, 暂无权查看或者修改");
}
}
private boolean hasAuth(AppDO appDO, User user) {
if (appDO.getCreatedBy().equalsIgnoreCase(user.getName())) {
return true;
}
return StringUtils.isNotBlank(appDO.getOwner())
&& appDO.getOwner().contains(user.getName());
}
private AppResp convert(AppDO appDO, Map<Long, DimensionResp> dimensionMap,
Map<Long, MetricResp> metricMap, User user) {
AppResp app = new AppResp();
BeanMapper.mapper(appDO, app);
AppConfig appConfig = JSONObject.parseObject(appDO.getConfig(), AppConfig.class);
appConfig.getItems().forEach(metricItem -> {
metricItem.setName(metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getName());
metricItem.getRelateItems().forEach(dimensionItem -> {
dimensionItem.setName(dimensionMap.getOrDefault(dimensionItem.getId(), new DimensionResp()).getName());
});
});
app.setConfig(appConfig);
app.setAppStatus(AppStatusEnum.fromCode(appDO.getStatus()));
app.setHasAdminRes(hasAuth(appDO, user));
return app;
}
private AppDetailResp convertDetail(AppDO appDO) {
AppDetailResp app = new AppDetailResp();
BeanMapper.mapper(appDO, app);
app.setConfig(JSONObject.parseObject(appDO.getConfig(), AppConfig.class));
app.setAppStatus(AppStatusEnum.fromCode(appDO.getStatus()));
return app;
}
private String getUniqueId() {
return UUID.randomUUID().toString().replaceAll("_", "");
}
}

View File

@@ -242,7 +242,8 @@ public class MetricServiceImpl implements MetricService {
return metricResp;
}
private MetricResp getMetric(Long id) {
@Override
public MetricResp getMetric(Long id) {
MetricDO metricDO = metricRepository.getMetricById(id);
if (metricDO == null) {
return null;

View File

@@ -0,0 +1,28 @@
package com.tencent.supersonic.headless.model.domain;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.model.request.AppQueryReq;
import com.tencent.supersonic.headless.api.model.request.AppReq;
import com.tencent.supersonic.headless.api.model.response.AppDetailResp;
import com.tencent.supersonic.headless.api.model.response.AppResp;
public interface AppService {
AppDetailResp save(AppReq app, User user);
AppDetailResp update(AppReq app, User user);
void online(Integer id, User user);
void offline(Integer id, User user);
void delete(Integer id, User user);
PageInfo<AppResp> pageApp(AppQueryReq appQueryReq, User user);
AppDetailResp getApp(Integer id, User user);
AppDetailResp getApp(Integer id);
}

View File

@@ -34,6 +34,8 @@ public interface MetricService {
MetricResp getMetric(Long id, User user);
MetricResp getMetric(Long id);
List<String> mockAlias(MetricReq metricReq, String mockType, User user);
Set<String> getMetricTags();

View File

@@ -0,0 +1,41 @@
package com.tencent.supersonic.headless.model.domain.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_app")
public class AppDO {
@TableId(type = IdType.AUTO)
private Long id;
private String name;
private String description;
private String config;
private Date endDate;
private Integer qps;
private String owner;
private Integer status;
private String appSecret;
private String createdBy;
private String updatedBy;
private Date createdAt;
private Date updatedAt;
}

View File

@@ -22,6 +22,12 @@ public class MetricQueryDefaultConfigDO {
private String defaultConfig;
private String appKey;
private String appSecret;
private String owner;
private Date createdAt;
private String createdBy;

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.model.infrastructure.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.model.domain.dataobject.AppDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface AppMapper extends BaseMapper<AppDO> {
}

View File

@@ -0,0 +1,84 @@
package com.tencent.supersonic.headless.model.rest;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.model.request.AppQueryReq;
import com.tencent.supersonic.headless.api.model.request.AppReq;
import com.tencent.supersonic.headless.api.model.response.AppDetailResp;
import com.tencent.supersonic.headless.api.model.response.AppResp;
import com.tencent.supersonic.headless.model.domain.AppService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestController
@RequestMapping("/api/semantic/app")
public class AppController {
@Autowired
private AppService appService;
@PostMapping
public boolean save(@RequestBody AppReq app,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.save(app, user);
return true;
}
@PutMapping
public boolean update(@RequestBody AppReq app,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.update(app, user);
return true;
}
@PutMapping("/online/{id}")
public boolean online(@PathVariable("id") Integer id,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.online(id, user);
return true;
}
@PutMapping("/offline/{id}")
public boolean offline(@PathVariable("id") Integer id,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.offline(id, user);
return true;
}
@DeleteMapping("/{id}")
public boolean delete(@PathVariable("id") Integer id,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.delete(id, user);
return true;
}
@GetMapping("/{id}")
public AppDetailResp getApp(@PathVariable("id") Integer id,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return appService.getApp(id, user);
}
@PostMapping("/page")
public PageInfo<AppResp> pageApp(@RequestBody AppQueryReq appQueryReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return appService.pageApp(appQueryReq, user);
}
}

View File

@@ -4,6 +4,7 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.headless.api.model.enums.DimensionTypeEnum;
import com.tencent.supersonic.headless.api.model.enums.IdentifyTypeEnum;
import com.tencent.supersonic.headless.api.model.pojo.Dim;
@@ -13,14 +14,14 @@ import com.tencent.supersonic.headless.api.model.pojo.Measure;
import com.tencent.supersonic.headless.api.model.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.model.request.ModelReq;
import com.tencent.supersonic.headless.api.model.response.ModelResp;
import com.tencent.supersonic.headless.model.domain.DatabaseService;
import com.tencent.supersonic.headless.model.domain.DimensionService;
import com.tencent.supersonic.headless.model.domain.DomainService;
import com.tencent.supersonic.headless.model.domain.ModelService;
import com.tencent.supersonic.headless.model.domain.repository.DateInfoRepository;
import com.tencent.supersonic.headless.model.domain.DatabaseService;
import com.tencent.supersonic.headless.model.domain.MetricService;
import com.tencent.supersonic.headless.model.domain.ModelRelaService;
import com.tencent.supersonic.headless.model.domain.ModelService;
import com.tencent.supersonic.headless.model.domain.dataobject.ModelDO;
import com.tencent.supersonic.headless.model.domain.repository.DateInfoRepository;
import com.tencent.supersonic.headless.model.domain.repository.ModelRepository;
import com.tencent.supersonic.headless.model.domain.utils.ModelConverter;
import org.junit.jupiter.api.Assertions;
@@ -160,6 +161,8 @@ class ModelServiceImplTest {
modelResp.setAlias("访问次数统计,PVUV统计");
modelResp.setAdmins(Lists.newArrayList("admin", "tom"));
modelResp.setViewers(Lists.newArrayList("alice", "lucy"));
modelResp.setStatus(StatusEnum.ONLINE.getCode());
modelResp.createdBy("admin");
ModelDetail modelDetail = new ModelDetail();
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name"));

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.query.service;
package com.tencent.supersonic.headless.query.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
@@ -7,6 +7,6 @@ import java.lang.annotation.Target;
@Target({ElementType.PARAMETER, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface DataPermission {
public @interface ApiDataPermission {
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.query.utils;
package com.tencent.supersonic.headless.query.annotation;
import java.lang.annotation.Target;
import java.lang.annotation.RetentionPolicy;
@@ -9,6 +9,6 @@ import java.lang.annotation.Documented;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface S2SQLPermissionAnnotation {
public @interface S2SQLDataPermission {
}

View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.headless.query.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ElementType.PARAMETER, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface StructDataPermission {
}

View File

@@ -0,0 +1,73 @@
package com.tencent.supersonic.headless.query.aspect;
import com.tencent.supersonic.common.pojo.Pair;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.SignatureUtils;
import com.tencent.supersonic.headless.api.model.enums.AppStatusEnum;
import com.tencent.supersonic.headless.api.model.response.AppDetailResp;
import com.tencent.supersonic.headless.model.domain.AppService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import javax.servlet.http.HttpServletRequest;
@Component
@Aspect
@Order(1)
@Slf4j
public class ApiDataAspect {
public static final String APPID = "appId";
private static final String TIMESTAMP = "timestamp";
private static final String SIGNATURE = "signature";
@Autowired
private AppService appService;
@Pointcut("@annotation(com.tencent.supersonic.headless.query.annotation.ApiDataPermission)")
private void apiPermissionCheck() {
}
@Around("apiPermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] objects = joinPoint.getArgs();
HttpServletRequest request = (HttpServletRequest) objects[1];
checkHeader(request);
return joinPoint.proceed();
}
private void checkHeader(HttpServletRequest request) {
String timestampStr = request.getHeader(TIMESTAMP);
String signature = request.getHeader(SIGNATURE);
String appId = request.getHeader(APPID);
if (StringUtils.isBlank(timestampStr)) {
throw new InvalidArgumentException("header中timestamp不可为空");
}
if (StringUtils.isBlank(signature)) {
throw new InvalidArgumentException("header中signature不可为空");
}
if (StringUtils.isBlank(appId)) {
throw new InvalidArgumentException("header中appId不可为空");
}
AppDetailResp appDetailResp = appService.getApp(Integer.parseInt(appId));
if (appDetailResp == null) {
throw new InvalidArgumentException("该appId对应的应用不存在");
}
if (!AppStatusEnum.ONLINE.equals(appDetailResp.getAppStatus())) {
throw new InvalidArgumentException("该应用暂时为非在线状态");
}
Pair<Boolean, String> checkResult = SignatureUtils.isValidSignature(appId, appDetailResp.getAppSecret(),
Long.parseLong(timestampStr), signature);
if (!checkResult.first) {
throw new InvalidArgumentException(checkResult.second);
}
}
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.query.utils;
package com.tencent.supersonic.headless.query.aspect;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@@ -16,6 +16,7 @@ import com.tencent.supersonic.headless.model.domain.ModelService;
import com.tencent.supersonic.headless.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.headless.model.domain.pojo.ModelFilter;
import com.tencent.supersonic.headless.query.service.AuthCommonService;
import com.tencent.supersonic.headless.query.utils.QueryStructUtils;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -58,7 +59,7 @@ public class S2SQLDataAspect {
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.headless.query.utils.S2SQLPermissionAnnotation)")
@Pointcut("@annotation(com.tencent.supersonic.headless.query.annotation.S2SQLDataPermission)")
private void s2SQLPermissionCheck() {
}

View File

@@ -1,6 +1,4 @@
package com.tencent.supersonic.headless.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
package com.tencent.supersonic.headless.query.aspect;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@@ -17,14 +15,7 @@ import com.tencent.supersonic.headless.model.domain.DimensionService;
import com.tencent.supersonic.headless.model.domain.ModelService;
import com.tencent.supersonic.headless.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.headless.query.service.AuthCommonService;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import com.tencent.supersonic.headless.query.utils.QueryStructUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
@@ -36,10 +27,21 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Component
@Aspect
@Slf4j
public class DataPermissionAOP {
public class StructDataAspect {
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
@@ -51,7 +53,7 @@ public class DataPermissionAOP {
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.headless.query.service.DataPermission)")
@Pointcut("@annotation(com.tencent.supersonic.headless.query.annotation.StructDataPermission)")
public void dataPermissionAOP() {
}

View File

@@ -0,0 +1,34 @@
package com.tencent.supersonic.headless.query.rest;
import com.tencent.supersonic.headless.api.query.request.QueryApiPreviewReq;
import com.tencent.supersonic.headless.api.query.request.QueryApiReq;
import com.tencent.supersonic.headless.api.query.response.ApiQueryResultResp;
import com.tencent.supersonic.headless.query.service.ApiQueryService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
@RestController
@RequestMapping("/api/semantic/apiQuery")
@Slf4j
public class ApiQueryController {
@Autowired
private ApiQueryService apiQueryService;
@PostMapping("/preview")
public ApiQueryResultResp preview(@RequestBody QueryApiPreviewReq queryApiReq) throws Exception {
return apiQueryService.preview(queryApiReq);
}
@PostMapping("/query")
public ApiQueryResultResp query(@RequestBody QueryApiReq queryApiReq, HttpServletRequest request) throws Exception {
return apiQueryService.query(queryApiReq, request);
}
}

View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.headless.query.service;
import com.tencent.supersonic.headless.api.query.request.QueryApiPreviewReq;
import com.tencent.supersonic.headless.api.query.request.QueryApiReq;
import com.tencent.supersonic.headless.api.query.response.ApiQueryResultResp;
import com.tencent.supersonic.headless.query.annotation.ApiDataPermission;
import javax.servlet.http.HttpServletRequest;
public interface ApiQueryService {
ApiQueryResultResp preview(QueryApiPreviewReq queryApiReq) throws Exception;
@ApiDataPermission
ApiQueryResultResp query(QueryApiReq queryApiReq, HttpServletRequest request) throws Exception;
}

View File

@@ -0,0 +1,135 @@
package com.tencent.supersonic.headless.query.service;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.model.pojo.Item;
import com.tencent.supersonic.headless.api.model.response.AppDetailResp;
import com.tencent.supersonic.headless.api.model.response.DimensionResp;
import com.tencent.supersonic.headless.api.model.response.MetricResp;
import com.tencent.supersonic.headless.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.query.pojo.ApiQuerySingleResult;
import com.tencent.supersonic.headless.api.query.request.QueryApiPreviewReq;
import com.tencent.supersonic.headless.api.query.request.QueryApiReq;
import com.tencent.supersonic.headless.api.query.request.QueryStructReq;
import com.tencent.supersonic.headless.api.query.response.ApiQueryResultResp;
import com.tencent.supersonic.headless.model.domain.AppService;
import com.tencent.supersonic.headless.model.domain.DimensionService;
import com.tencent.supersonic.headless.model.domain.MetricService;
import com.tencent.supersonic.headless.model.domain.pojo.DimensionFilter;
import com.tencent.supersonic.headless.query.annotation.ApiDataPermission;
import com.tencent.supersonic.headless.query.aspect.ApiDataAspect;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.servlet.http.HttpServletRequest;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Api service for other apps
* The current version defaults to query metrics data.
*/
@Service
public class ApiQueryServiceImpl implements ApiQueryService {
private static final long result_size = 10000;
private AppService appService;
private MetricService metricService;
private DimensionService dimensionService;
private QueryService queryService;
public ApiQueryServiceImpl(AppService appService,
MetricService metricService,
DimensionService dimensionService,
QueryService queryService) {
this.appService = appService;
this.metricService = metricService;
this.dimensionService = dimensionService;
this.queryService = queryService;
}
@Override
public ApiQueryResultResp preview(QueryApiPreviewReq queryApiReq) throws Exception {
Item item = queryApiReq.getItem();
ApiQuerySingleResult apiQuerySingleResult = query(item, queryApiReq.getDateConf());
return ApiQueryResultResp.builder().results(Lists.newArrayList(apiQuerySingleResult)).build();
}
public ApiQuerySingleResult query(Item item, DateConf dateConf) throws Exception {
MetricResp metricResp = metricService.getMetric(item.getId());
List<Item> items = item.getRelateItems();
List<DimensionResp> dimensionResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(items)) {
List<Long> ids = items.stream().map(Item::getId).collect(Collectors.toList());
DimensionFilter dimensionFilter = new DimensionFilter();
dimensionFilter.setIds(ids);
dimensionResps = dimensionService.getDimensions(dimensionFilter);
}
QueryStructReq queryStructReq = buildQueryStructReq(dimensionResps, metricResp, dateConf);
QueryResultWithSchemaResp queryResultWithSchemaResp =
queryService.queryByStruct(queryStructReq, User.getAppUser(0));
ApiQuerySingleResult apiQuerySingleResult = new ApiQuerySingleResult();
apiQuerySingleResult.setItem(item);
apiQuerySingleResult.setResult(queryResultWithSchemaResp);
return apiQuerySingleResult;
}
@Override
@ApiDataPermission
public ApiQueryResultResp query(QueryApiReq queryApiReq, HttpServletRequest request) throws Exception {
int appId = Integer.parseInt(request.getHeader(ApiDataAspect.APPID));
AppDetailResp appDetailResp = appService.getApp(appId);
Set<Long> idsInApp = appDetailResp.getConfig().getItems().stream()
.map(Item::getId).collect(Collectors.toSet());
if (!idsInApp.containsAll(queryApiReq.getIds())) {
throw new InvalidArgumentException("查询范围超过应用申请范围, 请检查");
}
List<ApiQuerySingleResult> results = Lists.newArrayList();
Map<Long, Item> map = appDetailResp.getConfig().getItems().stream()
.collect(Collectors.toMap(Item::getId, i -> i));
for (Long id : queryApiReq.getIds()) {
Item item = map.get(id);
ApiQuerySingleResult apiQuerySingleResult = query(item, queryApiReq.getDateConf());
results.add(apiQuerySingleResult);
}
return ApiQueryResultResp.builder().results(results).build();
}
private QueryStructReq buildQueryStructReq(List<DimensionResp> dimensionResps,
MetricResp metricResp, DateConf dateConf) {
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
modelIds.add(metricResp.getModelId());
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setGroups(dimensionResps.stream()
.map(DimensionResp::getBizName).collect(Collectors.toList()));
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricResp.getBizName());
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
queryStructReq.setDateInfo(dateConf);
queryStructReq.setModelIds(modelIds);
queryStructReq.setLimit(result_size);
return queryStructReq;
}
private String getTimeDimension(DateConf dateConf) {
if (Constants.MONTH.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.MONTH.getName();
} else if (Constants.WEEK.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.WEEK.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
}
}

View File

@@ -30,8 +30,9 @@ import com.tencent.supersonic.headless.model.domain.Catalog;
import com.tencent.supersonic.headless.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.headless.query.executor.QueryExecutor;
import com.tencent.supersonic.headless.query.parser.convert.QueryReqConverter;
import com.tencent.supersonic.headless.query.annotation.StructDataPermission;
import com.tencent.supersonic.headless.query.utils.QueryUtils;
import com.tencent.supersonic.headless.query.utils.S2SQLPermissionAnnotation;
import com.tencent.supersonic.headless.query.annotation.S2SQLDataPermission;
import com.tencent.supersonic.headless.query.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@@ -80,7 +81,7 @@ public class QueryServiceImpl implements QueryService {
}
@Override
@S2SQLPermissionAnnotation
@S2SQLDataPermission
@SneakyThrows
public Object queryBySql(QueryS2SQLReq queryS2SQLReq, User user) {
statUtils.initStatInfo(queryS2SQLReq, user);
@@ -150,7 +151,7 @@ public class QueryServiceImpl implements QueryService {
}
@Override
@DataPermission
@StructDataPermission
@SneakyThrows
public QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) {
return queryByStruct(queryStructCmd, user);

View File

@@ -541,3 +541,19 @@ CREATE TABLE `s2_metric_query_default_config` (
`updated_by` varchar(100) not null,
PRIMARY KEY (`id`)
);
CREATE TABLE `s2_app` (
id bigint AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(255),
description VARCHAR(255),
status INT,
config TEXT,
end_date TIMESTAMP,
qps INT,
app_secret VARCHAR(255),
owner VARCHAR(255),
created_at TIMESTAMP,
created_by VARCHAR(255),
updated_at TIMESTAMP,
updated_by VARCHAR(255)
);

View File

@@ -525,3 +525,20 @@ CREATE TABLE `s2_metric_query_default_config` (
`created_by` varchar(100) null,
`updated_by` varchar(100) null
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `s2_app`
(
id bigint primary key AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(255),
description VARCHAR(255),
status INT,
config TEXT,
end_date TIMESTAMP,
qps INT,
app_secret VARCHAR(255),
owner VARCHAR(255),
created_at TIMESTAMP,
created_by VARCHAR(255),
updated_at TIMESTAMP,
updated_by VARCHAR(255)
);

View File

@@ -145,4 +145,22 @@ CREATE TABLE `s2_metric_query_default_config`
--20231214
alter table s2_chat_query add column `similar_queries` varchar(1024) DEFAULT '';
alter table s2_model add column `source_type` varchar(128) DEFAULT NULL;
alter table s2_model add column `source_type` varchar(128) DEFAULT NULL;
CREATE TABLE `s2_app`
(
id bigint primary key AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(255),
description VARCHAR(255),
status INT,
config TEXT,
end_date TIMESTAMP,
qps INT,
app_secret VARCHAR(255),
owner VARCHAR(255),
created_at TIMESTAMP,
created_by VARCHAR(255),
updated_at TIMESTAMP,
updated_by VARCHAR(255)
);

View File

@@ -545,4 +545,22 @@ CREATE TABLE `s2_metric_query_default_config`(
`created_by` varchar(100) null,
`updated_by` varchar(100) not null,
PRIMARY KEY (`id`)
);
CREATE TABLE `s2_app`
(
id bigint AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(255),
description VARCHAR(255),
status INT,
config TEXT,
end_date TIMESTAMP,
qps INT,
app_key VARCHAR(255),
app_secret VARCHAR(255),
owner VARCHAR(255),
created_at TIMESTAMP,
created_by VARCHAR(255),
updated_at TIMESTAMP,
updated_by VARCHAR(255)
);