(improvement)(headless) Supports creating new metric by fields and metrics (#639)

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-01-17 14:21:48 +08:00
committed by GitHub
parent b2beecb5b8
commit 07e6924cfd
40 changed files with 647 additions and 221 deletions

View File

@@ -6,6 +6,7 @@ public enum StatusEnum {
ONLINE("ONLINE", 1),
OFFLINE("OFFLINE", 2),
DELETED("DELETED", 3),
UNAVAILABLE("UNAVAILABLE", 4),
UNKNOWN("UNKNOWN", -1);

View File

@@ -1,11 +1,5 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
@@ -19,6 +13,12 @@ import net.sf.jsqlparser.statement.select.SelectItem;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/**
* Sql Parser Select function Helper
*/
@@ -32,6 +32,11 @@ public class SqlParserSelectFunctionHelper {
return SqlParserSelectHelper.hasGroupBy(sql);
}
public static void main(String[] args) {
String sql = "select a from table";
System.out.println(hasAggregateFunction(sql));
}
public static boolean hasFunction(String sql, String functionName) {
Set<String> functions = getFunctions(sql);
if (!CollectionUtils.isEmpty(functions)) {

View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.headless.api.enums;
public enum MetricDefineType {
FIELD,
MEASURE,
METRIC
}

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public class Field {
private String fieldName;
}

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public class FieldParam {
private String fieldName;
}

View File

@@ -32,6 +32,11 @@ public class Measure {
this.bizName = bizName;
}
public Measure(String bizName, String constraint) {
this.bizName = bizName;
this.constraint = constraint;
}
public String getFieldName() {
return expr;
}

View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class MeasureParam {
private String bizName;
private String constraint;
private String agg;
public MeasureParam(String bizName, String constraint) {
this.bizName = bizName;
this.constraint = constraint;
}
}

View File

@@ -1,14 +1,13 @@
package com.tencent.supersonic.headless.api.pojo;
import java.util.List;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class MetricTypeParams {
public class MetricDefineByFieldParams extends MetricDefineParams {
private List<Measure> measures = Lists.newArrayList();
private String expr;
private List<FieldParam> fields = Lists.newArrayList();
}

View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.headless.api.pojo;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class MetricDefineByMeasureParams extends MetricDefineParams {
private List<MeasureParam> measures = Lists.newArrayList();
}

View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.headless.api.pojo;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class MetricDefineByMetricParams extends MetricDefineParams {
private List<MetricParam> metrics = Lists.newArrayList();
}

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public abstract class MetricDefineParams {
private String expr;
}

View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public class MetricParam {
private Long id;
private String bizName;
}

View File

@@ -25,6 +25,8 @@ public class ModelDetail {
private List<Measure> measures;
private List<Field> fields;
public String getSqlQuery() {
if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) {
sqlQuery = sqlQuery.substring(0, sqlQuery.length() - 1);
@@ -32,7 +34,7 @@ public class ModelDetail {
return sqlQuery;
}
public List<Dim> getTimeDims() {
public List<Dim> filterTimeDims() {
if (CollectionUtils.isEmpty(dimensions)) {
return Lists.newArrayList();
}

View File

@@ -4,13 +4,14 @@ import com.google.common.base.Objects;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import lombok.Data;
import lombok.ToString;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Data
@ToString(callSuper = true)
public class SchemaItem extends RecordInfo {
@@ -38,9 +39,6 @@ public class SchemaItem extends RecordInfo {
if (o == null || getClass() != o.getClass()) {
return false;
}
if (!super.equals(o)) {
return false;
}
SchemaItem that = (SchemaItem) o;
return Objects.equal(id, that.id) && Objects.equal(name, that.name)
&& Objects.equal(bizName, that.bizName) && Objects.equal(
@@ -50,7 +48,7 @@ public class SchemaItem extends RecordInfo {
@Override
public int hashCode() {
return Objects.hashCode(super.hashCode(), id, name, bizName, description, status, typeEnum, sensitiveLevel);
return Objects.hashCode(id, name, bizName, description, status, typeEnum, sensitiveLevel);
}
public static List<String> getAliasList(String alias) {

View File

@@ -30,6 +30,9 @@ public class MetricBaseReq extends SchemaItem {
private Map<String, Object> ext = new HashMap<>();
public String getTag() {
if (tags == null) {
return null;
}
if (CollectionUtils.isEmpty(tags)) {
return "";
}

View File

@@ -1,30 +1,49 @@
package com.tencent.supersonic.headless.api.request;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import lombok.Data;
import java.util.List;
@Data
public class MetricReq extends MetricBaseReq {
private MetricType metricType;
private MetricDefineType metricDefineType = MetricDefineType.MEASURE;
private MetricDefineByMeasureParams typeParams;
private MetricDefineByFieldParams metricDefineByFieldParams;
private MetricDefineByMetricParams metricDefineByMetricParams;
private MetricTypeParams typeParams;
public String getTypeParamsJson() {
if (metricDefineByFieldParams != null) {
return JSONObject.toJSONString(metricDefineByFieldParams);
} else if (typeParams != null) {
return JSONObject.toJSONString(typeParams);
} else if (metricDefineByMetricParams != null) {
return JSONObject.toJSONString(metricDefineByMetricParams);
}
return null;
}
public MetricType getMetricType() {
if (metricType != null) {
return metricType;
}
List<Measure> measureList = typeParams.getMeasures();
if (measureList.size() == 1 && typeParams.getExpr().trim().equalsIgnoreCase(measureList.get(0).getBizName())) {
return MetricType.ATOMIC;
} else if (measureList.size() >= 1) {
if (MetricDefineType.METRIC.equals(metricDefineType)) {
return MetricType.DERIVED;
}
throw new RuntimeException("measure can not be none");
if (MetricDefineType.MEASURE.equals(metricDefineType)) {
List<MeasureParam> measures = typeParams.getMeasures();
if (measures.size() > 1) {
return MetricType.DERIVED;
}
if (measures.size() == 1 && measures.get(0).getBizName()
.equalsIgnoreCase(typeParams.getExpr())) {
return MetricType.ATOMIC;
}
}
return MetricType.ATOMIC;
}
}

View File

@@ -42,7 +42,7 @@ public class ModelReq extends SchemaItem {
if (modelDetail == null) {
return Lists.newArrayList();
}
return modelDetail.getTimeDims();
return modelDetail.filterTimeDims();
}
public String getViewer() {

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.api.response;
import com.tencent.supersonic.headless.api.pojo.Entity;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import lombok.Data;
import lombok.ToString;
@@ -26,12 +25,6 @@ public class DomainResp extends SchemaItem {
private Integer isOpen = 0;
private Integer dimensionCnt;
private Integer metricCnt;
private Entity entity;
private boolean hasEditPermission = false;
@Override

View File

@@ -2,21 +2,23 @@ package com.tencent.supersonic.headless.api.response;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import lombok.Data;
import lombok.ToString;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@@ -33,8 +35,6 @@ public class MetricResp extends SchemaItem {
//ATOMIC DERIVED
private String type;
private MetricTypeParams typeParams;
private String dataFormatType;
private DataFormat dataFormat;
@@ -51,6 +51,14 @@ public class MetricResp extends SchemaItem {
private Map<String, Object> ext = new HashMap<>();
private MetricDefineType metricDefineType = MetricDefineType.MEASURE;
private MetricDefineByMeasureParams typeParams;
private MetricDefineByFieldParams metricDefineByFieldParams;
private MetricDefineByMetricParams metricDefineByMetricParams;
public void setTag(String tag) {
if (StringUtils.isBlank(tag)) {
tags = Lists.newArrayList();
@@ -59,14 +67,6 @@ public class MetricResp extends SchemaItem {
}
}
public Set<Long> getNecessaryDimensionIds() {
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
return Sets.newHashSet();
}
return relateDimension.getDrillDownDimensions().stream().filter(DrillDownDimension::isNecessary)
.map(DrillDownDimension::getDimensionId).collect(Collectors.toSet());
}
public String getRelaDimensionIdKey() {
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
return "";
@@ -78,6 +78,11 @@ public class MetricResp extends SchemaItem {
}
public String getDefaultAgg() {
return typeParams.getMeasures().get(0).getAgg();
if (typeParams != null
&& CollectionUtils.isNotEmpty(typeParams.getMeasures())) {
return typeParams.getMeasures().get(0).getAgg();
}
return "";
}
}

View File

@@ -3,16 +3,22 @@ package com.tencent.supersonic.headless.api.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.Identify;
import lombok.Data;
import lombok.ToString;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Data
@ToString(callSuper = true)
public class ModelResp extends SchemaItem {
private Long domainId;
@@ -66,7 +72,27 @@ public class ModelResp extends SchemaItem {
if (modelDetail == null) {
return Lists.newArrayList();
}
return modelDetail.getTimeDims();
return modelDetail.filterTimeDims();
}
public Set<String> getFieldList() {
Set<String> fieldSet = new HashSet<>();
if (modelDetail == null) {
return fieldSet;
}
if (!CollectionUtils.isEmpty(modelDetail.getIdentifiers())) {
fieldSet.addAll(modelDetail.getIdentifiers().stream()
.map(Identify::getFieldName).collect(Collectors.toSet()));
}
if (!CollectionUtils.isEmpty(modelDetail.getDimensions())) {
fieldSet.addAll(modelDetail.getDimensions().stream()
.map(Dim::getFieldName).collect(Collectors.toSet()));
}
if (!CollectionUtils.isEmpty(modelDetail.getMeasures())) {
fieldSet.addAll(modelDetail.getMeasures().stream()
.map(Measure::getFieldName).collect(Collectors.toSet()));
}
return fieldSet;
}
}

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.headless.core.manager;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
@@ -37,21 +37,21 @@ public class MetricYamlManager {
BeanUtils.copyProperties(metric, metricYamlTpl);
metricYamlTpl.setName(metric.getBizName());
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
MetricTypeParams exprMetricTypeParams = metric.getTypeParams();
MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams();
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
metricTypeParamsYamlTpl.setExpr(exprMetricTypeParams.getExpr());
List<Measure> measures = exprMetricTypeParams.getMeasures();
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
List<MeasureParam> measures = metricDefineParams.getMeasures();
metricTypeParamsYamlTpl.setMeasures(
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl);
return metricYamlTpl;
}
public static MeasureYamlTpl convert(Measure measure) {
public static MeasureYamlTpl convert(MeasureParam measure) {
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
measureYamlTpl.setName(measure.getBizName());
measureYamlTpl.setConstraint(measure.getConstraint());
measureYamlTpl.setAgg(measure.getAlias());
measureYamlTpl.setAgg(measure.getAgg());
return measureYamlTpl;
}

View File

@@ -94,7 +94,7 @@ public class ModelYamlManager {
}
measure.setAgg("count");
measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt"));
measure.setCreateMetric("true");
measure.setIsCreateMetric(1);
datasourceDetail.getMeasures().add(measure);
}

View File

@@ -316,9 +316,9 @@ public class SourceRender extends Renderer {
}
}
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope,
public void render(MetricQueryReq metricQueryReq, List<DataSource> dataSources, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere();
String queryWhere = metricQueryReq.getWhere();
Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>();
if (queryWhere != null && !queryWhere.isEmpty()) {
@@ -328,13 +328,13 @@ public class SourceRender extends Renderer {
}
if (dataSources.size() == 1) {
DataSource dataSource = dataSources.get(0);
super.tableView = renderOne("", fieldWhere, metricCommand.getMetrics(),
metricCommand.getDimensions(),
metricCommand.getWhere(), dataSource, scope, schema, nonAgg);
super.tableView = renderOne("", fieldWhere, metricQueryReq.getMetrics(),
metricQueryReq.getDimensions(),
metricQueryReq.getWhere(), dataSource, scope, schema, nonAgg);
return;
}
JoinRender joinRender = new JoinRender();
joinRender.render(metricCommand, dataSources, scope, schema, nonAgg);
joinRender.render(metricQueryReq, dataSources, scope, schema, nonAgg);
super.tableView = joinRender.getTableView();
}

View File

@@ -101,4 +101,6 @@ public class MetricDO {
private String ext;
private String defineType;
}

View File

@@ -11,8 +11,6 @@ public interface MetricDOCustomMapper {
void batchInsert(List<MetricDO> metricDOS);
void batchUpdate(List<MetricDO> metricDOS);
void batchUpdateStatus(List<MetricDO> metricDOS);
List<MetricDO> query(MetricFilter metricFilter);

View File

@@ -9,6 +9,7 @@ import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.request.PageMetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp;
@@ -33,30 +34,26 @@ import java.util.Set;
@RequestMapping("/api/semantic/metric")
public class MetricController {
private MetricService metricService;
public MetricController(MetricService metricService) {
this.metricService = metricService;
}
@PostMapping("/creatExprMetric")
public Boolean creatExprMetric(@RequestBody MetricReq metricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
public MetricResp createMetric(@RequestBody MetricReq metricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
metricService.createMetric(metricReq, user);
return true;
return metricService.createMetric(metricReq, user);
}
@PostMapping("/updateExprMetric")
public Boolean updateExprMetric(@RequestBody MetricReq metricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
public MetricResp updateMetric(@RequestBody MetricReq metricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
metricService.updateExprMetric(metricReq, user);
return true;
return metricService.updateMetric(metricReq, user);
}
@PostMapping("/batchUpdateStatus")
@@ -69,7 +66,7 @@ public class MetricController {
}
@PostMapping("/mockMetricAlias")
public List<String> mockMetricAlias(@RequestBody MetricReq metricReq,
public List<String> mockMetricAlias(@RequestBody MetricBaseReq metricReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
@@ -82,6 +79,11 @@ public class MetricController {
return metricService.getMetrics(metaFilter);
}
@GetMapping("/getMetricsToCreateNewMetric/{modelId}")
public List<MetricResp> getMetricsToCreateNewMetric(@PathVariable("modelId") Long modelId) {
return metricService.getMetricsToCreateNewMetric(modelId);
}
@PostMapping("/queryMetric")
public PageInfo<MetricResp> queryMetric(@RequestBody PageMetricReq pageMetricReq,
HttpServletRequest request,
@@ -92,13 +94,15 @@ public class MetricController {
@Deprecated
@GetMapping("getMetric/{modelId}/{bizName}")
public MetricResp getMetric(@PathVariable("modelId") Long modelId, @PathVariable("bizName") String bizName) {
public MetricResp getMetric(@PathVariable("modelId") Long modelId,
@PathVariable("bizName") String bizName) {
return metricService.getMetric(modelId, bizName);
}
@GetMapping("getMetric/{id}")
public MetricResp getMetric(@PathVariable("id") Long id,
HttpServletRequest request, HttpServletResponse response) {
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return metricService.getMetric(id, user);
}

View File

@@ -2,11 +2,11 @@ package com.tencent.supersonic.headless.server.service;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.request.PageMetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp;
@@ -17,11 +17,11 @@ import java.util.Set;
public interface MetricService {
void createMetric(MetricReq metricReq, User user) throws Exception;
MetricResp createMetric(MetricReq metricReq, User user) throws Exception;
void createMetricBatch(List<MetricReq> metricReqs, User user) throws Exception;
void updateExprMetric(MetricReq metricReq, User user) throws Exception;
MetricResp updateMetric(MetricReq metricReq, User user) throws Exception;
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
@@ -31,20 +31,20 @@ public interface MetricService {
List<MetricResp> getMetrics(MetaFilter metaFilter);
List<MetricResp> getMetricsToCreateNewMetric(Long modelId);
MetricResp getMetric(Long modelId, String bizName);
MetricResp getMetric(Long id, User user);
MetricResp getMetric(Long id);
List<String> mockAlias(MetricReq metricReq, String mockType, User user);
List<String> mockAlias(MetricBaseReq metricReq, String mockType, User user);
Set<String> getMetricTags();
List<DrillDownDimension> getDrillDownDimension(Long metricId);
List<DataItem> getDataItems(Long modelId);
void saveMetricQueryDefaultConfig(MetricQueryDefaultConfig defaultConfig, User user);
MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user);

View File

@@ -12,14 +12,13 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.request.PageMetricReq;
import com.tencent.supersonic.headless.api.response.DomainResp;
@@ -35,10 +34,9 @@ import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
@@ -83,13 +81,14 @@ public class MetricServiceImpl implements MetricService {
}
@Override
public void createMetric(MetricReq metricReq, User user) {
public MetricResp createMetric(MetricReq metricReq, User user) {
checkExist(Lists.newArrayList(metricReq));
checkParam(metricReq);
MetricCheckUtils.checkParam(metricReq);
metricReq.createdBy(user.getName());
MetricDO metricDO = MetricConverter.convert2MetricDO(metricReq);
metricRepository.createMetric(metricDO);
sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD);
return MetricConverter.convert2MetricResp(metricDO);
}
@Override
@@ -116,8 +115,8 @@ public class MetricServiceImpl implements MetricService {
}
@Override
public void updateExprMetric(MetricReq metricReq, User user) {
checkParam(metricReq);
public MetricResp updateMetric(MetricReq metricReq, User user) {
MetricCheckUtils.checkParam(metricReq);
checkExist(Lists.newArrayList(metricReq));
metricReq.updatedBy(user.getName());
MetricDO metricDO = metricRepository.getMetricById(metricReq.getId());
@@ -130,6 +129,7 @@ public class MetricServiceImpl implements MetricService {
dataItem.setNewName(metricDO.getName());
sendEvent(getDataItem(metricDO), EventType.UPDATE);
}
return MetricConverter.convert2MetricResp(metricDO);
}
@Override
@@ -209,6 +209,17 @@ public class MetricServiceImpl implements MetricService {
return convertList(queryMetric(metricFilter), Lists.newArrayList());
}
@Override
public List<MetricResp> getMetricsToCreateNewMetric(Long modelId) {
MetricFilter metricFilter = new MetricFilter();
metricFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metricFilter);
return metricResps.stream().filter(metricResp ->
MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
.collect(Collectors.toList());
}
private void fillAdminRes(List<MetricResp> metricResps, User user) {
List<ModelResp> modelResps = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelResps)) {
@@ -260,7 +271,7 @@ public class MetricServiceImpl implements MetricService {
}
@Override
public List<String> mockAlias(MetricReq metricReq, String mockType, User user) {
public List<String> mockAlias(MetricBaseReq metricReq, String mockType, User user) {
String mockAlias = chatGptHelper.mockAlias(mockType, metricReq.getName(), metricReq.getBizName(), "",
metricReq.getDescription(), !"".equals(metricReq.getDataFormatType()));
@@ -292,17 +303,6 @@ public class MetricServiceImpl implements MetricService {
return modelResp.getDrillDownDimensions();
}
@Override
public List<DataItem> getDataItems(Long modelId) {
MetricFilter metaFilter = new MetricFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricDO> metricDOS = queryMetric(metaFilter);
if (CollectionUtils.isEmpty(metricDOS)) {
return Lists.newArrayList();
}
return metricDOS.stream().map(this::getDataItem).collect(Collectors.toList());
}
@Override
public void saveMetricQueryDefaultConfig(MetricQueryDefaultConfig defaultConfig, User user) {
MetricQueryDefaultConfigDO defaultConfigDO =
@@ -329,31 +329,14 @@ public class MetricServiceImpl implements MetricService {
return metricQueryDefaultConfig;
}
private void checkParam(MetricReq metricReq) {
MetricTypeParams typeParams = metricReq.getTypeParams();
List<Measure> measures = typeParams.getMeasures();
if (CollectionUtils.isEmpty(measures)) {
throw new InvalidArgumentException("不可缺少度量");
}
if (StringUtils.isBlank(typeParams.getExpr())) {
throw new InvalidArgumentException("表达式不可为空");
}
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
}
private void checkExist(List<MetricReq> metricReqs) {
private void checkExist(List<MetricBaseReq> metricReqs) {
Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricResps = getMetricInSameDomain(modelId);
Map<String, MetricResp> bizNameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
for (MetricReq metricReq : metricReqs) {
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
for (MetricBaseReq metricReq : metricReqs) {
if (bizNameMap.containsKey(metricReq.getBizName())) {
MetricResp metricResp = bizNameMap.get(metricReq.getBizName());
if (!metricResp.getId().equals(metricReq.getId())) {

View File

@@ -30,12 +30,12 @@ import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.core.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.core.manager.MetricYamlManager;
import com.tencent.supersonic.headless.core.manager.ModelYamlManager;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;
@@ -211,8 +211,8 @@ public class ModelServiceImpl implements ModelService {
}
private void batchCreateMetric(ModelDO datasourceDO, User user) throws Exception {
List<MetricReq> exprMetricReqs = ModelConverter.convertMetricList(datasourceDO);
metricService.createMetricBatch(exprMetricReqs, user);
List<MetricReq> metricReqs = ModelConverter.convertMetricList(datasourceDO);
metricService.createMetricBatch(metricReqs, user);
}
private void checkName(ModelReq modelReq) {

View File

@@ -4,15 +4,12 @@ package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.DomainReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.DomainResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
public class DomainConvert {
@@ -43,13 +40,4 @@ public class DomainConvert {
return domainResp;
}
public static DomainResp convert(DomainDO domainDO, Map<Long, String> domainFullPathMap,
Map<Long, List<DimensionResp>> dimensionMap,
Map<Long, List<MetricResp>> metricMap) {
DomainResp domainResp = convert(domainDO, domainFullPathMap);
domainResp.setDimensionCnt(dimensionMap.getOrDefault(domainResp.getId(), Lists.newArrayList()).size());
domainResp.setMetricCnt(metricMap.getOrDefault(domainResp.getId(), Lists.newArrayList()).size());
return domainResp;
}
}

View File

@@ -0,0 +1,69 @@
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.request.MetricReq;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
public class MetricCheckUtils {
public static void checkParam(MetricReq metricReq) {
String expr = "";
if (MetricDefineType.METRIC.equals(metricReq.getMetricDefineType())) {
MetricDefineByMetricParams typeParams = metricReq.getMetricDefineByMetricParams();
if (typeParams == null) {
throw new InvalidArgumentException("指标定义参数不可为空");
}
expr = typeParams.getExpr();
if (CollectionUtils.isEmpty(typeParams.getMetrics())) {
throw new InvalidArgumentException("定义指标的指标列表参数不可为空");
}
if (hasAggregateFunction(expr)) {
throw new InvalidArgumentException("基于指标来创建指标,表达式中不可再包含聚合函数");
}
}
if (MetricDefineType.MEASURE.equals(metricReq.getMetricDefineType())) {
MetricDefineByMeasureParams typeParams = metricReq.getTypeParams();
if (typeParams == null) {
throw new InvalidArgumentException("指标定义参数不可为空");
}
expr = typeParams.getExpr();
if (CollectionUtils.isEmpty(typeParams.getMeasures())) {
throw new InvalidArgumentException("定义指标的度量列表参数不可为空");
}
if (hasAggregateFunction(expr)) {
throw new InvalidArgumentException("基于度量来创建指标,表达式中不可再包含聚合函数");
}
}
if (MetricDefineType.FIELD.equals(metricReq.getMetricDefineType())) {
MetricDefineByFieldParams typeParams = metricReq.getMetricDefineByFieldParams();
if (typeParams == null) {
throw new InvalidArgumentException("指标定义参数不可为空");
}
expr = typeParams.getExpr();
if (CollectionUtils.isEmpty(typeParams.getFields())) {
throw new InvalidArgumentException("定义指标的字段列表参数不可为空");
}
if (!hasAggregateFunction(expr)) {
throw new InvalidArgumentException("基于字段来创建指标,表达式中必须包含聚合函数");
}
}
if (StringUtils.isBlank(expr)) {
throw new InvalidArgumentException("表达式不可为空");
}
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
}
private static boolean hasAggregateFunction(String expr) {
String sql = String.format("select %s from table", expr);
return SqlParserSelectFunctionHelper.hasAggregateFunction(sql);
}
}

View File

@@ -1,11 +1,15 @@
package com.tencent.supersonic.headless.server.utils;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp;
@@ -13,6 +17,7 @@ import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import org.springframework.beans.BeanUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -22,44 +27,52 @@ public class MetricConverter {
MetricDO metricDO = new MetricDO();
BeanMapper.mapper(metricReq, metricDO);
metricDO.setType(metricReq.getMetricType().name());
metricDO.setTypeParams(JSONObject.toJSONString(metricReq.getTypeParams()));
metricDO.setTypeParams(metricReq.getTypeParamsJson());
metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat()));
metricDO.setTags(metricReq.getTag());
metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension()));
metricDO.setStatus(StatusEnum.ONLINE.getCode());
metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
metricDO.setDefineType(metricReq.getMetricDefineType().name());
return metricDO;
}
public static MetricDO convert(MetricDO metricDO, MetricReq metricReq) {
BeanMapper.mapper(metricReq, metricDO);
metricDO.setTypeParams(JSONObject.toJSONString(metricReq.getTypeParams()));
metricDO.setDefineType(metricReq.getMetricDefineType().name());
if (metricReq.getDataFormat() != null) {
metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat()));
}
if (metricReq.getRelateDimension() != null) {
metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension()));
}
metricDO.setTags(metricReq.getTag());
metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
if (metricReq.getTag() != null) {
metricDO.setTags(metricReq.getTag());
}
if (metricReq.getExt() != null) {
metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
}
if (metricReq.getTypeParamsJson() != null) {
metricDO.setTypeParams(metricReq.getTypeParamsJson());
}
return metricDO;
}
public static MetricResp convert2MetricResp(MetricDO metricDO) {
return convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
}
public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap, List<Long> collect) {
MetricResp metricResp = new MetricResp();
BeanUtils.copyProperties(metricDO, metricResp);
metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(), MetricTypeParams.class));
metricResp.setDataFormat(JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class));
ModelResp modelResp = modelMap.get(metricDO.getModelId());
if (modelResp != null) {
metricResp.setModelName(modelResp.getName());
metricResp.setDomainId(modelResp.getDomainId());
}
if (collect != null && collect.contains(metricDO.getId())) {
metricResp.setIsCollect(true);
} else {
metricResp.setIsCollect(false);
}
metricResp.setIsCollect(collect != null && collect.contains(metricDO.getId()));
metricResp.setTag(metricDO.getTags());
metricResp.setRelateDimension(JSONObject.parseObject(metricDO.getRelateDimensions(),
RelateDimension.class));
@@ -67,6 +80,19 @@ public class MetricConverter {
metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), Map.class));
}
metricResp.setTypeEnum(TypeEnums.METRIC);
if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(),
MetricDefineByMeasureParams.class));
} else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setMetricDefineByMetricParams(JSONObject.parseObject(metricDO.getTypeParams(),
MetricDefineByMetricParams.class));
} else if (MetricDefineType.FIELD.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setMetricDefineByFieldParams(JSONObject.parseObject(metricDO.getTypeParams(),
MetricDefineByFieldParams.class));
}
if (metricDO.getDefineType() != null) {
metricResp.setMetricDefineType(MetricDefineType.valueOf(metricDO.getDefineType()));
}
return metricResp;
}

View File

@@ -8,13 +8,14 @@ import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.enums.DimensionType;
import com.tencent.supersonic.headless.api.enums.IdentifyType;
import com.tencent.supersonic.headless.api.enums.MetricType;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.request.DimensionReq;
import com.tencent.supersonic.headless.api.request.MetricReq;
@@ -116,11 +117,13 @@ public class ModelConverter {
metricReq.setBizName(measure.getBizName().replaceFirst(modelDO.getBizName() + "_", ""));
metricReq.setDescription(measure.getName());
metricReq.setModelId(modelDO.getId());
metricReq.setMetricType(MetricType.ATOMIC);
MetricTypeParams exprTypeParams = new MetricTypeParams();
MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
exprTypeParams.setExpr(measure.getBizName());
exprTypeParams.setMeasures(Lists.newArrayList(measure));
MeasureParam measureParam = new MeasureParam();
BeanMapper.mapper(measure, measureParam);
exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
metricReq.setTypeParams(exprTypeParams);
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
return metricReq;
}

View File

@@ -19,6 +19,7 @@
<result column="data_format" jdbcType="VARCHAR" property="dataFormat" />
<result column="alias" jdbcType="VARCHAR" property="alias" />
<result column="tags" jdbcType="VARCHAR" property="tags" />
<result column="define_type" jdbcType="VARCHAR" property="defineType" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO">
<result column="type_params" jdbcType="LONGVARCHAR" property="typeParams" />
@@ -56,7 +57,7 @@
</sql>
<sql id="Base_Column_List">
id, model_id, name, biz_name, description, status, sensitive_level, type, created_at,
created_by, updated_at, updated_by, data_format_type, data_format, alias, tags
created_by, updated_at, updated_by, data_format_type, data_format, alias, tags, define_type
</sql>
<sql id="Blob_Column_List">
type_params
@@ -66,7 +67,7 @@
insert into s2_metric (model_id, name,
biz_name, description, type,status,sensitive_level,
created_at, created_by, updated_at,
updated_by, type_params
updated_by, type_params, define_type
)
values
<foreach collection="list" item="metric" separator=",">
@@ -76,30 +77,12 @@
#{metric.status,jdbcType=VARCHAR},#{metric.sensitiveLevel,jdbcType=VARCHAR},
#{metric.createdAt,jdbcType=TIMESTAMP}, #{metric.createdBy,jdbcType=VARCHAR},
#{metric.updatedAt,jdbcType=TIMESTAMP},
#{metric.updatedBy,jdbcType=VARCHAR}, #{metric.typeParams,jdbcType=LONGVARCHAR}
#{metric.updatedBy,jdbcType=VARCHAR}, #{metric.typeParams,jdbcType=LONGVARCHAR},
#{metric.defineType,jdbcType=VARCHAR}
)
</foreach>
</insert>
<update id="batchUpdate" parameterType="java.util.List">
<foreach collection="list" item="metric" separator=";">
update s2_metric
set model_id = #{metric.modelId,jdbcType=BIGINT},
name = #{metric.name,jdbcType=VARCHAR},
biz_name = #{metric.bizName,jdbcType=VARCHAR},
description = #{metric.description,jdbcType=VARCHAR},
type = #{metric.type,jdbcType=VARCHAR},
status = #{metric.status,jdbcType=VARCHAR},
created_at = #{metric.createdAt,jdbcType=TIMESTAMP},
created_by = #{metric.createdBy,jdbcType=VARCHAR},
updated_at = #{metric.updatedAt,jdbcType=TIMESTAMP},
updated_by = #{metric.updatedBy,jdbcType=VARCHAR},
sensitive_level = #{metric.sensitiveLevel,jdbcType=INTEGER},
type_params = #{metric.typeParams,jdbcType=LONGVARCHAR}
where id = #{metric.id,jdbcType=BIGINT}
</foreach>
</update>
<update id="batchUpdateStatus" parameterType="java.util.List">
<foreach collection="list" item="metric" separator=";">
update s2_metric

View File

@@ -0,0 +1,160 @@
package com.tencent.supersonic.headless.server.service;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.context.ApplicationEventPublisher;
import java.util.HashMap;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
public class MetricServiceImplTest {
@Test
void createMetric() throws Exception {
MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
ModelService modelService = Mockito.mock(ModelService.class);
MetricService metricService = mockMetricService(metricRepository, modelService);
MetricReq metricReq = buildMetricReq();
when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
MetricResp actualMetricResp = metricService.createMetric(metricReq, User.getFakeUser());
MetricResp expectedMetricResp = buildExpectedMetricResp();
Assertions.assertEquals(expectedMetricResp, actualMetricResp);
}
@Test
void updateMetric() throws Exception {
MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
ModelService modelService = Mockito.mock(ModelService.class);
MetricService metricService = mockMetricService(metricRepository, modelService);
MetricReq metricReq = buildMetricUpdateReq();
when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
MetricDO metricDO = MetricConverter.convert2MetricDO(buildMetricReq());
when(metricRepository.getMetricById(metricDO.getId())).thenReturn(metricDO);
MetricResp actualMetricResp = metricService.updateMetric(metricReq, User.getFakeUser());
MetricResp expectedMetricResp = buildExpectedMetricResp();
Assertions.assertEquals(expectedMetricResp, actualMetricResp);
}
private MetricService mockMetricService(MetricRepository metricRepository,
ModelService modelService) {
DomainService domainService = Mockito.mock(DomainService.class);
ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class);
CollectService collectService = Mockito.mock(CollectService.class);
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
return new MetricServiceImpl(metricRepository, modelService, domainService,
chatGptHelper, collectService, eventPublisher);
}
private MetricReq buildMetricReq() {
MetricReq metricReq = new MetricReq();
metricReq.setId(1L);
metricReq.setName("hr部门的访问次数");
metricReq.setBizName("pv");
metricReq.setDescription("SuperSonic的访问情况");
metricReq.setAlias("pv");
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
metricReq.setModelId(2L);
metricReq.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
DataFormat dataFormat = new DataFormat();
dataFormat.setDecimalPlaces(3);
dataFormat.setNeedMultiply100(false);
metricReq.setDataFormat(dataFormat);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricReq.setMetricDefineByMeasureParams(typeParams);
metricReq.setTags(Lists.newArrayList("核心指标"));
metricReq.setRelateDimension(
RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
new DrillDownDimension(1L),
new DrillDownDimension(1L, false))
).build());
metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
metricReq.setExt(new HashMap<>());
return metricReq;
}
private MetricResp buildExpectedMetricResp() {
MetricResp metricResp = new MetricResp();
metricResp.setId(1L);
metricResp.setName("hr部门的访问次数");
metricResp.setBizName("pv");
metricResp.setDescription("SuperSonic的访问情况");
metricResp.setAlias("pv");
metricResp.setMetricDefineType(MetricDefineType.MEASURE);
metricResp.setModelId(2L);
metricResp.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
DataFormat dataFormat = new DataFormat();
dataFormat.setDecimalPlaces(3);
dataFormat.setNeedMultiply100(false);
metricResp.setDataFormat(dataFormat);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricResp.setMetricDefineByMeasureParams(typeParams);
metricResp.setTags(Lists.newArrayList("核心指标"));
metricResp.setRelateDimension(
RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
new DrillDownDimension(1L),
new DrillDownDimension(1L, false))
).build());
metricResp.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
metricResp.setExt(new HashMap<>());
metricResp.setTypeEnum(TypeEnums.METRIC);
metricResp.setIsCollect(false);
metricResp.setType(MetricType.DERIVED.name());
metricResp.setStatus(StatusEnum.ONLINE.getCode());
return metricResp;
}
private MetricReq buildMetricUpdateReq() {
MetricReq metricReq = new MetricReq();
metricReq.setId(1L);
metricReq.setName("hr部门的访问次数");
metricReq.setBizName("pv");
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricReq.setMetricDefineByMeasureParams(typeParams);
return metricReq;
}
private ModelResp mockModelResp() {
ModelResp modelResp = new ModelResp();
modelResp.setId(2L);
modelResp.setDomainId(1L);
return modelResp;
}
}

View File

@@ -56,6 +56,18 @@ class ModelServiceImplTest {
Assertions.assertEquals("alice", actualModelResp.getUpdatedBy());
}
@Test
void updateModel_updateAdmin() throws Exception {
ModelRepository modelRepository = Mockito.mock(ModelRepository.class);
ModelService modelService = mockModelService(modelRepository);
ModelReq modelReq = mockModelReq_updateAdmin();
ModelDO modelDO = ModelConverter.convert(mockModelReq(), User.getFakeUser());
when(modelRepository.getModelById(modelReq.getId())).thenReturn(modelDO);
ModelResp actualModelResp = modelService.updateModel(modelReq, User.getFakeUser());
ModelResp expectedModelResp = buildExpectedModelResp();
Assertions.assertEquals(expectedModelResp, actualModelResp);
}
private ModelService mockModelService(ModelRepository modelRepository) {
MetricService metricService = Mockito.mock(MetricService.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
@@ -137,8 +149,8 @@ class ModelServiceImplTest {
measures.add(measure2);
modelDetail.setMeasures(measures);
modelDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a, user_name "
+ "as uv_a FROM s2_pv_uv_statis");
modelDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a,"
+ " user_name as uv_a FROM s2_pv_uv_statis");
modelDetail.setQueryType("sql_query");
modelReq.setDomainId(1L);
modelReq.setFilterSql("where user_name = 'tom'");
@@ -146,6 +158,36 @@ class ModelServiceImplTest {
return modelReq;
}
private ModelReq mockModelReq_updateAdmin() {
ModelReq modelReq = new ModelReq();
modelReq.setId(1L);
modelReq.setName("PVUV统计");
modelReq.setBizName("s2_pv_uv_statis");
ModelDetail modelDetail = new ModelDetail();
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyType.primary.name(), "user_name"));
modelDetail.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionType.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
Dim dimension2 = new Dim("", "page", DimensionType.categorical.name(), 0);
dimension2.setExpr("page");
dimensions.add(dimension2);
modelDetail.setDimensions(dimensions);
List<Measure> measures = new ArrayList<>();
Measure measure1 = new Measure("访问次数", "pv", AggOperatorEnum.SUM.name(), 1);
measures.add(measure1);
Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1);
measures.add(measure2);
modelDetail.setMeasures(measures);
modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, "
+ "user_name as uv FROM s2_pv_uv_statis");
modelDetail.setQueryType("sql_query");
modelReq.setModelDetail(modelDetail);
return modelReq;
}
private ModelResp buildExpectedModelResp() {
ModelResp modelResp = new ModelResp();
modelResp.setName("PVUV统计");
@@ -197,6 +239,7 @@ class ModelServiceImplTest {
modelResp.setDescription("PVUV统计_a");
modelResp.setDatabaseId(2L);
modelResp.setDomainId(1L);
modelResp.setStatus(StatusEnum.ONLINE.getCode());
modelResp.setAlias("访问次数统计,PVUV统计");
modelResp.setAdmins(Lists.newArrayList("admin"));
modelResp.setViewers(Lists.newArrayList("alice"));

View File

@@ -15,12 +15,14 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.enums.DimensionType;
import com.tencent.supersonic.headless.api.enums.IdentifyType;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.request.DimensionReq;
@@ -329,15 +331,16 @@ public class ModelDemoDataLoader {
metricReq.setDescription("停留时长");
metricReq.setTags(Collections.singletonList("核心指标"));
metricReq.setAlias("访问时长");
MetricTypeParams metricTypeParams = new MetricTypeParams();
MetricDefineByMeasureParams metricTypeParams = new MetricDefineByMeasureParams();
metricTypeParams.setExpr("s2_stay_time_statis_stay_hours");
List<Measure> measures = new ArrayList<>();
Measure measure = new Measure("停留时长",
"s2_stay_time_statis_stay_hours", AggOperatorEnum.SUM.getOperator(), 1);
List<MeasureParam> measures = new ArrayList<>();
MeasureParam measure = new MeasureParam("s2_stay_time_statis_stay_hours",
"", AggOperatorEnum.SUM.getOperator());
measures.add(measure);
metricTypeParams.setMeasures(measures);
metricReq.setTypeParams(metricTypeParams);
metricService.updateExprMetric(metricReq, user);
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
metricService.updateMetric(metricReq, user);
}
public void addAuthGroup_1() {

View File

@@ -174,9 +174,9 @@ CREATE TABLE IF NOT EXISTS `s2_metric` (
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal
`status` INT NOT NULL ,
`sensitive_level` INT NOT NULL ,
`type` varchar(50) NOT NULL , -- type proxy,expr
`type` varchar(50) NOT NULL , -- ATOMIC, DERIVED
`type_params` LONGVARCHAR DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
@@ -188,6 +188,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` (
`tags` varchar(500) DEFAULT NULL,
`relate_dimensions` varchar(500) DEFAULT NULL,
`ext` LONGVARCHAR DEFAULT NULL ,
`define_type` varchar(50) NOT NULL, -- MEASURE, FIELD, METRIC
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_metric IS 'metric information table';

View File

@@ -279,28 +279,31 @@ CREATE TABLE `s2_domain` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='主题域基础信息表';
CREATE TABLE `s2_metric` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`model_id` bigint(20) DEFAULT NULL,
`name` varchar(255) NOT NULL COMMENT '指标名称',
`biz_name` varchar(255) NOT NULL COMMENT '字段名称',
`description` varchar(500) DEFAULT NULL COMMENT '描述',
`status` int(10) NOT NULL COMMENT '指标状态,0未启用,1启用',
`sensitive_level` int(10) NOT NULL COMMENT '敏感级别',
`type` varchar(50) NOT NULL COMMENT '指标类型',
`type_params` text NOT NULL COMMENT '类型参数',
`created_at` datetime NOT NULL COMMENT '创建时间',
`created_by` varchar(100) NOT NULL COMMENT '创建',
`updated_at` datetime NOT NULL COMMENT '更新时间',
`updated_by` varchar(100) NOT NULL COMMENT '更新',
`data_format_type` varchar(50) DEFAULT NULL COMMENT '数值类型',
`data_format` varchar(500) DEFAULT NULL COMMENT '数值类型参数',
`alias` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`tags` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`relate_dimensions` varchar(500) DEFAULT NULL COMMENT '指标相关维度',
`ext` text DEFAULT NULL ,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='指标表';
CREATE TABLE `s2_metric`
(
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`model_id` bigint(20) DEFAULT NULL,
`name` varchar(255) NOT NULL COMMENT '指标名称',
`biz_name` varchar(255) NOT NULL COMMENT '字段名称',
`description` varchar(500) DEFAULT NULL COMMENT '描述',
`status` int(10) NOT NULL COMMENT '指标状态',
`sensitive_level` int(10) NOT NULL COMMENT '敏感级别',
`type` varchar(50) NOT NULL COMMENT '指标类型',
`type_params` text NOT NULL COMMENT '类型参数',
`created_at` datetime NOT NULL COMMENT '创建时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
`updated_at` datetime NOT NULL COMMENT '更新时间',
`updated_by` varchar(100) NOT NULL COMMENT '更新人',
`data_format_type` varchar(50) DEFAULT NULL COMMENT '数值类型',
`data_format` varchar(500) DEFAULT NULL COMMENT '数值类型参数',
`alias` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`tags` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`relate_dimensions` varchar(500) DEFAULT NULL COMMENT '指标相关维度',
`ext` text DEFAULT NULL,
`define_type` varchar(50) DEFAULT NULL, -- MEASURE, FIELD, METRIC
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8 COMMENT ='指标表';
CREATE TABLE `s2_model` (

View File

@@ -164,3 +164,8 @@ CREATE TABLE `s2_app`
`created_by` varchar(255) null,
`updated_by` varchar(255) null
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--20240115
alter table s2_metric add column `define_type` varchar(50) DEFAULT NULL; -- MEASURE, FIELD, METRIC
update s2_metric set define_type = 'MEASURE';