(improvement)(headless) Supports creating new metric by fields and metrics (#639)

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-01-17 14:21:48 +08:00
committed by GitHub
parent b2beecb5b8
commit 07e6924cfd
40 changed files with 647 additions and 221 deletions

View File

@@ -6,6 +6,7 @@ public enum StatusEnum {
ONLINE("ONLINE", 1), ONLINE("ONLINE", 1),
OFFLINE("OFFLINE", 2), OFFLINE("OFFLINE", 2),
DELETED("DELETED", 3), DELETED("DELETED", 3),
UNAVAILABLE("UNAVAILABLE", 4),
UNKNOWN("UNKNOWN", -1); UNKNOWN("UNKNOWN", -1);

View File

@@ -1,11 +1,5 @@
package com.tencent.supersonic.common.util.jsqlparser; package com.tencent.supersonic.common.util.jsqlparser;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Expression;
@@ -19,6 +13,12 @@ import net.sf.jsqlparser.statement.select.SelectItem;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/** /**
* Sql Parser Select function Helper * Sql Parser Select function Helper
*/ */
@@ -32,6 +32,11 @@ public class SqlParserSelectFunctionHelper {
return SqlParserSelectHelper.hasGroupBy(sql); return SqlParserSelectHelper.hasGroupBy(sql);
} }
public static void main(String[] args) {
String sql = "select a from table";
System.out.println(hasAggregateFunction(sql));
}
public static boolean hasFunction(String sql, String functionName) { public static boolean hasFunction(String sql, String functionName) {
Set<String> functions = getFunctions(sql); Set<String> functions = getFunctions(sql);
if (!CollectionUtils.isEmpty(functions)) { if (!CollectionUtils.isEmpty(functions)) {

View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.headless.api.enums;
public enum MetricDefineType {
FIELD,
MEASURE,
METRIC
}

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public class Field {
private String fieldName;
}

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public class FieldParam {
private String fieldName;
}

View File

@@ -32,6 +32,11 @@ public class Measure {
this.bizName = bizName; this.bizName = bizName;
} }
public Measure(String bizName, String constraint) {
this.bizName = bizName;
this.constraint = constraint;
}
public String getFieldName() { public String getFieldName() {
return expr; return expr;
} }

View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class MeasureParam {
private String bizName;
private String constraint;
private String agg;
public MeasureParam(String bizName, String constraint) {
this.bizName = bizName;
this.constraint = constraint;
}
}

View File

@@ -1,14 +1,13 @@
package com.tencent.supersonic.headless.api.pojo; package com.tencent.supersonic.headless.api.pojo;
import java.util.List;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import lombok.Data; import lombok.Data;
import java.util.List;
@Data @Data
public class MetricTypeParams { public class MetricDefineByFieldParams extends MetricDefineParams {
private List<Measure> measures = Lists.newArrayList(); private List<FieldParam> fields = Lists.newArrayList();
private String expr;
} }

View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.headless.api.pojo;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class MetricDefineByMeasureParams extends MetricDefineParams {
private List<MeasureParam> measures = Lists.newArrayList();
}

View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.headless.api.pojo;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class MetricDefineByMetricParams extends MetricDefineParams {
private List<MetricParam> metrics = Lists.newArrayList();
}

View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public abstract class MetricDefineParams {
private String expr;
}

View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public class MetricParam {
private Long id;
private String bizName;
}

View File

@@ -25,6 +25,8 @@ public class ModelDetail {
private List<Measure> measures; private List<Measure> measures;
private List<Field> fields;
public String getSqlQuery() { public String getSqlQuery() {
if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) { if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) {
sqlQuery = sqlQuery.substring(0, sqlQuery.length() - 1); sqlQuery = sqlQuery.substring(0, sqlQuery.length() - 1);
@@ -32,7 +34,7 @@ public class ModelDetail {
return sqlQuery; return sqlQuery;
} }
public List<Dim> getTimeDims() { public List<Dim> filterTimeDims() {
if (CollectionUtils.isEmpty(dimensions)) { if (CollectionUtils.isEmpty(dimensions)) {
return Lists.newArrayList(); return Lists.newArrayList();
} }

View File

@@ -4,13 +4,14 @@ import com.google.common.base.Objects;
import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import lombok.Data; import lombok.Data;
import lombok.ToString; import lombok.ToString;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@Data @Data
@ToString(callSuper = true) @ToString(callSuper = true)
public class SchemaItem extends RecordInfo { public class SchemaItem extends RecordInfo {
@@ -38,9 +39,6 @@ public class SchemaItem extends RecordInfo {
if (o == null || getClass() != o.getClass()) { if (o == null || getClass() != o.getClass()) {
return false; return false;
} }
if (!super.equals(o)) {
return false;
}
SchemaItem that = (SchemaItem) o; SchemaItem that = (SchemaItem) o;
return Objects.equal(id, that.id) && Objects.equal(name, that.name) return Objects.equal(id, that.id) && Objects.equal(name, that.name)
&& Objects.equal(bizName, that.bizName) && Objects.equal( && Objects.equal(bizName, that.bizName) && Objects.equal(
@@ -50,7 +48,7 @@ public class SchemaItem extends RecordInfo {
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hashCode(super.hashCode(), id, name, bizName, description, status, typeEnum, sensitiveLevel); return Objects.hashCode(id, name, bizName, description, status, typeEnum, sensitiveLevel);
} }
public static List<String> getAliasList(String alias) { public static List<String> getAliasList(String alias) {

View File

@@ -30,6 +30,9 @@ public class MetricBaseReq extends SchemaItem {
private Map<String, Object> ext = new HashMap<>(); private Map<String, Object> ext = new HashMap<>();
public String getTag() { public String getTag() {
if (tags == null) {
return null;
}
if (CollectionUtils.isEmpty(tags)) { if (CollectionUtils.isEmpty(tags)) {
return ""; return "";
} }

View File

@@ -1,30 +1,49 @@
package com.tencent.supersonic.headless.api.request; package com.tencent.supersonic.headless.api.request;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.MetricType; import com.tencent.supersonic.headless.api.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
@Data @Data
public class MetricReq extends MetricBaseReq { public class MetricReq extends MetricBaseReq {
private MetricType metricType; private MetricDefineType metricDefineType = MetricDefineType.MEASURE;
private MetricDefineByMeasureParams typeParams;
private MetricDefineByFieldParams metricDefineByFieldParams;
private MetricDefineByMetricParams metricDefineByMetricParams;
private MetricTypeParams typeParams; public String getTypeParamsJson() {
if (metricDefineByFieldParams != null) {
return JSONObject.toJSONString(metricDefineByFieldParams);
} else if (typeParams != null) {
return JSONObject.toJSONString(typeParams);
} else if (metricDefineByMetricParams != null) {
return JSONObject.toJSONString(metricDefineByMetricParams);
}
return null;
}
public MetricType getMetricType() { public MetricType getMetricType() {
if (metricType != null) { if (MetricDefineType.METRIC.equals(metricDefineType)) {
return metricType;
}
List<Measure> measureList = typeParams.getMeasures();
if (measureList.size() == 1 && typeParams.getExpr().trim().equalsIgnoreCase(measureList.get(0).getBizName())) {
return MetricType.ATOMIC;
} else if (measureList.size() >= 1) {
return MetricType.DERIVED; return MetricType.DERIVED;
} }
throw new RuntimeException("measure can not be none"); if (MetricDefineType.MEASURE.equals(metricDefineType)) {
List<MeasureParam> measures = typeParams.getMeasures();
if (measures.size() > 1) {
return MetricType.DERIVED;
}
if (measures.size() == 1 && measures.get(0).getBizName()
.equalsIgnoreCase(typeParams.getExpr())) {
return MetricType.ATOMIC;
}
}
return MetricType.ATOMIC;
} }
} }

View File

@@ -42,7 +42,7 @@ public class ModelReq extends SchemaItem {
if (modelDetail == null) { if (modelDetail == null) {
return Lists.newArrayList(); return Lists.newArrayList();
} }
return modelDetail.getTimeDims(); return modelDetail.filterTimeDims();
} }
public String getViewer() { public String getViewer() {

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.api.response; package com.tencent.supersonic.headless.api.response;
import com.tencent.supersonic.headless.api.pojo.Entity;
import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import lombok.Data; import lombok.Data;
import lombok.ToString; import lombok.ToString;
@@ -26,12 +25,6 @@ public class DomainResp extends SchemaItem {
private Integer isOpen = 0; private Integer isOpen = 0;
private Integer dimensionCnt;
private Integer metricCnt;
private Entity entity;
private boolean hasEditPermission = false; private boolean hasEditPermission = false;
@Override @Override

View File

@@ -2,21 +2,23 @@ package com.tencent.supersonic.headless.api.response;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.DataFormat; import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import lombok.Data; import lombok.Data;
import lombok.ToString; import lombok.ToString;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -33,8 +35,6 @@ public class MetricResp extends SchemaItem {
//ATOMIC DERIVED //ATOMIC DERIVED
private String type; private String type;
private MetricTypeParams typeParams;
private String dataFormatType; private String dataFormatType;
private DataFormat dataFormat; private DataFormat dataFormat;
@@ -51,6 +51,14 @@ public class MetricResp extends SchemaItem {
private Map<String, Object> ext = new HashMap<>(); private Map<String, Object> ext = new HashMap<>();
private MetricDefineType metricDefineType = MetricDefineType.MEASURE;
private MetricDefineByMeasureParams typeParams;
private MetricDefineByFieldParams metricDefineByFieldParams;
private MetricDefineByMetricParams metricDefineByMetricParams;
public void setTag(String tag) { public void setTag(String tag) {
if (StringUtils.isBlank(tag)) { if (StringUtils.isBlank(tag)) {
tags = Lists.newArrayList(); tags = Lists.newArrayList();
@@ -59,14 +67,6 @@ public class MetricResp extends SchemaItem {
} }
} }
public Set<Long> getNecessaryDimensionIds() {
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
return Sets.newHashSet();
}
return relateDimension.getDrillDownDimensions().stream().filter(DrillDownDimension::isNecessary)
.map(DrillDownDimension::getDimensionId).collect(Collectors.toSet());
}
public String getRelaDimensionIdKey() { public String getRelaDimensionIdKey() {
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) { if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
return ""; return "";
@@ -78,6 +78,11 @@ public class MetricResp extends SchemaItem {
} }
public String getDefaultAgg() { public String getDefaultAgg() {
return typeParams.getMeasures().get(0).getAgg(); if (typeParams != null
&& CollectionUtils.isNotEmpty(typeParams.getMeasures())) {
return typeParams.getMeasures().get(0).getAgg();
}
return "";
} }
} }

View File

@@ -3,16 +3,22 @@ package com.tencent.supersonic.headless.api.response;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.Identify;
import lombok.Data; import lombok.Data;
import lombok.ToString;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Data @Data
@ToString(callSuper = true)
public class ModelResp extends SchemaItem { public class ModelResp extends SchemaItem {
private Long domainId; private Long domainId;
@@ -66,7 +72,27 @@ public class ModelResp extends SchemaItem {
if (modelDetail == null) { if (modelDetail == null) {
return Lists.newArrayList(); return Lists.newArrayList();
} }
return modelDetail.getTimeDims(); return modelDetail.filterTimeDims();
}
public Set<String> getFieldList() {
Set<String> fieldSet = new HashSet<>();
if (modelDetail == null) {
return fieldSet;
}
if (!CollectionUtils.isEmpty(modelDetail.getIdentifiers())) {
fieldSet.addAll(modelDetail.getIdentifiers().stream()
.map(Identify::getFieldName).collect(Collectors.toSet()));
}
if (!CollectionUtils.isEmpty(modelDetail.getDimensions())) {
fieldSet.addAll(modelDetail.getDimensions().stream()
.map(Dim::getFieldName).collect(Collectors.toSet()));
}
if (!CollectionUtils.isEmpty(modelDetail.getMeasures())) {
fieldSet.addAll(modelDetail.getMeasures().stream()
.map(Measure::getFieldName).collect(Collectors.toSet()));
}
return fieldSet;
} }
} }

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.headless.core.manager; package com.tencent.supersonic.headless.core.manager;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
@@ -37,21 +37,21 @@ public class MetricYamlManager {
BeanUtils.copyProperties(metric, metricYamlTpl); BeanUtils.copyProperties(metric, metricYamlTpl);
metricYamlTpl.setName(metric.getBizName()); metricYamlTpl.setName(metric.getBizName());
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy())); metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
MetricTypeParams exprMetricTypeParams = metric.getTypeParams(); MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams();
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl(); MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
metricTypeParamsYamlTpl.setExpr(exprMetricTypeParams.getExpr()); metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
List<Measure> measures = exprMetricTypeParams.getMeasures(); List<MeasureParam> measures = metricDefineParams.getMeasures();
metricTypeParamsYamlTpl.setMeasures( metricTypeParamsYamlTpl.setMeasures(
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList())); measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl); metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl);
return metricYamlTpl; return metricYamlTpl;
} }
public static MeasureYamlTpl convert(Measure measure) { public static MeasureYamlTpl convert(MeasureParam measure) {
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl(); MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
measureYamlTpl.setName(measure.getBizName()); measureYamlTpl.setName(measure.getBizName());
measureYamlTpl.setConstraint(measure.getConstraint()); measureYamlTpl.setConstraint(measure.getConstraint());
measureYamlTpl.setAgg(measure.getAlias()); measureYamlTpl.setAgg(measure.getAgg());
return measureYamlTpl; return measureYamlTpl;
} }

View File

@@ -94,7 +94,7 @@ public class ModelYamlManager {
} }
measure.setAgg("count"); measure.setAgg("count");
measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt")); measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt"));
measure.setCreateMetric("true"); measure.setIsCreateMetric(1);
datasourceDetail.getMeasures().add(measure); datasourceDetail.getMeasures().add(measure);
} }

View File

@@ -316,9 +316,9 @@ public class SourceRender extends Renderer {
} }
} }
public void render(MetricQueryReq metricCommand, List<DataSource> dataSources, SqlValidatorScope scope, public void render(MetricQueryReq metricQueryReq, List<DataSource> dataSources, SqlValidatorScope scope,
HeadlessSchema schema, boolean nonAgg) throws Exception { HeadlessSchema schema, boolean nonAgg) throws Exception {
String queryWhere = metricCommand.getWhere(); String queryWhere = metricQueryReq.getWhere();
Set<String> whereFields = new HashSet<>(); Set<String> whereFields = new HashSet<>();
List<String> fieldWhere = new ArrayList<>(); List<String> fieldWhere = new ArrayList<>();
if (queryWhere != null && !queryWhere.isEmpty()) { if (queryWhere != null && !queryWhere.isEmpty()) {
@@ -328,13 +328,13 @@ public class SourceRender extends Renderer {
} }
if (dataSources.size() == 1) { if (dataSources.size() == 1) {
DataSource dataSource = dataSources.get(0); DataSource dataSource = dataSources.get(0);
super.tableView = renderOne("", fieldWhere, metricCommand.getMetrics(), super.tableView = renderOne("", fieldWhere, metricQueryReq.getMetrics(),
metricCommand.getDimensions(), metricQueryReq.getDimensions(),
metricCommand.getWhere(), dataSource, scope, schema, nonAgg); metricQueryReq.getWhere(), dataSource, scope, schema, nonAgg);
return; return;
} }
JoinRender joinRender = new JoinRender(); JoinRender joinRender = new JoinRender();
joinRender.render(metricCommand, dataSources, scope, schema, nonAgg); joinRender.render(metricQueryReq, dataSources, scope, schema, nonAgg);
super.tableView = joinRender.getTableView(); super.tableView = joinRender.getTableView();
} }

View File

@@ -101,4 +101,6 @@ public class MetricDO {
private String ext; private String ext;
private String defineType;
} }

View File

@@ -11,8 +11,6 @@ public interface MetricDOCustomMapper {
void batchInsert(List<MetricDO> metricDOS); void batchInsert(List<MetricDO> metricDOS);
void batchUpdate(List<MetricDO> metricDOS);
void batchUpdateStatus(List<MetricDO> metricDOS); void batchUpdateStatus(List<MetricDO> metricDOS);
List<MetricDO> query(MetricFilter metricFilter); List<MetricDO> query(MetricFilter metricFilter);

View File

@@ -9,6 +9,7 @@ import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.request.MetaBatchReq; import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.request.MetricReq; import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.request.PageMetricReq; import com.tencent.supersonic.headless.api.request.PageMetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricResp;
@@ -33,30 +34,26 @@ import java.util.Set;
@RequestMapping("/api/semantic/metric") @RequestMapping("/api/semantic/metric")
public class MetricController { public class MetricController {
private MetricService metricService; private MetricService metricService;
public MetricController(MetricService metricService) { public MetricController(MetricService metricService) {
this.metricService = metricService; this.metricService = metricService;
} }
@PostMapping("/creatExprMetric") @PostMapping("/creatExprMetric")
public Boolean creatExprMetric(@RequestBody MetricReq metricReq, public MetricResp createMetric(@RequestBody MetricReq metricReq,
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response) throws Exception { HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response); User user = UserHolder.findUser(request, response);
metricService.createMetric(metricReq, user); return metricService.createMetric(metricReq, user);
return true;
} }
@PostMapping("/updateExprMetric") @PostMapping("/updateExprMetric")
public Boolean updateExprMetric(@RequestBody MetricReq metricReq, public MetricResp updateMetric(@RequestBody MetricReq metricReq,
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response) throws Exception { HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response); User user = UserHolder.findUser(request, response);
metricService.updateExprMetric(metricReq, user); return metricService.updateMetric(metricReq, user);
return true;
} }
@PostMapping("/batchUpdateStatus") @PostMapping("/batchUpdateStatus")
@@ -69,7 +66,7 @@ public class MetricController {
} }
@PostMapping("/mockMetricAlias") @PostMapping("/mockMetricAlias")
public List<String> mockMetricAlias(@RequestBody MetricReq metricReq, public List<String> mockMetricAlias(@RequestBody MetricBaseReq metricReq,
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response) { HttpServletResponse response) {
User user = UserHolder.findUser(request, response); User user = UserHolder.findUser(request, response);
@@ -82,6 +79,11 @@ public class MetricController {
return metricService.getMetrics(metaFilter); return metricService.getMetrics(metaFilter);
} }
@GetMapping("/getMetricsToCreateNewMetric/{modelId}")
public List<MetricResp> getMetricsToCreateNewMetric(@PathVariable("modelId") Long modelId) {
return metricService.getMetricsToCreateNewMetric(modelId);
}
@PostMapping("/queryMetric") @PostMapping("/queryMetric")
public PageInfo<MetricResp> queryMetric(@RequestBody PageMetricReq pageMetricReq, public PageInfo<MetricResp> queryMetric(@RequestBody PageMetricReq pageMetricReq,
HttpServletRequest request, HttpServletRequest request,
@@ -92,13 +94,15 @@ public class MetricController {
@Deprecated @Deprecated
@GetMapping("getMetric/{modelId}/{bizName}") @GetMapping("getMetric/{modelId}/{bizName}")
public MetricResp getMetric(@PathVariable("modelId") Long modelId, @PathVariable("bizName") String bizName) { public MetricResp getMetric(@PathVariable("modelId") Long modelId,
@PathVariable("bizName") String bizName) {
return metricService.getMetric(modelId, bizName); return metricService.getMetric(modelId, bizName);
} }
@GetMapping("getMetric/{id}") @GetMapping("getMetric/{id}")
public MetricResp getMetric(@PathVariable("id") Long id, public MetricResp getMetric(@PathVariable("id") Long id,
HttpServletRequest request, HttpServletResponse response) { HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response); User user = UserHolder.findUser(request, response);
return metricService.getMetric(id, user); return metricService.getMetric(id, user);
} }

View File

@@ -2,11 +2,11 @@ package com.tencent.supersonic.headless.server.service;
import com.github.pagehelper.PageInfo; import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.request.MetaBatchReq; import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.request.MetricReq; import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.request.PageMetricReq; import com.tencent.supersonic.headless.api.request.PageMetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricResp;
@@ -17,11 +17,11 @@ import java.util.Set;
public interface MetricService { public interface MetricService {
void createMetric(MetricReq metricReq, User user) throws Exception; MetricResp createMetric(MetricReq metricReq, User user) throws Exception;
void createMetricBatch(List<MetricReq> metricReqs, User user) throws Exception; void createMetricBatch(List<MetricReq> metricReqs, User user) throws Exception;
void updateExprMetric(MetricReq metricReq, User user) throws Exception; MetricResp updateMetric(MetricReq metricReq, User user) throws Exception;
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
@@ -31,20 +31,20 @@ public interface MetricService {
List<MetricResp> getMetrics(MetaFilter metaFilter); List<MetricResp> getMetrics(MetaFilter metaFilter);
List<MetricResp> getMetricsToCreateNewMetric(Long modelId);
MetricResp getMetric(Long modelId, String bizName); MetricResp getMetric(Long modelId, String bizName);
MetricResp getMetric(Long id, User user); MetricResp getMetric(Long id, User user);
MetricResp getMetric(Long id); MetricResp getMetric(Long id);
List<String> mockAlias(MetricReq metricReq, String mockType, User user); List<String> mockAlias(MetricBaseReq metricReq, String mockType, User user);
Set<String> getMetricTags(); Set<String> getMetricTags();
List<DrillDownDimension> getDrillDownDimension(Long metricId); List<DrillDownDimension> getDrillDownDimension(Long metricId);
List<DataItem> getDataItems(Long modelId);
void saveMetricQueryDefaultConfig(MetricQueryDefaultConfig defaultConfig, User user); void saveMetricQueryDefaultConfig(MetricQueryDefaultConfig defaultConfig, User user);
MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user); MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user);

View File

@@ -12,14 +12,13 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType; import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
import com.tencent.supersonic.headless.api.request.MetaBatchReq; import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.request.MetricReq; import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.request.PageMetricReq; import com.tencent.supersonic.headless.api.request.PageMetricReq;
import com.tencent.supersonic.headless.api.response.DomainResp; import com.tencent.supersonic.headless.api.response.DomainResp;
@@ -35,10 +34,9 @@ import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -83,13 +81,14 @@ public class MetricServiceImpl implements MetricService {
} }
@Override @Override
public void createMetric(MetricReq metricReq, User user) { public MetricResp createMetric(MetricReq metricReq, User user) {
checkExist(Lists.newArrayList(metricReq)); checkExist(Lists.newArrayList(metricReq));
checkParam(metricReq); MetricCheckUtils.checkParam(metricReq);
metricReq.createdBy(user.getName()); metricReq.createdBy(user.getName());
MetricDO metricDO = MetricConverter.convert2MetricDO(metricReq); MetricDO metricDO = MetricConverter.convert2MetricDO(metricReq);
metricRepository.createMetric(metricDO); metricRepository.createMetric(metricDO);
sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD); sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD);
return MetricConverter.convert2MetricResp(metricDO);
} }
@Override @Override
@@ -116,8 +115,8 @@ public class MetricServiceImpl implements MetricService {
} }
@Override @Override
public void updateExprMetric(MetricReq metricReq, User user) { public MetricResp updateMetric(MetricReq metricReq, User user) {
checkParam(metricReq); MetricCheckUtils.checkParam(metricReq);
checkExist(Lists.newArrayList(metricReq)); checkExist(Lists.newArrayList(metricReq));
metricReq.updatedBy(user.getName()); metricReq.updatedBy(user.getName());
MetricDO metricDO = metricRepository.getMetricById(metricReq.getId()); MetricDO metricDO = metricRepository.getMetricById(metricReq.getId());
@@ -130,6 +129,7 @@ public class MetricServiceImpl implements MetricService {
dataItem.setNewName(metricDO.getName()); dataItem.setNewName(metricDO.getName());
sendEvent(getDataItem(metricDO), EventType.UPDATE); sendEvent(getDataItem(metricDO), EventType.UPDATE);
} }
return MetricConverter.convert2MetricResp(metricDO);
} }
@Override @Override
@@ -209,6 +209,17 @@ public class MetricServiceImpl implements MetricService {
return convertList(queryMetric(metricFilter), Lists.newArrayList()); return convertList(queryMetric(metricFilter), Lists.newArrayList());
} }
@Override
public List<MetricResp> getMetricsToCreateNewMetric(Long modelId) {
MetricFilter metricFilter = new MetricFilter();
metricFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metricFilter);
return metricResps.stream().filter(metricResp ->
MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
.collect(Collectors.toList());
}
private void fillAdminRes(List<MetricResp> metricResps, User user) { private void fillAdminRes(List<MetricResp> metricResps, User user) {
List<ModelResp> modelResps = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); List<ModelResp> modelResps = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelResps)) { if (CollectionUtils.isEmpty(modelResps)) {
@@ -260,7 +271,7 @@ public class MetricServiceImpl implements MetricService {
} }
@Override @Override
public List<String> mockAlias(MetricReq metricReq, String mockType, User user) { public List<String> mockAlias(MetricBaseReq metricReq, String mockType, User user) {
String mockAlias = chatGptHelper.mockAlias(mockType, metricReq.getName(), metricReq.getBizName(), "", String mockAlias = chatGptHelper.mockAlias(mockType, metricReq.getName(), metricReq.getBizName(), "",
metricReq.getDescription(), !"".equals(metricReq.getDataFormatType())); metricReq.getDescription(), !"".equals(metricReq.getDataFormatType()));
@@ -292,17 +303,6 @@ public class MetricServiceImpl implements MetricService {
return modelResp.getDrillDownDimensions(); return modelResp.getDrillDownDimensions();
} }
@Override
public List<DataItem> getDataItems(Long modelId) {
MetricFilter metaFilter = new MetricFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricDO> metricDOS = queryMetric(metaFilter);
if (CollectionUtils.isEmpty(metricDOS)) {
return Lists.newArrayList();
}
return metricDOS.stream().map(this::getDataItem).collect(Collectors.toList());
}
@Override @Override
public void saveMetricQueryDefaultConfig(MetricQueryDefaultConfig defaultConfig, User user) { public void saveMetricQueryDefaultConfig(MetricQueryDefaultConfig defaultConfig, User user) {
MetricQueryDefaultConfigDO defaultConfigDO = MetricQueryDefaultConfigDO defaultConfigDO =
@@ -329,31 +329,14 @@ public class MetricServiceImpl implements MetricService {
return metricQueryDefaultConfig; return metricQueryDefaultConfig;
} }
private void checkParam(MetricReq metricReq) { private void checkExist(List<MetricBaseReq> metricReqs) {
MetricTypeParams typeParams = metricReq.getTypeParams();
List<Measure> measures = typeParams.getMeasures();
if (CollectionUtils.isEmpty(measures)) {
throw new InvalidArgumentException("不可缺少度量");
}
if (StringUtils.isBlank(typeParams.getExpr())) {
throw new InvalidArgumentException("表达式不可为空");
}
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
}
private void checkExist(List<MetricReq> metricReqs) {
Long modelId = metricReqs.get(0).getModelId(); Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricResps = getMetricInSameDomain(modelId); List<MetricResp> metricResps = getMetricInSameDomain(modelId);
Map<String, MetricResp> bizNameMap = metricResps.stream() Map<String, MetricResp> bizNameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1)); .collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap = metricResps.stream() Map<String, MetricResp> nameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1)); .collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
for (MetricReq metricReq : metricReqs) { for (MetricBaseReq metricReq : metricReqs) {
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
if (bizNameMap.containsKey(metricReq.getBizName())) { if (bizNameMap.containsKey(metricReq.getBizName())) {
MetricResp metricResp = bizNameMap.get(metricReq.getBizName()); MetricResp metricResp = bizNameMap.get(metricReq.getBizName());
if (!metricResp.getId().equals(metricReq.getId())) { if (!metricResp.getId().equals(metricReq.getId())) {

View File

@@ -30,12 +30,12 @@ import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.core.manager.DimensionYamlManager; import com.tencent.supersonic.headless.core.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.core.manager.MetricYamlManager; import com.tencent.supersonic.headless.core.manager.MetricYamlManager;
import com.tencent.supersonic.headless.core.manager.ModelYamlManager; import com.tencent.supersonic.headless.core.manager.ModelYamlManager;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO; import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO; import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository; import com.tencent.supersonic.headless.server.persistence.repository.DateInfoRepository;
@@ -211,8 +211,8 @@ public class ModelServiceImpl implements ModelService {
} }
private void batchCreateMetric(ModelDO datasourceDO, User user) throws Exception { private void batchCreateMetric(ModelDO datasourceDO, User user) throws Exception {
List<MetricReq> exprMetricReqs = ModelConverter.convertMetricList(datasourceDO); List<MetricReq> metricReqs = ModelConverter.convertMetricList(datasourceDO);
metricService.createMetricBatch(exprMetricReqs, user); metricService.createMetricBatch(metricReqs, user);
} }
private void checkName(ModelReq modelReq) { private void checkName(ModelReq modelReq) {

View File

@@ -4,15 +4,12 @@ package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.DomainReq; import com.tencent.supersonic.headless.api.request.DomainReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.DomainResp; import com.tencent.supersonic.headless.api.response.DomainResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO; import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import java.util.Map; import java.util.Map;
public class DomainConvert { public class DomainConvert {
@@ -43,13 +40,4 @@ public class DomainConvert {
return domainResp; return domainResp;
} }
public static DomainResp convert(DomainDO domainDO, Map<Long, String> domainFullPathMap,
Map<Long, List<DimensionResp>> dimensionMap,
Map<Long, List<MetricResp>> metricMap) {
DomainResp domainResp = convert(domainDO, domainFullPathMap);
domainResp.setDimensionCnt(dimensionMap.getOrDefault(domainResp.getId(), Lists.newArrayList()).size());
domainResp.setMetricCnt(metricMap.getOrDefault(domainResp.getId(), Lists.newArrayList()).size());
return domainResp;
}
} }

View File

@@ -0,0 +1,69 @@
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.request.MetricReq;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
public class MetricCheckUtils {
public static void checkParam(MetricReq metricReq) {
String expr = "";
if (MetricDefineType.METRIC.equals(metricReq.getMetricDefineType())) {
MetricDefineByMetricParams typeParams = metricReq.getMetricDefineByMetricParams();
if (typeParams == null) {
throw new InvalidArgumentException("指标定义参数不可为空");
}
expr = typeParams.getExpr();
if (CollectionUtils.isEmpty(typeParams.getMetrics())) {
throw new InvalidArgumentException("定义指标的指标列表参数不可为空");
}
if (hasAggregateFunction(expr)) {
throw new InvalidArgumentException("基于指标来创建指标,表达式中不可再包含聚合函数");
}
}
if (MetricDefineType.MEASURE.equals(metricReq.getMetricDefineType())) {
MetricDefineByMeasureParams typeParams = metricReq.getTypeParams();
if (typeParams == null) {
throw new InvalidArgumentException("指标定义参数不可为空");
}
expr = typeParams.getExpr();
if (CollectionUtils.isEmpty(typeParams.getMeasures())) {
throw new InvalidArgumentException("定义指标的度量列表参数不可为空");
}
if (hasAggregateFunction(expr)) {
throw new InvalidArgumentException("基于度量来创建指标,表达式中不可再包含聚合函数");
}
}
if (MetricDefineType.FIELD.equals(metricReq.getMetricDefineType())) {
MetricDefineByFieldParams typeParams = metricReq.getMetricDefineByFieldParams();
if (typeParams == null) {
throw new InvalidArgumentException("指标定义参数不可为空");
}
expr = typeParams.getExpr();
if (CollectionUtils.isEmpty(typeParams.getFields())) {
throw new InvalidArgumentException("定义指标的字段列表参数不可为空");
}
if (!hasAggregateFunction(expr)) {
throw new InvalidArgumentException("基于字段来创建指标,表达式中必须包含聚合函数");
}
}
if (StringUtils.isBlank(expr)) {
throw new InvalidArgumentException("表达式不可为空");
}
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
}
private static boolean hasAggregateFunction(String expr) {
String sql = String.format("select %s from table", expr);
return SqlParserSelectFunctionHelper.hasAggregateFunction(sql);
}
}

View File

@@ -1,11 +1,15 @@
package com.tencent.supersonic.headless.server.utils; package com.tencent.supersonic.headless.server.utils;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DataFormat; import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.request.MetricReq; import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricResp;
@@ -13,6 +17,7 @@ import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -22,44 +27,52 @@ public class MetricConverter {
MetricDO metricDO = new MetricDO(); MetricDO metricDO = new MetricDO();
BeanMapper.mapper(metricReq, metricDO); BeanMapper.mapper(metricReq, metricDO);
metricDO.setType(metricReq.getMetricType().name()); metricDO.setType(metricReq.getMetricType().name());
metricDO.setTypeParams(JSONObject.toJSONString(metricReq.getTypeParams())); metricDO.setTypeParams(metricReq.getTypeParamsJson());
metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat())); metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat()));
metricDO.setTags(metricReq.getTag()); metricDO.setTags(metricReq.getTag());
metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension())); metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension()));
metricDO.setStatus(StatusEnum.ONLINE.getCode()); metricDO.setStatus(StatusEnum.ONLINE.getCode());
metricDO.setExt(JSONObject.toJSONString(metricReq.getExt())); metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
metricDO.setDefineType(metricReq.getMetricDefineType().name());
return metricDO; return metricDO;
} }
public static MetricDO convert(MetricDO metricDO, MetricReq metricReq) { public static MetricDO convert(MetricDO metricDO, MetricReq metricReq) {
BeanMapper.mapper(metricReq, metricDO); BeanMapper.mapper(metricReq, metricDO);
metricDO.setTypeParams(JSONObject.toJSONString(metricReq.getTypeParams())); metricDO.setDefineType(metricReq.getMetricDefineType().name());
if (metricReq.getDataFormat() != null) { if (metricReq.getDataFormat() != null) {
metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat())); metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat()));
} }
if (metricReq.getRelateDimension() != null) { if (metricReq.getRelateDimension() != null) {
metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension())); metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension()));
} }
metricDO.setTags(metricReq.getTag()); if (metricReq.getTag() != null) {
metricDO.setExt(JSONObject.toJSONString(metricReq.getExt())); metricDO.setTags(metricReq.getTag());
}
if (metricReq.getExt() != null) {
metricDO.setExt(JSONObject.toJSONString(metricReq.getExt()));
}
if (metricReq.getTypeParamsJson() != null) {
metricDO.setTypeParams(metricReq.getTypeParamsJson());
}
return metricDO; return metricDO;
} }
public static MetricResp convert2MetricResp(MetricDO metricDO) {
return convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
}
public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap, List<Long> collect) { public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap, List<Long> collect) {
MetricResp metricResp = new MetricResp(); MetricResp metricResp = new MetricResp();
BeanUtils.copyProperties(metricDO, metricResp); BeanUtils.copyProperties(metricDO, metricResp);
metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(), MetricTypeParams.class));
metricResp.setDataFormat(JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class)); metricResp.setDataFormat(JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class));
ModelResp modelResp = modelMap.get(metricDO.getModelId()); ModelResp modelResp = modelMap.get(metricDO.getModelId());
if (modelResp != null) { if (modelResp != null) {
metricResp.setModelName(modelResp.getName()); metricResp.setModelName(modelResp.getName());
metricResp.setDomainId(modelResp.getDomainId()); metricResp.setDomainId(modelResp.getDomainId());
} }
if (collect != null && collect.contains(metricDO.getId())) { metricResp.setIsCollect(collect != null && collect.contains(metricDO.getId()));
metricResp.setIsCollect(true);
} else {
metricResp.setIsCollect(false);
}
metricResp.setTag(metricDO.getTags()); metricResp.setTag(metricDO.getTags());
metricResp.setRelateDimension(JSONObject.parseObject(metricDO.getRelateDimensions(), metricResp.setRelateDimension(JSONObject.parseObject(metricDO.getRelateDimensions(),
RelateDimension.class)); RelateDimension.class));
@@ -67,6 +80,19 @@ public class MetricConverter {
metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), Map.class)); metricResp.setExt(JSONObject.parseObject(metricDO.getExt(), Map.class));
} }
metricResp.setTypeEnum(TypeEnums.METRIC); metricResp.setTypeEnum(TypeEnums.METRIC);
if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(),
MetricDefineByMeasureParams.class));
} else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setMetricDefineByMetricParams(JSONObject.parseObject(metricDO.getTypeParams(),
MetricDefineByMetricParams.class));
} else if (MetricDefineType.FIELD.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setMetricDefineByFieldParams(JSONObject.parseObject(metricDO.getTypeParams(),
MetricDefineByFieldParams.class));
}
if (metricDO.getDefineType() != null) {
metricResp.setMetricDefineType(MetricDefineType.valueOf(metricDO.getDefineType()));
}
return metricResp; return metricResp;
} }

View File

@@ -8,13 +8,14 @@ import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.enums.DimensionType; import com.tencent.supersonic.headless.api.enums.DimensionType;
import com.tencent.supersonic.headless.api.enums.IdentifyType; import com.tencent.supersonic.headless.api.enums.IdentifyType;
import com.tencent.supersonic.headless.api.enums.MetricType; import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.SemanticType; import com.tencent.supersonic.headless.api.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.request.DimensionReq; import com.tencent.supersonic.headless.api.request.DimensionReq;
import com.tencent.supersonic.headless.api.request.MetricReq; import com.tencent.supersonic.headless.api.request.MetricReq;
@@ -116,11 +117,13 @@ public class ModelConverter {
metricReq.setBizName(measure.getBizName().replaceFirst(modelDO.getBizName() + "_", "")); metricReq.setBizName(measure.getBizName().replaceFirst(modelDO.getBizName() + "_", ""));
metricReq.setDescription(measure.getName()); metricReq.setDescription(measure.getName());
metricReq.setModelId(modelDO.getId()); metricReq.setModelId(modelDO.getId());
metricReq.setMetricType(MetricType.ATOMIC); MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
MetricTypeParams exprTypeParams = new MetricTypeParams();
exprTypeParams.setExpr(measure.getBizName()); exprTypeParams.setExpr(measure.getBizName());
exprTypeParams.setMeasures(Lists.newArrayList(measure)); MeasureParam measureParam = new MeasureParam();
BeanMapper.mapper(measure, measureParam);
exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
metricReq.setTypeParams(exprTypeParams); metricReq.setTypeParams(exprTypeParams);
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
return metricReq; return metricReq;
} }

View File

@@ -19,6 +19,7 @@
<result column="data_format" jdbcType="VARCHAR" property="dataFormat" /> <result column="data_format" jdbcType="VARCHAR" property="dataFormat" />
<result column="alias" jdbcType="VARCHAR" property="alias" /> <result column="alias" jdbcType="VARCHAR" property="alias" />
<result column="tags" jdbcType="VARCHAR" property="tags" /> <result column="tags" jdbcType="VARCHAR" property="tags" />
<result column="define_type" jdbcType="VARCHAR" property="defineType" />
</resultMap> </resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO"> <resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO">
<result column="type_params" jdbcType="LONGVARCHAR" property="typeParams" /> <result column="type_params" jdbcType="LONGVARCHAR" property="typeParams" />
@@ -56,7 +57,7 @@
</sql> </sql>
<sql id="Base_Column_List"> <sql id="Base_Column_List">
id, model_id, name, biz_name, description, status, sensitive_level, type, created_at, id, model_id, name, biz_name, description, status, sensitive_level, type, created_at,
created_by, updated_at, updated_by, data_format_type, data_format, alias, tags created_by, updated_at, updated_by, data_format_type, data_format, alias, tags, define_type
</sql> </sql>
<sql id="Blob_Column_List"> <sql id="Blob_Column_List">
type_params type_params
@@ -66,7 +67,7 @@
insert into s2_metric (model_id, name, insert into s2_metric (model_id, name,
biz_name, description, type,status,sensitive_level, biz_name, description, type,status,sensitive_level,
created_at, created_by, updated_at, created_at, created_by, updated_at,
updated_by, type_params updated_by, type_params, define_type
) )
values values
<foreach collection="list" item="metric" separator=","> <foreach collection="list" item="metric" separator=",">
@@ -76,30 +77,12 @@
#{metric.status,jdbcType=VARCHAR},#{metric.sensitiveLevel,jdbcType=VARCHAR}, #{metric.status,jdbcType=VARCHAR},#{metric.sensitiveLevel,jdbcType=VARCHAR},
#{metric.createdAt,jdbcType=TIMESTAMP}, #{metric.createdBy,jdbcType=VARCHAR}, #{metric.createdAt,jdbcType=TIMESTAMP}, #{metric.createdBy,jdbcType=VARCHAR},
#{metric.updatedAt,jdbcType=TIMESTAMP}, #{metric.updatedAt,jdbcType=TIMESTAMP},
#{metric.updatedBy,jdbcType=VARCHAR}, #{metric.typeParams,jdbcType=LONGVARCHAR} #{metric.updatedBy,jdbcType=VARCHAR}, #{metric.typeParams,jdbcType=LONGVARCHAR},
#{metric.defineType,jdbcType=VARCHAR}
) )
</foreach> </foreach>
</insert> </insert>
<update id="batchUpdate" parameterType="java.util.List">
<foreach collection="list" item="metric" separator=";">
update s2_metric
set model_id = #{metric.modelId,jdbcType=BIGINT},
name = #{metric.name,jdbcType=VARCHAR},
biz_name = #{metric.bizName,jdbcType=VARCHAR},
description = #{metric.description,jdbcType=VARCHAR},
type = #{metric.type,jdbcType=VARCHAR},
status = #{metric.status,jdbcType=VARCHAR},
created_at = #{metric.createdAt,jdbcType=TIMESTAMP},
created_by = #{metric.createdBy,jdbcType=VARCHAR},
updated_at = #{metric.updatedAt,jdbcType=TIMESTAMP},
updated_by = #{metric.updatedBy,jdbcType=VARCHAR},
sensitive_level = #{metric.sensitiveLevel,jdbcType=INTEGER},
type_params = #{metric.typeParams,jdbcType=LONGVARCHAR}
where id = #{metric.id,jdbcType=BIGINT}
</foreach>
</update>
<update id="batchUpdateStatus" parameterType="java.util.List"> <update id="batchUpdateStatus" parameterType="java.util.List">
<foreach collection="list" item="metric" separator=";"> <foreach collection="list" item="metric" separator=";">
update s2_metric update s2_metric

View File

@@ -0,0 +1,160 @@
package com.tencent.supersonic.headless.server.service;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.context.ApplicationEventPublisher;
import java.util.HashMap;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
public class MetricServiceImplTest {
@Test
void createMetric() throws Exception {
MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
ModelService modelService = Mockito.mock(ModelService.class);
MetricService metricService = mockMetricService(metricRepository, modelService);
MetricReq metricReq = buildMetricReq();
when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
MetricResp actualMetricResp = metricService.createMetric(metricReq, User.getFakeUser());
MetricResp expectedMetricResp = buildExpectedMetricResp();
Assertions.assertEquals(expectedMetricResp, actualMetricResp);
}
@Test
void updateMetric() throws Exception {
MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
ModelService modelService = Mockito.mock(ModelService.class);
MetricService metricService = mockMetricService(metricRepository, modelService);
MetricReq metricReq = buildMetricUpdateReq();
when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
MetricDO metricDO = MetricConverter.convert2MetricDO(buildMetricReq());
when(metricRepository.getMetricById(metricDO.getId())).thenReturn(metricDO);
MetricResp actualMetricResp = metricService.updateMetric(metricReq, User.getFakeUser());
MetricResp expectedMetricResp = buildExpectedMetricResp();
Assertions.assertEquals(expectedMetricResp, actualMetricResp);
}
private MetricService mockMetricService(MetricRepository metricRepository,
ModelService modelService) {
DomainService domainService = Mockito.mock(DomainService.class);
ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class);
CollectService collectService = Mockito.mock(CollectService.class);
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
return new MetricServiceImpl(metricRepository, modelService, domainService,
chatGptHelper, collectService, eventPublisher);
}
private MetricReq buildMetricReq() {
MetricReq metricReq = new MetricReq();
metricReq.setId(1L);
metricReq.setName("hr部门的访问次数");
metricReq.setBizName("pv");
metricReq.setDescription("SuperSonic的访问情况");
metricReq.setAlias("pv");
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
metricReq.setModelId(2L);
metricReq.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
DataFormat dataFormat = new DataFormat();
dataFormat.setDecimalPlaces(3);
dataFormat.setNeedMultiply100(false);
metricReq.setDataFormat(dataFormat);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricReq.setMetricDefineByMeasureParams(typeParams);
metricReq.setTags(Lists.newArrayList("核心指标"));
metricReq.setRelateDimension(
RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
new DrillDownDimension(1L),
new DrillDownDimension(1L, false))
).build());
metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
metricReq.setExt(new HashMap<>());
return metricReq;
}
private MetricResp buildExpectedMetricResp() {
MetricResp metricResp = new MetricResp();
metricResp.setId(1L);
metricResp.setName("hr部门的访问次数");
metricResp.setBizName("pv");
metricResp.setDescription("SuperSonic的访问情况");
metricResp.setAlias("pv");
metricResp.setMetricDefineType(MetricDefineType.MEASURE);
metricResp.setModelId(2L);
metricResp.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
DataFormat dataFormat = new DataFormat();
dataFormat.setDecimalPlaces(3);
dataFormat.setNeedMultiply100(false);
metricResp.setDataFormat(dataFormat);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricResp.setMetricDefineByMeasureParams(typeParams);
metricResp.setTags(Lists.newArrayList("核心指标"));
metricResp.setRelateDimension(
RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
new DrillDownDimension(1L),
new DrillDownDimension(1L, false))
).build());
metricResp.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
metricResp.setExt(new HashMap<>());
metricResp.setTypeEnum(TypeEnums.METRIC);
metricResp.setIsCollect(false);
metricResp.setType(MetricType.DERIVED.name());
metricResp.setStatus(StatusEnum.ONLINE.getCode());
return metricResp;
}
private MetricReq buildMetricUpdateReq() {
MetricReq metricReq = new MetricReq();
metricReq.setId(1L);
metricReq.setName("hr部门的访问次数");
metricReq.setBizName("pv");
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricReq.setMetricDefineByMeasureParams(typeParams);
return metricReq;
}
private ModelResp mockModelResp() {
ModelResp modelResp = new ModelResp();
modelResp.setId(2L);
modelResp.setDomainId(1L);
return modelResp;
}
}

View File

@@ -56,6 +56,18 @@ class ModelServiceImplTest {
Assertions.assertEquals("alice", actualModelResp.getUpdatedBy()); Assertions.assertEquals("alice", actualModelResp.getUpdatedBy());
} }
@Test
void updateModel_updateAdmin() throws Exception {
ModelRepository modelRepository = Mockito.mock(ModelRepository.class);
ModelService modelService = mockModelService(modelRepository);
ModelReq modelReq = mockModelReq_updateAdmin();
ModelDO modelDO = ModelConverter.convert(mockModelReq(), User.getFakeUser());
when(modelRepository.getModelById(modelReq.getId())).thenReturn(modelDO);
ModelResp actualModelResp = modelService.updateModel(modelReq, User.getFakeUser());
ModelResp expectedModelResp = buildExpectedModelResp();
Assertions.assertEquals(expectedModelResp, actualModelResp);
}
private ModelService mockModelService(ModelRepository modelRepository) { private ModelService mockModelService(ModelRepository modelRepository) {
MetricService metricService = Mockito.mock(MetricService.class); MetricService metricService = Mockito.mock(MetricService.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class); DimensionService dimensionService = Mockito.mock(DimensionService.class);
@@ -137,8 +149,8 @@ class ModelServiceImplTest {
measures.add(measure2); measures.add(measure2);
modelDetail.setMeasures(measures); modelDetail.setMeasures(measures);
modelDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a, user_name " modelDetail.setSqlQuery("SELECT imp_date_a, user_name_a, page_a, 1 as pv_a,"
+ "as uv_a FROM s2_pv_uv_statis"); + " user_name as uv_a FROM s2_pv_uv_statis");
modelDetail.setQueryType("sql_query"); modelDetail.setQueryType("sql_query");
modelReq.setDomainId(1L); modelReq.setDomainId(1L);
modelReq.setFilterSql("where user_name = 'tom'"); modelReq.setFilterSql("where user_name = 'tom'");
@@ -146,6 +158,36 @@ class ModelServiceImplTest {
return modelReq; return modelReq;
} }
private ModelReq mockModelReq_updateAdmin() {
ModelReq modelReq = new ModelReq();
modelReq.setId(1L);
modelReq.setName("PVUV统计");
modelReq.setBizName("s2_pv_uv_statis");
ModelDetail modelDetail = new ModelDetail();
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyType.primary.name(), "user_name"));
modelDetail.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionType.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
Dim dimension2 = new Dim("", "page", DimensionType.categorical.name(), 0);
dimension2.setExpr("page");
dimensions.add(dimension2);
modelDetail.setDimensions(dimensions);
List<Measure> measures = new ArrayList<>();
Measure measure1 = new Measure("访问次数", "pv", AggOperatorEnum.SUM.name(), 1);
measures.add(measure1);
Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1);
measures.add(measure2);
modelDetail.setMeasures(measures);
modelDetail.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, "
+ "user_name as uv FROM s2_pv_uv_statis");
modelDetail.setQueryType("sql_query");
modelReq.setModelDetail(modelDetail);
return modelReq;
}
private ModelResp buildExpectedModelResp() { private ModelResp buildExpectedModelResp() {
ModelResp modelResp = new ModelResp(); ModelResp modelResp = new ModelResp();
modelResp.setName("PVUV统计"); modelResp.setName("PVUV统计");
@@ -197,6 +239,7 @@ class ModelServiceImplTest {
modelResp.setDescription("PVUV统计_a"); modelResp.setDescription("PVUV统计_a");
modelResp.setDatabaseId(2L); modelResp.setDatabaseId(2L);
modelResp.setDomainId(1L); modelResp.setDomainId(1L);
modelResp.setStatus(StatusEnum.ONLINE.getCode());
modelResp.setAlias("访问次数统计,PVUV统计"); modelResp.setAlias("访问次数统计,PVUV统计");
modelResp.setAdmins(Lists.newArrayList("admin")); modelResp.setAdmins(Lists.newArrayList("admin"));
modelResp.setViewers(Lists.newArrayList("alice")); modelResp.setViewers(Lists.newArrayList("alice"));

View File

@@ -15,12 +15,14 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.headless.api.enums.DataType; import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.enums.DimensionType; import com.tencent.supersonic.headless.api.enums.DimensionType;
import com.tencent.supersonic.headless.api.enums.IdentifyType; import com.tencent.supersonic.headless.api.enums.IdentifyType;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.SemanticType; import com.tencent.supersonic.headless.api.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams; import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.ModelDetail; import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.request.DatabaseReq; import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.request.DimensionReq; import com.tencent.supersonic.headless.api.request.DimensionReq;
@@ -329,15 +331,16 @@ public class ModelDemoDataLoader {
metricReq.setDescription("停留时长"); metricReq.setDescription("停留时长");
metricReq.setTags(Collections.singletonList("核心指标")); metricReq.setTags(Collections.singletonList("核心指标"));
metricReq.setAlias("访问时长"); metricReq.setAlias("访问时长");
MetricTypeParams metricTypeParams = new MetricTypeParams(); MetricDefineByMeasureParams metricTypeParams = new MetricDefineByMeasureParams();
metricTypeParams.setExpr("s2_stay_time_statis_stay_hours"); metricTypeParams.setExpr("s2_stay_time_statis_stay_hours");
List<Measure> measures = new ArrayList<>(); List<MeasureParam> measures = new ArrayList<>();
Measure measure = new Measure("停留时长", MeasureParam measure = new MeasureParam("s2_stay_time_statis_stay_hours",
"s2_stay_time_statis_stay_hours", AggOperatorEnum.SUM.getOperator(), 1); "", AggOperatorEnum.SUM.getOperator());
measures.add(measure); measures.add(measure);
metricTypeParams.setMeasures(measures); metricTypeParams.setMeasures(measures);
metricReq.setTypeParams(metricTypeParams); metricReq.setTypeParams(metricTypeParams);
metricService.updateExprMetric(metricReq, user); metricReq.setMetricDefineType(MetricDefineType.MEASURE);
metricService.updateMetric(metricReq, user);
} }
public void addAuthGroup_1() { public void addAuthGroup_1() {

View File

@@ -174,9 +174,9 @@ CREATE TABLE IF NOT EXISTS `s2_metric` (
`name` varchar(255) NOT NULL , `name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL , `description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal `status` INT NOT NULL ,
`sensitive_level` INT NOT NULL , `sensitive_level` INT NOT NULL ,
`type` varchar(50) NOT NULL , -- type proxy,expr `type` varchar(50) NOT NULL , -- ATOMIC, DERIVED
`type_params` LONGVARCHAR DEFAULT NULL , `type_params` LONGVARCHAR DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL , `created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL , `created_by` varchar(100) NOT NULL ,
@@ -188,6 +188,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` (
`tags` varchar(500) DEFAULT NULL, `tags` varchar(500) DEFAULT NULL,
`relate_dimensions` varchar(500) DEFAULT NULL, `relate_dimensions` varchar(500) DEFAULT NULL,
`ext` LONGVARCHAR DEFAULT NULL , `ext` LONGVARCHAR DEFAULT NULL ,
`define_type` varchar(50) NOT NULL, -- MEASURE, FIELD, METRIC
PRIMARY KEY (`id`) PRIMARY KEY (`id`)
); );
COMMENT ON TABLE s2_metric IS 'metric information table'; COMMENT ON TABLE s2_metric IS 'metric information table';

View File

@@ -279,28 +279,31 @@ CREATE TABLE `s2_domain` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='主题域基础信息表'; ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='主题域基础信息表';
CREATE TABLE `s2_metric` ( CREATE TABLE `s2_metric`
`id` bigint(20) NOT NULL AUTO_INCREMENT, (
`model_id` bigint(20) DEFAULT NULL, `id` bigint(20) NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL COMMENT '指标名称', `model_id` bigint(20) DEFAULT NULL,
`biz_name` varchar(255) NOT NULL COMMENT '字段名称', `name` varchar(255) NOT NULL COMMENT '指标名称',
`description` varchar(500) DEFAULT NULL COMMENT '描述', `biz_name` varchar(255) NOT NULL COMMENT '字段名称',
`status` int(10) NOT NULL COMMENT '指标状态,0未启用,1启用', `description` varchar(500) DEFAULT NULL COMMENT '描述',
`sensitive_level` int(10) NOT NULL COMMENT '敏感级别', `status` int(10) NOT NULL COMMENT '指标状态',
`type` varchar(50) NOT NULL COMMENT '指标类型', `sensitive_level` int(10) NOT NULL COMMENT '敏感级别',
`type_params` text NOT NULL COMMENT '类型参数', `type` varchar(50) NOT NULL COMMENT '指标类型',
`created_at` datetime NOT NULL COMMENT '创建时间', `type_params` text NOT NULL COMMENT '类型参数',
`created_by` varchar(100) NOT NULL COMMENT '创建', `created_at` datetime NOT NULL COMMENT '创建时间',
`updated_at` datetime NOT NULL COMMENT '更新时间', `created_by` varchar(100) NOT NULL COMMENT '创建人',
`updated_by` varchar(100) NOT NULL COMMENT '更新', `updated_at` datetime NOT NULL COMMENT '更新时间',
`data_format_type` varchar(50) DEFAULT NULL COMMENT '数值类型', `updated_by` varchar(100) NOT NULL COMMENT '更新人',
`data_format` varchar(500) DEFAULT NULL COMMENT '数值类型参数', `data_format_type` varchar(50) DEFAULT NULL COMMENT '数值类型',
`alias` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL, `data_format` varchar(500) DEFAULT NULL COMMENT '数值类型参数',
`tags` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL, `alias` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`relate_dimensions` varchar(500) DEFAULT NULL COMMENT '指标相关维度', `tags` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`ext` text DEFAULT NULL , `relate_dimensions` varchar(500) DEFAULT NULL COMMENT '指标相关维度',
PRIMARY KEY (`id`) `ext` text DEFAULT NULL,
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='指标表'; `define_type` varchar(50) DEFAULT NULL, -- MEASURE, FIELD, METRIC
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8 COMMENT ='指标表';
CREATE TABLE `s2_model` ( CREATE TABLE `s2_model` (

View File

@@ -164,3 +164,8 @@ CREATE TABLE `s2_app`
`created_by` varchar(255) null, `created_by` varchar(255) null,
`updated_by` varchar(255) null `updated_by` varchar(255) null
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--20240115
alter table s2_metric add column `define_type` varchar(50) DEFAULT NULL; -- MEASURE, FIELD, METRIC
update s2_metric set define_type = 'MEASURE';