topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 20:51:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) (improvement)(headless) Supports creating new metric by fields and metrics and convert struct to sql (#654)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-01-19 14:51:13 +08:00
committed by GitHub
parent 7af5afc3eb
commit b40670b0e3
27 changed files with 238 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -31,6 +31,16 @@ import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -43,15 +53,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -142,7 +143,8 @@ public class SemanticSchemaManager {
|| identifiers.contains(f.getFieldName())) {
continue;
}
datasource.getMeasures().add(Measure.builder().name(f.getFieldName()).agg("").build());
datasource.getMeasures().add(Measure.builder().name(f.getFieldName())
.expr(f.getFieldName()).agg("").build());
}
}
return datasource;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
View File

@@ -27,6 +27,8 @@ public class MetaFilter {
private List<Long> ids;
private List<String> fieldsDepend;
public MetaFilter(List<Long> modelIds) {
this.modelIds = modelIds;
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java
View File

@@ -40,7 +40,7 @@ public class MetricController {
this.metricService = metricService;
}
@PostMapping("/creatExprMetric")
@PostMapping("/createMetric")
public MetricResp createMetric(@RequestBody MetricReq metricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
@@ -48,7 +48,7 @@ public class MetricController {
return metricService.createMetric(metricReq, user);
}
@PostMapping("/updateExprMetric")
@PostMapping("/updateMetric")
public MetricResp updateMetric(@RequestBody MetricReq metricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {

7
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java
View File

@@ -3,10 +3,12 @@ package com.tencent.supersonic.headless.server.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.ModelReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.ModelService;
import org.springframework.web.bind.annotation.DeleteMapping;
@@ -92,4 +94,9 @@ public class ModelController {
return true;
}
@PostMapping("/getUnAvailableItem")
public UnAvailableItemResp getUnAvailableItem(@RequestBody FieldRemovedReq fieldRemovedReq) {
return modelService.getUnAvailableItem(fieldRemovedReq);
}
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java
View File

@@ -4,13 +4,14 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.ModelReq;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.MeasureResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
@@ -34,7 +35,7 @@ public interface ModelService {
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
List<MeasureResp> getMeasureListOfModel(List<Long> modelIds);
UnAvailableItemResp getUnAvailableItem(FieldRemovedReq fieldRemovedReq);
List<ModelResp> getModelListWithAuth(User user, Long domainId, AuthType authType);

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View File

@@ -215,13 +215,29 @@ public class DimensionServiceImpl implements DimensionService {
DimensionFilter dimensionFilter = new DimensionFilter();
BeanUtils.copyProperties(metaFilter, dimensionFilter);
List<DimensionDO> dimensionDOS = dimensionRepository.getDimension(dimensionFilter);
return convertList(dimensionDOS, modelService.getModelMap());
List<DimensionResp> dimensionResps = convertList(dimensionDOS, modelService.getModelMap());
if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
return filterByField(dimensionResps, metaFilter.getFieldsDepend());
}
return dimensionResps;
}
private List<DimensionResp> getDimensions(Long modelId) {
return getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
}
private List<DimensionResp> filterByField(List<DimensionResp> dimensionResps, List<String> fields) {
List<DimensionResp> dimensionFiltered = Lists.newArrayList();
for (DimensionResp dimensionResp : dimensionResps) {
for (String field : fields) {
if (dimensionResp.getExpr().contains(field)) {
dimensionFiltered.add(dimensionResp);
}
}
}
return dimensionFiltered;
}
@Override
public List<DimensionResp> getDimensionInModelCluster(Long modelId) {
ModelResp modelResp = modelService.getModel(modelId);

36
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -16,6 +16,7 @@ import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.MetricBaseReq;
@@ -206,7 +207,36 @@ public class MetricServiceImpl implements MetricService {
public List<MetricResp> getMetrics(MetaFilter metaFilter) {
MetricFilter metricFilter = new MetricFilter();
BeanUtils.copyProperties(metaFilter, metricFilter);
return convertList(queryMetric(metricFilter), Lists.newArrayList());
List<MetricResp> metricResps = convertList(queryMetric(metricFilter));
if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
return filterByField(metricResps, metaFilter.getFieldsDepend());
}
return metricResps;
}
private List<MetricResp> filterByField(List<MetricResp> metricResps, List<String> fields) {
List<MetricResp> metricRespFiltered = Lists.newArrayList();
for (MetricResp metricResp : metricResps) {
for (String field : fields) {
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
.stream().map(MetricParam::getId).collect(Collectors.toList());
List<MetricResp> metricById = metricResps.stream()
.filter(metric -> ids.contains(metric.getId()))
.collect(Collectors.toList());
for (MetricResp metric : metricById) {
if (metric.getExpr().contains(field)) {
metricRespFiltered.add(metricResp);
}
}
} else {
if (metricResp.getExpr().contains(field)) {
metricRespFiltered.add(metricResp);
}
}
}
}
return metricRespFiltered;
}
@Override
@@ -362,6 +392,10 @@ public class MetricServiceImpl implements MetricService {
return getMetrics(new MetaFilter(modelIds));
}
private List<MetricResp> convertList(List<MetricDO> metricDOS) {
return convertList(metricDOS, Lists.newArrayList());
}
private List<MetricResp> convertList(List<MetricDO> metricDOS, List<Long> collect) {
List<MetricResp> metricResps = Lists.newArrayList();
Map<Long, ModelResp> modelMap = modelService.getModelMap();

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -17,6 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.request.DateInfoReq;
import com.tencent.supersonic.headless.api.request.DimensionReq;
import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
import com.tencent.supersonic.headless.api.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.request.MetricReq;
import com.tencent.supersonic.headless.api.request.ModelReq;
@@ -25,11 +26,11 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.DomainResp;
import com.tencent.supersonic.headless.api.response.MeasureResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.core.manager.DimensionYamlManager;
import com.tencent.supersonic.headless.core.manager.MetricYamlManager;
import com.tencent.supersonic.headless.core.manager.ModelYamlManager;
@@ -197,12 +198,13 @@ public class ModelServiceImpl implements ModelService {
}
@Override
public List<MeasureResp> getMeasureListOfModel(List<Long> modelIds) {
ModelFilter modelFilter = new ModelFilter();
modelFilter.setIds(modelIds);
List<ModelResp> modelResps = getModelList(modelFilter);
return modelResps.stream().flatMap(modelResp -> modelResp.getModelDetail().getMeasures()
.stream().map(measure -> ModelConverter.convert(measure, modelResp))).collect(Collectors.toList());
public UnAvailableItemResp getUnAvailableItem(FieldRemovedReq fieldRemovedReq) {
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(fieldRemovedReq.getModelId()));
metaFilter.setFieldsDepend(fieldRemovedReq.getFields());
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
return UnAvailableItemResp.builder().dimensionResps(dimensionResps)
.metricResps(metricResps).build();
}
private void batchCreateDimension(ModelDO modelDO, User user) throws Exception {

61
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -155,49 +155,7 @@ public class QueryServiceImpl implements QueryService {
@Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception {
SemanticQueryResp semanticQueryResp = null;
TaskStatusEnum state = TaskStatusEnum.SUCCESS;
log.info("[queryStructReq:{}]", queryStructReq);
try {
//1.initStatInfo
statUtils.initStatInfo(queryStructReq, user);
//2.query from cache
Object query = queryCache.query(queryStructReq);
if (Objects.nonNull(query)) {
return (SemanticQueryResp) query;
}
StatUtils.get().setUseResultCache(false);
//3 parse
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
queryStatement = queryParser.parse(queryStatement);
//4 plan
QueryExecutor queryExecutor = queryPlanner.plan(queryStatement);
//5 execute
if (queryExecutor != null) {
semanticQueryResp = queryExecutor.execute(queryStatement);
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
}
}
//6 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp);
if (setCacheSuccess) {
// if semanticQueryResp is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq));
}
if (Objects.isNull(semanticQueryResp)) {
state = TaskStatusEnum.ERROR;
}
return semanticQueryResp;
} catch (Exception e) {
log.error("exception in queryByStruct, e: ", e);
state = TaskStatusEnum.ERROR;
throw e;
} finally {
statUtils.statInfo2DbAsync(state);
}
return (SemanticQueryResp) queryBySql(queryStructReq.convert(queryStructReq), user);
}
private QueryStatement buildQueryStatement(QueryStructReq queryStructReq) throws Exception {
@@ -419,6 +377,17 @@ public class QueryServiceImpl implements QueryService {
return false;
}
private SemanticQueryResp queryByCache(String key, Object queryCmd) {
Object resultObject = cacheManager.get(key);
if (Objects.nonNull(resultObject)) {
log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString());
statUtils.updateResultCacheKey(key);
return (SemanticQueryResp) resultObject;
}
return null;
}
private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
QuerySqlReq querySQLReq = new QuerySqlReq();
List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
@@ -436,4 +405,10 @@ public class QueryServiceImpl implements QueryService {
querySQLReq.setSql(sql);
return querySQLReq;
}
private String getKeyByModelIds(List<Long> modelIds) {
return String.join(",", modelIds.stream()
.map(Object::toString).collect(Collectors.toList()));
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java
View File

@@ -28,7 +28,7 @@ public class MetricCheckUtils {
}
}
if (MetricDefineType.MEASURE.equals(metricReq.getMetricDefineType())) {
MetricDefineByMeasureParams typeParams = metricReq.getTypeParams();
MetricDefineByMeasureParams typeParams = metricReq.getMetricDefineByMeasureParams();
if (typeParams == null) {
throw new InvalidArgumentException("指标定义参数不可为空");
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
View File

@@ -81,7 +81,7 @@ public class MetricConverter {
}
metricResp.setTypeEnum(TypeEnums.METRIC);
if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(),
metricResp.setMetricDefineByMeasureParams(JSONObject.parseObject(metricDO.getTypeParams(),
MetricDefineByMeasureParams.class));
} else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setMetricDefineByMetricParams(JSONObject.parseObject(metricDO.getTypeParams(),

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java
View File

@@ -122,7 +122,7 @@ public class ModelConverter {
MeasureParam measureParam = new MeasureParam();
BeanMapper.mapper(measure, measureParam);
exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
metricReq.setTypeParams(exprTypeParams);
metricReq.setMetricDefineByMeasureParams(exprTypeParams);
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
return metricReq;
}
@@ -202,8 +202,7 @@ public class ModelConverter {
private static ModelDetail getModelDetail(ModelReq modelReq) {
ModelDetail modelDetail = new ModelDetail();
BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
List<Measure> measures = modelDetail.getMeasures();
List<Measure> measures = modelReq.getModelDetail().getMeasures();
for (Measure measure : measures) {
if (StringUtils.isBlank(measure.getBizName())) {
continue;
@@ -216,6 +215,7 @@ public class ModelConverter {
measure.setBizName(String.format("%s_%s", modelReq.getBizName(), oriFieldName));
}
}
BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
return modelDetail;
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -279,7 +279,7 @@ public class QueryReqConverter {
// check metrics has derived
if (!metricResps.stream()
.anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(),
m.getTypeParams()))) {
m.getMetricDefineByMeasureParams()))) {
return;
}
Set<String> allFields = new HashSet<>();
@@ -298,7 +298,8 @@ public class QueryReqConverter {
if (!CollectionUtils.isEmpty(metricResps)) {
for (MetricResp metricResp : metricResps) {
if (metrics.contains(metricResp.getBizName())) {
if (MetricType.isDerived(metricResp.getMetricDefineType(), metricResp.getTypeParams())) {
if (MetricType.isDerived(metricResp.getMetricDefineType(),
metricResp.getMetricDefineByMeasureParams())) {
String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, allMeasures,
dimensionResps,
sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric,

6
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
View File

@@ -88,7 +88,7 @@ public class MetricServiceImplTest {
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricReq.setTypeParams(typeParams);
metricReq.setMetricDefineByMeasureParams(typeParams);
metricReq.setTags(Lists.newArrayList("核心指标"));
metricReq.setRelateDimension(
RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
@@ -119,7 +119,7 @@ public class MetricServiceImplTest {
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricResp.setTypeParams(typeParams);
metricResp.setMetricDefineByMeasureParams(typeParams);
metricResp.setTags(Lists.newArrayList("核心指标"));
metricResp.setRelateDimension(
RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
@@ -146,7 +146,7 @@ public class MetricServiceImplTest {
new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricReq.setTypeParams(typeParams);
metricReq.setMetricDefineByMeasureParams(typeParams);
return metricReq;
}