(improvement)(headless) (improvement)(headless) Supports creating new metric by fields and metrics and convert struct to sql (#654)

Co-authored-by: jolunoluo
2025-12-10 11:07:06 +00:00 · 2024-01-19 14:51:13 +08:00
parent 7af5afc3eb
commit b40670b0e3
27 changed files with 238 additions and 118 deletions
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java
@@ -1,10 +1,14 @@
 package com.tencent.supersonic.headless.api.pojo;

+import lombok.Builder;
 import lombok.Data;

@Data
+@Builder
 public class Field {

    private String fieldName;

+    private String dataType;
+
 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java
@@ -15,8 +15,6 @@ public class Measure {

    private String expr;

-    private String createMetric;
-
    private String bizName;

    private Integer isCreateMetric = 0;
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java
@@ -7,4 +7,6 @@ public abstract class MetricDefineParams {

    private String expr;

+    private String filterSql;
+
 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java
@@ -43,4 +43,28 @@ public class ModelDetail {
                .collect(Collectors.toList());
    }

+    public List<Field> getFields() {
+        if (!CollectionUtils.isEmpty(fields)) {
+            return fields;
+        }
+        List<Field> fieldList = Lists.newArrayList();
+        //Compatible with older versions
+        if (!CollectionUtils.isEmpty(identifiers)) {
+            fieldList.addAll(identifiers.stream()
+                    .map(identify -> Field.builder().fieldName(identify.getFieldName()).build())
+                    .collect(Collectors.toSet()));
+        }
+        if (!CollectionUtils.isEmpty(dimensions)) {
+            fieldList.addAll(dimensions.stream()
+                    .map(dim -> Field.builder().fieldName(dim.getFieldName()).build())
+                    .collect(Collectors.toSet()));
+        }
+        if (!CollectionUtils.isEmpty(measures)) {
+            fieldList.addAll(measures.stream()
+                    .map(measure -> Field.builder().fieldName(measure.getFieldName()).build())
+                    .collect(Collectors.toSet()));
+        }
+        return fieldList;
+    }
+
 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/FieldRemovedReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/FieldRemovedReq.java
@@ -0,0 +1,13 @@
+package com.tencent.supersonic.headless.api.request;
+
+import lombok.Data;
+import java.util.List;
+
+@Data
+public class FieldRemovedReq {
+
+    private Long modelId;
+
+    private List<String> fields;
+
+}
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java
@@ -12,23 +12,24 @@ import lombok.Data;
 public class MetricReq extends MetricBaseReq {

    private MetricDefineType metricDefineType = MetricDefineType.MEASURE;
-    private MetricDefineByMeasureParams typeParams;
+    private MetricDefineByMeasureParams metricDefineByMeasureParams;
    private MetricDefineByFieldParams metricDefineByFieldParams;
    private MetricDefineByMetricParams metricDefineByMetricParams;

    public String getTypeParamsJson() {
-        if (metricDefineByFieldParams != null) {
+        if (MetricDefineType.FIELD.equals(metricDefineType) && metricDefineByFieldParams != null) {
            return JSONObject.toJSONString(metricDefineByFieldParams);
-        } else if (typeParams != null) {
-            return JSONObject.toJSONString(typeParams);
-        } else if (metricDefineByMetricParams != null) {
+        } else if (MetricDefineType.MEASURE.equals(metricDefineType) && metricDefineByMeasureParams != null) {
+            return JSONObject.toJSONString(metricDefineByMeasureParams);
+        } else if (MetricDefineType.METRIC.equals(metricDefineType) && metricDefineByMetricParams != null) {
            return JSONObject.toJSONString(metricDefineByMetricParams);
        }
        return null;
    }

    public MetricType getMetricType() {
-        return MetricType.isDerived(metricDefineType, typeParams) ? MetricType.DERIVED : MetricType.ATOMIC;
+        return MetricType.isDerived(metricDefineType, metricDefineByMeasureParams)
+                ? MetricType.DERIVED : MetricType.ATOMIC;
    }

 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java
@@ -12,10 +12,6 @@ import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateModeUtils;
 import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
@@ -39,6 +35,11 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.util.CollectionUtils;

+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+

@Data
@Slf4j
@@ -266,4 +267,11 @@ public class QueryStructReq extends SemanticQueryReq {
        return sql;
    }

+    public String getModelName() {
+        if (StringUtils.isNotBlank(modelName)) {
+            return modelName;
+        }
+        return "table";
+    }
+
 }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java
@@ -10,15 +10,16 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
+import lombok.Data;
+import lombok.ToString;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
-import lombok.Data;
-import lombok.ToString;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;


@Data
@@ -52,7 +53,7 @@ public class MetricResp extends SchemaItem {

    private MetricDefineType metricDefineType = MetricDefineType.MEASURE;

-    private MetricDefineByMeasureParams typeParams;
+    private MetricDefineByMeasureParams metricDefineByMeasureParams;

    private MetricDefineByFieldParams metricDefineByFieldParams;

@@ -77,9 +78,20 @@ public class MetricResp extends SchemaItem {
    }

    public String getDefaultAgg() {
-        if (typeParams != null
-                && CollectionUtils.isNotEmpty(typeParams.getMeasures())) {
-            return typeParams.getMeasures().get(0).getAgg();
+        if (metricDefineByMeasureParams != null
+                && CollectionUtils.isNotEmpty(metricDefineByMeasureParams.getMeasures())) {
+            return metricDefineByMeasureParams.getMeasures().get(0).getAgg();
+        }
+        return "";
+    }
+
+    public String getExpr() {
+        if (MetricDefineType.MEASURE.equals(metricDefineType)) {
+            return metricDefineByMeasureParams.getExpr();
+        } else if (MetricDefineType.METRIC.equals(metricDefineType)) {
+            return metricDefineByMetricParams.getExpr();
+        } else if (MetricDefineType.FIELD.equals(metricDefineType)) {
+            return metricDefineByFieldParams.getExpr();
        }
        return "";
    }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/UnAvailableItemResp.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/UnAvailableItemResp.java
@@ -0,0 +1,16 @@
+package com.tencent.supersonic.headless.api.response;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+public class UnAvailableItemResp {
+
+    private List<MetricResp> metricResps;
+
+    private List<DimensionResp> dimensionResps;
+
+}
--- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java
+++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java
@@ -46,7 +46,7 @@ public class MetricYamlManager {
        metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
        MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
        if (MetricDefineType.MEASURE.equals(metric.getMetricDefineType())) {
-            MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams();
+            MetricDefineByMeasureParams metricDefineParams = metric.getMetricDefineByMeasureParams();
            metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
            List<MeasureParam> measures = metricDefineParams.getMeasures();
            metricTypeParamsYamlTpl.setMeasures(
--- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java
+++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java
@@ -1,17 +1,18 @@
 package com.tencent.supersonic.headless.core.pojo.yaml;

 import lombok.Data;
+import org.apache.commons.compress.utils.Lists;

 import java.util.List;

@Data
 public class MetricTypeParamsYamlTpl {

-    private List<MeasureYamlTpl> measures;
+    private List<MeasureYamlTpl> measures = Lists.newArrayList();

-    private List<MetricParamYamlTpl> metrics;
+    private List<MetricParamYamlTpl> metrics = Lists.newArrayList();

-    private List<FieldParamYamlTpl> fields;
+    private List<FieldParamYamlTpl> fields = Lists.newArrayList();

    private String expr;

--- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
+++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
@@ -1,12 +1,5 @@
 package com.tencent.supersonic.headless.core.utils;

-import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
-import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.MONTH;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.WEEK;
-
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
@@ -23,6 +16,15 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.response.DimensionResp;
 import com.tencent.supersonic.headless.api.response.MetricResp;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.logging.log4j.util.Strings;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Collections;
@@ -34,14 +36,13 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Triple;
-import org.apache.logging.log4j.util.Strings;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
+
+import static com.tencent.supersonic.common.pojo.Constants.DAY;
+import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
+import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.MONTH;
+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.WEEK;

 /**
 * tools functions to analyze queryStructReq
@@ -333,7 +334,7 @@ public class SqlGenerateUtils {

    public String getExpr(MetricResp metricResp) {
        if (Objects.isNull(metricResp.getMetricDefineType())) {
-            return metricResp.getTypeParams().getExpr();
+            return metricResp.getMetricDefineByMeasureParams().getExpr();
        }
        if (metricResp.getMetricDefineType().equals(MetricDefineType.METRIC)) {
            return metricResp.getMetricDefineByMetricParams().getExpr();
@@ -342,6 +343,6 @@ public class SqlGenerateUtils {
            return metricResp.getMetricDefineByFieldParams().getExpr();
        }
        // measure add agg function
-        return metricResp.getTypeParams().getExpr();
+        return metricResp.getMetricDefineByMeasureParams().getExpr();
    }
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
@@ -31,6 +31,16 @@ import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
 import com.tencent.supersonic.headless.server.service.Catalog;
 import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Triple;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -43,15 +53,6 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.Triple;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.cache.annotation.EnableCaching;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;


@Slf4j
@@ -142,7 +143,8 @@ public class SemanticSchemaManager {
                        || identifiers.contains(f.getFieldName())) {
                    continue;
                }
-                datasource.getMeasures().add(Measure.builder().name(f.getFieldName()).agg("").build());
+                datasource.getMeasures().add(Measure.builder().name(f.getFieldName())
+                        .expr(f.getFieldName()).agg("").build());
            }
        }
        return datasource;
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
@@ -27,6 +27,8 @@ public class MetaFilter {

    private List<Long> ids;

+    private List<String> fieldsDepend;
+
    public MetaFilter(List<Long> modelIds) {
        this.modelIds = modelIds;
    }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java
@@ -40,7 +40,7 @@ public class MetricController {
        this.metricService = metricService;
    }

-    @PostMapping("/creatExprMetric")
+    @PostMapping("/createMetric")
    public MetricResp createMetric(@RequestBody MetricReq metricReq,
                                   HttpServletRequest request,
                                   HttpServletResponse response) throws Exception {
@@ -48,7 +48,7 @@ public class MetricController {
        return metricService.createMetric(metricReq, user);
    }

-    @PostMapping("/updateExprMetric")
+    @PostMapping("/updateMetric")
    public MetricResp updateMetric(@RequestBody MetricReq metricReq,
                                HttpServletRequest request,
                                HttpServletResponse response) throws Exception {
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java
@@ -3,10 +3,12 @@ package com.tencent.supersonic.headless.server.rest;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
+import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
 import com.tencent.supersonic.headless.api.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.request.ModelReq;
 import com.tencent.supersonic.headless.api.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.response.ModelResp;
+import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
 import com.tencent.supersonic.headless.server.pojo.ModelFilter;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import org.springframework.web.bind.annotation.DeleteMapping;
@@ -92,4 +94,9 @@ public class ModelController {
        return true;
    }

+    @PostMapping("/getUnAvailableItem")
+    public UnAvailableItemResp getUnAvailableItem(@RequestBody FieldRemovedReq fieldRemovedReq) {
+        return modelService.getUnAvailableItem(fieldRemovedReq);
+    }
+
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java
@@ -4,13 +4,14 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
+import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
 import com.tencent.supersonic.headless.api.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.request.ModelReq;
 import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
 import com.tencent.supersonic.headless.api.response.DatabaseResp;
-import com.tencent.supersonic.headless.api.response.MeasureResp;
 import com.tencent.supersonic.headless.api.response.ModelResp;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
+import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
 import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
@@ -34,7 +35,7 @@ public interface ModelService {

    ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);

-    List<MeasureResp> getMeasureListOfModel(List<Long> modelIds);
+    UnAvailableItemResp getUnAvailableItem(FieldRemovedReq fieldRemovedReq);

    List<ModelResp> getModelListWithAuth(User user, Long domainId, AuthType authType);

--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
@@ -215,13 +215,29 @@ public class DimensionServiceImpl implements DimensionService {
        DimensionFilter dimensionFilter = new DimensionFilter();
        BeanUtils.copyProperties(metaFilter, dimensionFilter);
        List<DimensionDO> dimensionDOS = dimensionRepository.getDimension(dimensionFilter);
-        return convertList(dimensionDOS, modelService.getModelMap());
+        List<DimensionResp> dimensionResps = convertList(dimensionDOS, modelService.getModelMap());
+        if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
+            return filterByField(dimensionResps, metaFilter.getFieldsDepend());
+        }
+        return dimensionResps;
    }

    private List<DimensionResp> getDimensions(Long modelId) {
        return getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
    }

+    private List<DimensionResp> filterByField(List<DimensionResp> dimensionResps, List<String> fields) {
+        List<DimensionResp> dimensionFiltered = Lists.newArrayList();
+        for (DimensionResp dimensionResp : dimensionResps) {
+            for (String field : fields) {
+                if (dimensionResp.getExpr().contains(field)) {
+                    dimensionFiltered.add(dimensionResp);
+                }
+            }
+        }
+        return dimensionFiltered;
+    }
+
    @Override
    public List<DimensionResp> getDimensionInModelCluster(Long modelId) {
        ModelResp modelResp = modelService.getModel(modelId);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
@@ -16,6 +16,7 @@ import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.ChatGptHelper;
 import com.tencent.supersonic.headless.api.enums.MetricDefineType;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
+import com.tencent.supersonic.headless.api.pojo.MetricParam;
 import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
 import com.tencent.supersonic.headless.api.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.request.MetricBaseReq;
@@ -206,7 +207,36 @@ public class MetricServiceImpl implements MetricService {
    public List<MetricResp> getMetrics(MetaFilter metaFilter) {
        MetricFilter metricFilter = new MetricFilter();
        BeanUtils.copyProperties(metaFilter, metricFilter);
-        return convertList(queryMetric(metricFilter), Lists.newArrayList());
+        List<MetricResp> metricResps = convertList(queryMetric(metricFilter));
+        if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
+            return filterByField(metricResps, metaFilter.getFieldsDepend());
+        }
+        return metricResps;
+    }
+
+    private List<MetricResp> filterByField(List<MetricResp> metricResps, List<String> fields) {
+        List<MetricResp> metricRespFiltered = Lists.newArrayList();
+        for (MetricResp metricResp : metricResps) {
+            for (String field : fields) {
+                if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
+                    List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
+                            .stream().map(MetricParam::getId).collect(Collectors.toList());
+                    List<MetricResp> metricById = metricResps.stream()
+                            .filter(metric -> ids.contains(metric.getId()))
+                            .collect(Collectors.toList());
+                    for (MetricResp metric : metricById) {
+                        if (metric.getExpr().contains(field)) {
+                            metricRespFiltered.add(metricResp);
+                        }
+                    }
+                } else {
+                    if (metricResp.getExpr().contains(field)) {
+                        metricRespFiltered.add(metricResp);
+                    }
+                }
+            }
+        }
+        return metricRespFiltered;
    }

    @Override
@@ -362,6 +392,10 @@ public class MetricServiceImpl implements MetricService {
        return getMetrics(new MetaFilter(modelIds));
    }

+    private List<MetricResp> convertList(List<MetricDO> metricDOS) {
+        return convertList(metricDOS, Lists.newArrayList());
+    }
+
    private List<MetricResp> convertList(List<MetricDO> metricDOS, List<Long> collect) {
        List<MetricResp> metricResps = Lists.newArrayList();
        Map<Long, ModelResp> modelMap = modelService.getModelMap();
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
@@ -17,6 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;
 import com.tencent.supersonic.headless.api.request.DateInfoReq;
 import com.tencent.supersonic.headless.api.request.DimensionReq;
+import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
 import com.tencent.supersonic.headless.api.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.request.MetricReq;
 import com.tencent.supersonic.headless.api.request.ModelReq;
@@ -25,11 +26,11 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.response.DimSchemaResp;
 import com.tencent.supersonic.headless.api.response.DimensionResp;
 import com.tencent.supersonic.headless.api.response.DomainResp;
-import com.tencent.supersonic.headless.api.response.MeasureResp;
 import com.tencent.supersonic.headless.api.response.MetricResp;
 import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
 import com.tencent.supersonic.headless.api.response.ModelResp;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
+import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
 import com.tencent.supersonic.headless.core.manager.DimensionYamlManager;
 import com.tencent.supersonic.headless.core.manager.MetricYamlManager;
 import com.tencent.supersonic.headless.core.manager.ModelYamlManager;
@@ -197,12 +198,13 @@ public class ModelServiceImpl implements ModelService {
    }

    @Override
-    public List<MeasureResp> getMeasureListOfModel(List<Long> modelIds) {
-        ModelFilter modelFilter = new ModelFilter();
-        modelFilter.setIds(modelIds);
-        List<ModelResp> modelResps = getModelList(modelFilter);
-        return modelResps.stream().flatMap(modelResp -> modelResp.getModelDetail().getMeasures()
-                .stream().map(measure -> ModelConverter.convert(measure, modelResp))).collect(Collectors.toList());
+    public UnAvailableItemResp getUnAvailableItem(FieldRemovedReq fieldRemovedReq) {
+        MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(fieldRemovedReq.getModelId()));
+        metaFilter.setFieldsDepend(fieldRemovedReq.getFields());
+        List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
+        List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
+        return UnAvailableItemResp.builder().dimensionResps(dimensionResps)
+                .metricResps(metricResps).build();
    }

    private void batchCreateDimension(ModelDO modelDO, User user) throws Exception {
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
@@ -155,49 +155,7 @@ public class QueryServiceImpl implements QueryService {

    @Override
    public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception {
-        SemanticQueryResp semanticQueryResp = null;
-        TaskStatusEnum state = TaskStatusEnum.SUCCESS;
-        log.info("[queryStructReq:{}]", queryStructReq);
-        try {
-            //1.initStatInfo
-            statUtils.initStatInfo(queryStructReq, user);
-            //2.query from cache
-            Object query = queryCache.query(queryStructReq);
-            if (Objects.nonNull(query)) {
-                return (SemanticQueryResp) query;
-            }
-            StatUtils.get().setUseResultCache(false);
-            //3 parse
-            QueryStatement queryStatement = buildQueryStatement(queryStructReq);
-            queryStatement = queryParser.parse(queryStatement);
-
-            //4 plan
-            QueryExecutor queryExecutor = queryPlanner.plan(queryStatement);
-
-            //5 execute
-            if (queryExecutor != null) {
-                semanticQueryResp = queryExecutor.execute(queryStatement);
-                if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
-                    queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
-                }
-            }
-            //6 reset cache and set stateInfo
-            Boolean setCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp);
-            if (setCacheSuccess) {
-                // if semanticQueryResp is not null, update cache data
-                statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq));
-            }
-            if (Objects.isNull(semanticQueryResp)) {
-                state = TaskStatusEnum.ERROR;
-            }
-            return semanticQueryResp;
-        } catch (Exception e) {
-            log.error("exception in queryByStruct, e: ", e);
-            state = TaskStatusEnum.ERROR;
-            throw e;
-        } finally {
-            statUtils.statInfo2DbAsync(state);
-        }
+        return (SemanticQueryResp) queryBySql(queryStructReq.convert(queryStructReq), user);
    }

    private QueryStatement buildQueryStatement(QueryStructReq queryStructReq) throws Exception {
@@ -419,6 +377,17 @@ public class QueryServiceImpl implements QueryService {
        return false;
    }

+    private SemanticQueryResp queryByCache(String key, Object queryCmd) {
+
+        Object resultObject = cacheManager.get(key);
+        if (Objects.nonNull(resultObject)) {
+            log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString());
+            statUtils.updateResultCacheKey(key);
+            return (SemanticQueryResp) resultObject;
+        }
+        return null;
+    }
+
    private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
        QuerySqlReq querySQLReq = new QuerySqlReq();
        List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
@@ -436,4 +405,10 @@ public class QueryServiceImpl implements QueryService {
        querySQLReq.setSql(sql);
        return querySQLReq;
    }
+
+    private String getKeyByModelIds(List<Long> modelIds) {
+        return String.join(",", modelIds.stream()
+                .map(Object::toString).collect(Collectors.toList()));
+    }
+
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java
@@ -28,7 +28,7 @@ public class MetricCheckUtils {
            }
        }
        if (MetricDefineType.MEASURE.equals(metricReq.getMetricDefineType())) {
-            MetricDefineByMeasureParams typeParams = metricReq.getTypeParams();
+            MetricDefineByMeasureParams typeParams = metricReq.getMetricDefineByMeasureParams();
            if (typeParams == null) {
                throw new InvalidArgumentException("指标定义参数不可为空");
            }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
@@ -81,7 +81,7 @@ public class MetricConverter {
        }
        metricResp.setTypeEnum(TypeEnums.METRIC);
        if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {
-            metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(),
+            metricResp.setMetricDefineByMeasureParams(JSONObject.parseObject(metricDO.getTypeParams(),
                    MetricDefineByMeasureParams.class));
        } else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) {
            metricResp.setMetricDefineByMetricParams(JSONObject.parseObject(metricDO.getTypeParams(),
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java
@@ -122,7 +122,7 @@ public class ModelConverter {
        MeasureParam measureParam = new MeasureParam();
        BeanMapper.mapper(measure, measureParam);
        exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
-        metricReq.setTypeParams(exprTypeParams);
+        metricReq.setMetricDefineByMeasureParams(exprTypeParams);
        metricReq.setMetricDefineType(MetricDefineType.MEASURE);
        return metricReq;
    }
@@ -202,8 +202,7 @@ public class ModelConverter {

    private static ModelDetail getModelDetail(ModelReq modelReq) {
        ModelDetail modelDetail = new ModelDetail();
-        BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
-        List<Measure> measures = modelDetail.getMeasures();
+        List<Measure> measures = modelReq.getModelDetail().getMeasures();
        for (Measure measure : measures) {
            if (StringUtils.isBlank(measure.getBizName())) {
                continue;
@@ -216,6 +215,7 @@ public class ModelConverter {
                measure.setBizName(String.format("%s_%s", modelReq.getBizName(), oriFieldName));
            }
        }
+        BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
        return modelDetail;
    }

--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
@@ -279,7 +279,7 @@ public class QueryReqConverter {
        // check metrics has derived
        if (!metricResps.stream()
                .anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(),
-                        m.getTypeParams()))) {
+                        m.getMetricDefineByMeasureParams()))) {
            return;
        }
        Set<String> allFields = new HashSet<>();
@@ -298,7 +298,8 @@ public class QueryReqConverter {
        if (!CollectionUtils.isEmpty(metricResps)) {
            for (MetricResp metricResp : metricResps) {
                if (metrics.contains(metricResp.getBizName())) {
-                    if (MetricType.isDerived(metricResp.getMetricDefineType(), metricResp.getTypeParams())) {
+                    if (MetricType.isDerived(metricResp.getMetricDefineType(),
+                            metricResp.getMetricDefineByMeasureParams())) {
                        String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, allMeasures,
                                dimensionResps,
                                sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric,
--- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
+++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
@@ -88,7 +88,7 @@ public class MetricServiceImplTest {
                new MeasureParam("s2_pv", "department='hr'"),
                new MeasureParam("s2_uv", "department='hr'")));
        typeParams.setExpr("s2_pv/s2_uv");
-        metricReq.setTypeParams(typeParams);
+        metricReq.setMetricDefineByMeasureParams(typeParams);
        metricReq.setTags(Lists.newArrayList("核心指标"));
        metricReq.setRelateDimension(
                RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
@@ -119,7 +119,7 @@ public class MetricServiceImplTest {
                new MeasureParam("s2_pv", "department='hr'"),
                new MeasureParam("s2_uv", "department='hr'")));
        typeParams.setExpr("s2_pv/s2_uv");
-        metricResp.setTypeParams(typeParams);
+        metricResp.setMetricDefineByMeasureParams(typeParams);
        metricResp.setTags(Lists.newArrayList("核心指标"));
        metricResp.setRelateDimension(
                RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
@@ -146,7 +146,7 @@ public class MetricServiceImplTest {
                new MeasureParam("s2_pv", "department='hr'"),
                new MeasureParam("s2_uv", "department='hr'")));
        typeParams.setExpr("s2_pv/s2_uv");
-        metricReq.setTypeParams(typeParams);
+        metricReq.setMetricDefineByMeasureParams(typeParams);
        return metricReq;
    }

--- a/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java
+++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java
@@ -338,7 +338,7 @@ public class ModelDemoDataLoader {
                "", AggOperatorEnum.SUM.getOperator());
        measures.add(measure);
        metricTypeParams.setMeasures(measures);
-        metricReq.setTypeParams(metricTypeParams);
+        metricReq.setMetricDefineByMeasureParams(metricTypeParams);
        metricReq.setMetricDefineType(MetricDefineType.MEASURE);
        metricService.updateMetric(metricReq, user);
    }