(improvement)(headless) (improvement)(headless) Supports creating new metric by fields and metrics and convert struct to sql (#654)

Co-authored-by: jolunoluo

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Field.java

@@ -1,10 +1,14 @@
 package com.tencent.supersonic.headless.api.pojo;
 
+import lombok.Builder;
 import lombok.Data;
 
 @Data
+@Builder
 public class Field {
 
     private String fieldName;
 
+    private String dataType;
+
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java

@@ -15,8 +15,6 @@ public class Measure {
 
     private String expr;
 
-    private String createMetric;
-
     private String bizName;
 
     private Integer isCreateMetric = 0;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineParams.java

@@ -7,4 +7,6 @@ public abstract class MetricDefineParams {
 
     private String expr;
 
+    private String filterSql;
+
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java

@@ -43,4 +43,28 @@ public class ModelDetail {
                 .collect(Collectors.toList());
     }
 
+    public List<Field> getFields() {
+        if (!CollectionUtils.isEmpty(fields)) {
+            return fields;
+        }
+        List<Field> fieldList = Lists.newArrayList();
+        //Compatible with older versions
+        if (!CollectionUtils.isEmpty(identifiers)) {
+            fieldList.addAll(identifiers.stream()
+                    .map(identify -> Field.builder().fieldName(identify.getFieldName()).build())
+                    .collect(Collectors.toSet()));
+        }
+        if (!CollectionUtils.isEmpty(dimensions)) {
+            fieldList.addAll(dimensions.stream()
+                    .map(dim -> Field.builder().fieldName(dim.getFieldName()).build())
+                    .collect(Collectors.toSet()));
+        }
+        if (!CollectionUtils.isEmpty(measures)) {
+            fieldList.addAll(measures.stream()
+                    .map(measure -> Field.builder().fieldName(measure.getFieldName()).build())
+                    .collect(Collectors.toSet()));
+        }
+        return fieldList;
+    }
+
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/request/FieldRemovedReq.java

@@ -0,0 +1,13 @@
+package com.tencent.supersonic.headless.api.request;
+
+import lombok.Data;
+import java.util.List;
+
+@Data
+public class FieldRemovedReq {
+
+    private Long modelId;
+
+    private List<String> fields;
+
+}

headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java

@@ -12,23 +12,24 @@ import lombok.Data;
 public class MetricReq extends MetricBaseReq {
 
     private MetricDefineType metricDefineType = MetricDefineType.MEASURE;
-    private MetricDefineByMeasureParams typeParams;
+    private MetricDefineByMeasureParams metricDefineByMeasureParams;
     private MetricDefineByFieldParams metricDefineByFieldParams;
     private MetricDefineByMetricParams metricDefineByMetricParams;
 
     public String getTypeParamsJson() {
-        if (metricDefineByFieldParams != null) {
+        if (MetricDefineType.FIELD.equals(metricDefineType) && metricDefineByFieldParams != null) {
             return JSONObject.toJSONString(metricDefineByFieldParams);
-        } else if (typeParams != null) {
+        } else if (MetricDefineType.MEASURE.equals(metricDefineType) && metricDefineByMeasureParams != null) {
-            return JSONObject.toJSONString(typeParams);
+            return JSONObject.toJSONString(metricDefineByMeasureParams);
-        } else if (metricDefineByMetricParams != null) {
+        } else if (MetricDefineType.METRIC.equals(metricDefineType) && metricDefineByMetricParams != null) {
             return JSONObject.toJSONString(metricDefineByMetricParams);
         }
         return null;
     }
 
     public MetricType getMetricType() {
-        return MetricType.isDerived(metricDefineType, typeParams) ? MetricType.DERIVED : MetricType.ATOMIC;
+        return MetricType.isDerived(metricDefineType, metricDefineByMeasureParams)
+                ? MetricType.DERIVED : MetricType.ATOMIC;
     }
 
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java

@@ -12,10 +12,6 @@ import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateModeUtils;
 import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
@@ -39,6 +35,11 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.util.CollectionUtils;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 
 @Data
 @Slf4j
@@ -266,4 +267,11 @@ public class QueryStructReq extends SemanticQueryReq {
         return sql;
     }
 
+    public String getModelName() {
+        if (StringUtils.isNotBlank(modelName)) {
+            return modelName;
+        }
+        return "table";
+    }
+
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java

@@ -10,15 +10,16 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
+import lombok.Data;
+import lombok.ToString;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
-import lombok.Data;
-import lombok.ToString;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
 
 
 @Data
@@ -52,7 +53,7 @@ public class MetricResp extends SchemaItem {
 
     private MetricDefineType metricDefineType = MetricDefineType.MEASURE;
 
-    private MetricDefineByMeasureParams typeParams;
+    private MetricDefineByMeasureParams metricDefineByMeasureParams;
 
     private MetricDefineByFieldParams metricDefineByFieldParams;
 
@@ -77,9 +78,20 @@ public class MetricResp extends SchemaItem {
     }
 
     public String getDefaultAgg() {
-        if (typeParams != null
+        if (metricDefineByMeasureParams != null
-                && CollectionUtils.isNotEmpty(typeParams.getMeasures())) {
+                && CollectionUtils.isNotEmpty(metricDefineByMeasureParams.getMeasures())) {
-            return typeParams.getMeasures().get(0).getAgg();
+            return metricDefineByMeasureParams.getMeasures().get(0).getAgg();
+        }
+        return "";
+    }
+
+    public String getExpr() {
+        if (MetricDefineType.MEASURE.equals(metricDefineType)) {
+            return metricDefineByMeasureParams.getExpr();
+        } else if (MetricDefineType.METRIC.equals(metricDefineType)) {
+            return metricDefineByMetricParams.getExpr();
+        } else if (MetricDefineType.FIELD.equals(metricDefineType)) {
+            return metricDefineByFieldParams.getExpr();
         }
         return "";
     }

headless/api/src/main/java/com/tencent/supersonic/headless/api/response/UnAvailableItemResp.java

@@ -0,0 +1,16 @@
+package com.tencent.supersonic.headless.api.response;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@Builder
+public class UnAvailableItemResp {
+
+    private List<MetricResp> metricResps;
+
+    private List<DimensionResp> dimensionResps;
+
+}

headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java

@@ -46,7 +46,7 @@ public class MetricYamlManager {
         metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
         MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
         if (MetricDefineType.MEASURE.equals(metric.getMetricDefineType())) {
-            MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams();
+            MetricDefineByMeasureParams metricDefineParams = metric.getMetricDefineByMeasureParams();
             metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
             List<MeasureParam> measures = metricDefineParams.getMeasures();
             metricTypeParamsYamlTpl.setMeasures(

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java

@@ -1,17 +1,18 @@
 package com.tencent.supersonic.headless.core.pojo.yaml;
 
 import lombok.Data;
+import org.apache.commons.compress.utils.Lists;
 
 import java.util.List;
 
 @Data
 public class MetricTypeParamsYamlTpl {
 
-    private List<MeasureYamlTpl> measures;
+    private List<MeasureYamlTpl> measures = Lists.newArrayList();
 
-    private List<MetricParamYamlTpl> metrics;
+    private List<MetricParamYamlTpl> metrics = Lists.newArrayList();
 
-    private List<FieldParamYamlTpl> fields;
+    private List<FieldParamYamlTpl> fields = Lists.newArrayList();
 
     private String expr;
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java

@@ -1,12 +1,5 @@
 package com.tencent.supersonic.headless.core.utils;
 
-import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
-import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.MONTH;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.WEEK;
-
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
@@ -23,6 +16,15 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.response.DimensionResp;
 import com.tencent.supersonic.headless.api.response.MetricResp;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.logging.log4j.util.Strings;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Collections;
@@ -34,14 +36,13 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
+
-import org.apache.commons.lang3.StringUtils;
+import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import org.apache.commons.lang3.tuple.ImmutablePair;
+import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
-import org.apache.commons.lang3.tuple.Triple;
+import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
-import org.apache.logging.log4j.util.Strings;
+import static com.tencent.supersonic.common.pojo.Constants.MONTH;
-import org.springframework.beans.factory.annotation.Value;
+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-import org.springframework.stereotype.Component;
+import static com.tencent.supersonic.common.pojo.Constants.WEEK;
-import org.springframework.util.CollectionUtils;
 
 /**
  * tools functions to analyze queryStructReq
@@ -333,7 +334,7 @@ public class SqlGenerateUtils {
 
     public String getExpr(MetricResp metricResp) {
         if (Objects.isNull(metricResp.getMetricDefineType())) {
-            return metricResp.getTypeParams().getExpr();
+            return metricResp.getMetricDefineByMeasureParams().getExpr();
         }
         if (metricResp.getMetricDefineType().equals(MetricDefineType.METRIC)) {
             return metricResp.getMetricDefineByMetricParams().getExpr();
@@ -342,6 +343,6 @@ public class SqlGenerateUtils {
             return metricResp.getMetricDefineByFieldParams().getExpr();
         }
         // measure add agg function
-        return metricResp.getTypeParams().getExpr();
+        return metricResp.getMetricDefineByMeasureParams().getExpr();
     }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java

@@ -31,6 +31,16 @@ import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
 import com.tencent.supersonic.headless.server.service.Catalog;
 import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Triple;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -43,15 +53,6 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.Triple;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.cache.annotation.EnableCaching;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
 
 
 @Slf4j
@@ -142,7 +143,8 @@ public class SemanticSchemaManager {
                         || identifiers.contains(f.getFieldName())) {
                     continue;
                 }
-                datasource.getMeasures().add(Measure.builder().name(f.getFieldName()).agg("").build());
+                datasource.getMeasures().add(Measure.builder().name(f.getFieldName())
+                        .expr(f.getFieldName()).agg("").build());
             }
         }
         return datasource;

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java

@@ -27,6 +27,8 @@ public class MetaFilter {
 
     private List<Long> ids;
 
+    private List<String> fieldsDepend;
+
     public MetaFilter(List<Long> modelIds) {
         this.modelIds = modelIds;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricController.java

@@ -40,7 +40,7 @@ public class MetricController {
         this.metricService = metricService;
     }
 
-    @PostMapping("/creatExprMetric")
+    @PostMapping("/createMetric")
     public MetricResp createMetric(@RequestBody MetricReq metricReq,
                                    HttpServletRequest request,
                                    HttpServletResponse response) throws Exception {
@@ -48,7 +48,7 @@ public class MetricController {
         return metricService.createMetric(metricReq, user);
     }
 
-    @PostMapping("/updateExprMetric")
+    @PostMapping("/updateMetric")
     public MetricResp updateMetric(@RequestBody MetricReq metricReq,
                                 HttpServletRequest request,
                                 HttpServletResponse response) throws Exception {

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/ModelController.java

@@ -3,10 +3,12 @@ package com.tencent.supersonic.headless.server.rest;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
+import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
 import com.tencent.supersonic.headless.api.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.request.ModelReq;
 import com.tencent.supersonic.headless.api.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.response.ModelResp;
+import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
 import com.tencent.supersonic.headless.server.pojo.ModelFilter;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import org.springframework.web.bind.annotation.DeleteMapping;
@@ -92,4 +94,9 @@ public class ModelController {
         return true;
     }
 
+    @PostMapping("/getUnAvailableItem")
+    public UnAvailableItemResp getUnAvailableItem(@RequestBody FieldRemovedReq fieldRemovedReq) {
+        return modelService.getUnAvailableItem(fieldRemovedReq);
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java

@@ -4,13 +4,14 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
+import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
 import com.tencent.supersonic.headless.api.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.request.ModelReq;
 import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
 import com.tencent.supersonic.headless.api.response.DatabaseResp;
-import com.tencent.supersonic.headless.api.response.MeasureResp;
 import com.tencent.supersonic.headless.api.response.ModelResp;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
+import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
 import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
 import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
@@ -34,7 +35,7 @@ public interface ModelService {
 
     ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
 
-    List<MeasureResp> getMeasureListOfModel(List<Long> modelIds);
+    UnAvailableItemResp getUnAvailableItem(FieldRemovedReq fieldRemovedReq);
 
     List<ModelResp> getModelListWithAuth(User user, Long domainId, AuthType authType);
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java

@@ -215,13 +215,29 @@ public class DimensionServiceImpl implements DimensionService {
         DimensionFilter dimensionFilter = new DimensionFilter();
         BeanUtils.copyProperties(metaFilter, dimensionFilter);
         List<DimensionDO> dimensionDOS = dimensionRepository.getDimension(dimensionFilter);
-        return convertList(dimensionDOS, modelService.getModelMap());
+        List<DimensionResp> dimensionResps = convertList(dimensionDOS, modelService.getModelMap());
+        if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
+            return filterByField(dimensionResps, metaFilter.getFieldsDepend());
+        }
+        return dimensionResps;
     }
 
     private List<DimensionResp> getDimensions(Long modelId) {
         return getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
     }
 
+    private List<DimensionResp> filterByField(List<DimensionResp> dimensionResps, List<String> fields) {
+        List<DimensionResp> dimensionFiltered = Lists.newArrayList();
+        for (DimensionResp dimensionResp : dimensionResps) {
+            for (String field : fields) {
+                if (dimensionResp.getExpr().contains(field)) {
+                    dimensionFiltered.add(dimensionResp);
+                }
+            }
+        }
+        return dimensionFiltered;
+    }
+
     @Override
     public List<DimensionResp> getDimensionInModelCluster(Long modelId) {
         ModelResp modelResp = modelService.getModel(modelId);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java

@@ -16,6 +16,7 @@ import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.ChatGptHelper;
 import com.tencent.supersonic.headless.api.enums.MetricDefineType;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
+import com.tencent.supersonic.headless.api.pojo.MetricParam;
 import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
 import com.tencent.supersonic.headless.api.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.request.MetricBaseReq;
@@ -206,7 +207,36 @@ public class MetricServiceImpl implements MetricService {
     public List<MetricResp> getMetrics(MetaFilter metaFilter) {
         MetricFilter metricFilter = new MetricFilter();
         BeanUtils.copyProperties(metaFilter, metricFilter);
-        return convertList(queryMetric(metricFilter), Lists.newArrayList());
+        List<MetricResp> metricResps = convertList(queryMetric(metricFilter));
+        if (!CollectionUtils.isEmpty(metaFilter.getFieldsDepend())) {
+            return filterByField(metricResps, metaFilter.getFieldsDepend());
+        }
+        return metricResps;
+    }
+
+    private List<MetricResp> filterByField(List<MetricResp> metricResps, List<String> fields) {
+        List<MetricResp> metricRespFiltered = Lists.newArrayList();
+        for (MetricResp metricResp : metricResps) {
+            for (String field : fields) {
+                if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
+                    List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
+                            .stream().map(MetricParam::getId).collect(Collectors.toList());
+                    List<MetricResp> metricById = metricResps.stream()
+                            .filter(metric -> ids.contains(metric.getId()))
+                            .collect(Collectors.toList());
+                    for (MetricResp metric : metricById) {
+                        if (metric.getExpr().contains(field)) {
+                            metricRespFiltered.add(metricResp);
+                        }
+                    }
+                } else {
+                    if (metricResp.getExpr().contains(field)) {
+                        metricRespFiltered.add(metricResp);
+                    }
+                }
+            }
+        }
+        return metricRespFiltered;
     }
 
     @Override
@@ -362,6 +392,10 @@ public class MetricServiceImpl implements MetricService {
         return getMetrics(new MetaFilter(modelIds));
     }
 
+    private List<MetricResp> convertList(List<MetricDO> metricDOS) {
+        return convertList(metricDOS, Lists.newArrayList());
+    }
+
     private List<MetricResp> convertList(List<MetricDO> metricDOS, List<Long> collect) {
         List<MetricResp> metricResps = Lists.newArrayList();
         Map<Long, ModelResp> modelMap = modelService.getModelMap();

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -17,6 +17,7 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;
 import com.tencent.supersonic.headless.api.request.DateInfoReq;
 import com.tencent.supersonic.headless.api.request.DimensionReq;
+import com.tencent.supersonic.headless.api.request.FieldRemovedReq;
 import com.tencent.supersonic.headless.api.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.request.MetricReq;
 import com.tencent.supersonic.headless.api.request.ModelReq;
@@ -25,11 +26,11 @@ import com.tencent.supersonic.headless.api.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.response.DimSchemaResp;
 import com.tencent.supersonic.headless.api.response.DimensionResp;
 import com.tencent.supersonic.headless.api.response.DomainResp;
-import com.tencent.supersonic.headless.api.response.MeasureResp;
 import com.tencent.supersonic.headless.api.response.MetricResp;
 import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
 import com.tencent.supersonic.headless.api.response.ModelResp;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
+import com.tencent.supersonic.headless.api.response.UnAvailableItemResp;
 import com.tencent.supersonic.headless.core.manager.DimensionYamlManager;
 import com.tencent.supersonic.headless.core.manager.MetricYamlManager;
 import com.tencent.supersonic.headless.core.manager.ModelYamlManager;
@@ -197,12 +198,13 @@ public class ModelServiceImpl implements ModelService {
     }
 
     @Override
-    public List<MeasureResp> getMeasureListOfModel(List<Long> modelIds) {
+    public UnAvailableItemResp getUnAvailableItem(FieldRemovedReq fieldRemovedReq) {
-        ModelFilter modelFilter = new ModelFilter();
+        MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(fieldRemovedReq.getModelId()));
-        modelFilter.setIds(modelIds);
+        metaFilter.setFieldsDepend(fieldRemovedReq.getFields());
-        List<ModelResp> modelResps = getModelList(modelFilter);
+        List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
-        return modelResps.stream().flatMap(modelResp -> modelResp.getModelDetail().getMeasures()
+        List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
-                .stream().map(measure -> ModelConverter.convert(measure, modelResp))).collect(Collectors.toList());
+        return UnAvailableItemResp.builder().dimensionResps(dimensionResps)
+                .metricResps(metricResps).build();
     }
 
     private void batchCreateDimension(ModelDO modelDO, User user) throws Exception {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java

@@ -155,49 +155,7 @@ public class QueryServiceImpl implements QueryService {
 
     @Override
     public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception {
-        SemanticQueryResp semanticQueryResp = null;
+        return (SemanticQueryResp) queryBySql(queryStructReq.convert(queryStructReq), user);
-        TaskStatusEnum state = TaskStatusEnum.SUCCESS;
-        log.info("[queryStructReq:{}]", queryStructReq);
-        try {
-            //1.initStatInfo
-            statUtils.initStatInfo(queryStructReq, user);
-            //2.query from cache
-            Object query = queryCache.query(queryStructReq);
-            if (Objects.nonNull(query)) {
-                return (SemanticQueryResp) query;
-            }
-            StatUtils.get().setUseResultCache(false);
-            //3 parse
-            QueryStatement queryStatement = buildQueryStatement(queryStructReq);
-            queryStatement = queryParser.parse(queryStatement);
-
-            //4 plan
-            QueryExecutor queryExecutor = queryPlanner.plan(queryStatement);
-
-            //5 execute
-            if (queryExecutor != null) {
-                semanticQueryResp = queryExecutor.execute(queryStatement);
-                if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
-                    queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
-                }
-            }
-            //6 reset cache and set stateInfo
-            Boolean setCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp);
-            if (setCacheSuccess) {
-                // if semanticQueryResp is not null, update cache data
-                statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq));
-            }
-            if (Objects.isNull(semanticQueryResp)) {
-                state = TaskStatusEnum.ERROR;
-            }
-            return semanticQueryResp;
-        } catch (Exception e) {
-            log.error("exception in queryByStruct, e: ", e);
-            state = TaskStatusEnum.ERROR;
-            throw e;
-        } finally {
-            statUtils.statInfo2DbAsync(state);
-        }
     }
 
     private QueryStatement buildQueryStatement(QueryStructReq queryStructReq) throws Exception {
@@ -419,6 +377,17 @@ public class QueryServiceImpl implements QueryService {
         return false;
     }
 
+    private SemanticQueryResp queryByCache(String key, Object queryCmd) {
+
+        Object resultObject = cacheManager.get(key);
+        if (Objects.nonNull(resultObject)) {
+            log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString());
+            statUtils.updateResultCacheKey(key);
+            return (SemanticQueryResp) resultObject;
+        }
+        return null;
+    }
+
     private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
         QuerySqlReq querySQLReq = new QuerySqlReq();
         List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
@@ -436,4 +405,10 @@ public class QueryServiceImpl implements QueryService {
         querySQLReq.setSql(sql);
         return querySQLReq;
     }
+
+    private String getKeyByModelIds(List<Long> modelIds) {
+        return String.join(",", modelIds.stream()
+                .map(Object::toString).collect(Collectors.toList()));
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java

@@ -28,7 +28,7 @@ public class MetricCheckUtils {
             }
         }
         if (MetricDefineType.MEASURE.equals(metricReq.getMetricDefineType())) {
-            MetricDefineByMeasureParams typeParams = metricReq.getTypeParams();
+            MetricDefineByMeasureParams typeParams = metricReq.getMetricDefineByMeasureParams();
             if (typeParams == null) {
                 throw new InvalidArgumentException("指标定义参数不可为空");
             }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java

@@ -81,7 +81,7 @@ public class MetricConverter {
         }
         metricResp.setTypeEnum(TypeEnums.METRIC);
         if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {
-            metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(),
+            metricResp.setMetricDefineByMeasureParams(JSONObject.parseObject(metricDO.getTypeParams(),
                     MetricDefineByMeasureParams.class));
         } else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) {
             metricResp.setMetricDefineByMetricParams(JSONObject.parseObject(metricDO.getTypeParams(),

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -122,7 +122,7 @@ public class ModelConverter {
         MeasureParam measureParam = new MeasureParam();
         BeanMapper.mapper(measure, measureParam);
         exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
-        metricReq.setTypeParams(exprTypeParams);
+        metricReq.setMetricDefineByMeasureParams(exprTypeParams);
         metricReq.setMetricDefineType(MetricDefineType.MEASURE);
         return metricReq;
     }
@@ -202,8 +202,7 @@ public class ModelConverter {
 
     private static ModelDetail getModelDetail(ModelReq modelReq) {
         ModelDetail modelDetail = new ModelDetail();
-        BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
+        List<Measure> measures = modelReq.getModelDetail().getMeasures();
-        List<Measure> measures = modelDetail.getMeasures();
         for (Measure measure : measures) {
             if (StringUtils.isBlank(measure.getBizName())) {
                 continue;
@@ -216,6 +215,7 @@ public class ModelConverter {
                 measure.setBizName(String.format("%s_%s", modelReq.getBizName(), oriFieldName));
             }
         }
+        BeanMapper.mapper(modelReq.getModelDetail(), modelDetail);
         return modelDetail;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java

@@ -279,7 +279,7 @@ public class QueryReqConverter {
         // check metrics has derived
         if (!metricResps.stream()
                 .anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(),
-                        m.getTypeParams()))) {
+                        m.getMetricDefineByMeasureParams()))) {
             return;
         }
         Set<String> allFields = new HashSet<>();
@@ -298,7 +298,8 @@ public class QueryReqConverter {
         if (!CollectionUtils.isEmpty(metricResps)) {
             for (MetricResp metricResp : metricResps) {
                 if (metrics.contains(metricResp.getBizName())) {
-                    if (MetricType.isDerived(metricResp.getMetricDefineType(), metricResp.getTypeParams())) {
+                    if (MetricType.isDerived(metricResp.getMetricDefineType(),
+                            metricResp.getMetricDefineByMeasureParams())) {
                         String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, allMeasures,
                                 dimensionResps,
                                 sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric,

headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java

@@ -88,7 +88,7 @@ public class MetricServiceImplTest {
                 new MeasureParam("s2_pv", "department='hr'"),
                 new MeasureParam("s2_uv", "department='hr'")));
         typeParams.setExpr("s2_pv/s2_uv");
-        metricReq.setTypeParams(typeParams);
+        metricReq.setMetricDefineByMeasureParams(typeParams);
         metricReq.setTags(Lists.newArrayList("核心指标"));
         metricReq.setRelateDimension(
                 RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
@@ -119,7 +119,7 @@ public class MetricServiceImplTest {
                 new MeasureParam("s2_pv", "department='hr'"),
                 new MeasureParam("s2_uv", "department='hr'")));
         typeParams.setExpr("s2_pv/s2_uv");
-        metricResp.setTypeParams(typeParams);
+        metricResp.setMetricDefineByMeasureParams(typeParams);
         metricResp.setTags(Lists.newArrayList("核心指标"));
         metricResp.setRelateDimension(
                 RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
@@ -146,7 +146,7 @@ public class MetricServiceImplTest {
                 new MeasureParam("s2_pv", "department='hr'"),
                 new MeasureParam("s2_uv", "department='hr'")));
         typeParams.setExpr("s2_pv/s2_uv");
-        metricReq.setTypeParams(typeParams);
+        metricReq.setMetricDefineByMeasureParams(typeParams);
         return metricReq;
     }
 

launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java

@@ -338,7 +338,7 @@ public class ModelDemoDataLoader {
                 "", AggOperatorEnum.SUM.getOperator());
         measures.add(measure);
         metricTypeParams.setMeasures(measures);
-        metricReq.setTypeParams(metricTypeParams);
+        metricReq.setMetricDefineByMeasureParams(metricTypeParams);
         metricReq.setMetricDefineType(MetricDefineType.MEASURE);
         metricService.updateMetric(metricReq, user);
     }