opt createDimension and add createBatchTag (#803)

common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java

@@ -12,6 +12,7 @@ import static com.tencent.supersonic.common.pojo.Constants.YEAR;
 import com.google.common.base.Strings;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
+
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;
@@ -22,6 +23,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.StringJoiner;
 import java.util.regex.Pattern;
+
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -328,6 +330,9 @@ public class DateModeUtils {
     }
 
     public String getDateWhereStr(DateConf dateInfo, ItemDateResp dateDate) {
+        if (Objects.isNull(dateInfo)) {
+            return "";
+        }
         String dateStr = "";
         switch (dateInfo.getDateMode()) {
             case BETWEEN:

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/MetaBatchReq.java

@@ -8,6 +8,13 @@ public class MetaBatchReq {
 
     private List<Long> ids;
 
+    private List<String> bizNames;
+
+    private List<Long> modelIds;
+
+    /**
+     * 最后变更的状态
+     */
     private Integer status;
 
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagBatchCreateReq.java

@@ -0,0 +1,19 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
+import lombok.Data;
+import lombok.ToString;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@ToString
+@Data
+public class TagBatchCreateReq {
+    @NotNull
+    private Long modelId;
+    private SchemaElementType type;
+    private List<Long> itemIds;
+
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagFilter.java

@@ -12,5 +12,6 @@ public class TagFilter extends MetaFilter {
     private String type;
     private List<Integer> statusList;
     private TagDefineType tagDefineType;
+    private List<String> bizNames;
 
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DimensionController.java

@@ -43,12 +43,11 @@ public class DimensionController {
      * @param dimensionReq
      */
     @PostMapping("/createDimension")
-    public Boolean createDimension(@RequestBody DimensionReq dimensionReq,
+    public DimensionResp createDimension(@RequestBody DimensionReq dimensionReq,
                                    HttpServletRequest request,
                                    HttpServletResponse response) throws Exception {
         User user = UserHolder.findUser(request, response);
-        dimensionService.createDimension(dimensionReq, user);
+        return dimensionService.createDimension(dimensionReq, user);
-        return true;
     }
 
     @PostMapping("/updateDimension")

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java

@@ -5,6 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
 import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
+import com.tencent.supersonic.headless.api.pojo.request.TagBatchCreateReq;
 import com.tencent.supersonic.headless.api.pojo.request.TagReq;
 import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
@@ -51,6 +52,22 @@ public class TagController {
         return tagMetaService.create(tagReq, user);
     }
 
+    /**
+     * 从现有维度/指标批量新建标签
+     * @param tagBatchReq
+     * @param request
+     * @param response
+     * @return
+     * @throws Exception
+     */
+    @PostMapping("/create/batch")
+    public Integer createBatch(@RequestBody TagBatchCreateReq tagBatchReq,
+                          HttpServletRequest request,
+                          HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        return tagMetaService.createBatch(tagBatchReq, user);
+    }
+
     /**
      * 编辑标签信息
      * @param tagReq

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java

@@ -22,7 +22,7 @@ public interface DimensionService {
 
     void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
 
-    void createDimension(DimensionReq dimensionReq, User user) throws Exception;
+    DimensionResp createDimension(DimensionReq dimensionReq, User user) throws Exception;
 
     void createDimensionBatch(List<DimensionReq> dimensionReqs, User user) throws Exception;
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagMetaService.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.service;
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
+import com.tencent.supersonic.headless.api.pojo.request.TagBatchCreateReq;
 import com.tencent.supersonic.headless.api.pojo.request.TagReq;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.pojo.TagFilter;
@@ -24,4 +25,6 @@ public interface TagMetaService {
     PageInfo<TagResp> queryPage(TagFilterPage tagFilterPage, User user);
 
     Boolean batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
+
+    Integer createBatch(TagBatchCreateReq tagBatchReq, User user);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java

@@ -97,12 +97,13 @@ public class DimensionServiceImpl implements DimensionService {
     }
 
     @Override
-    public void createDimension(DimensionReq dimensionReq, User user) {
+    public DimensionResp createDimension(DimensionReq dimensionReq, User user) {
         checkExist(Lists.newArrayList(dimensionReq));
         dimensionReq.createdBy(user.getName());
         DimensionDO dimensionDO = DimensionConverter.convert2DimensionDO(dimensionReq);
         dimensionRepository.createDimension(dimensionDO);
         sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD);
+        return DimensionConverter.convert2DimensionResp(dimensionDO);
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java

@@ -13,21 +13,29 @@ import com.tencent.supersonic.common.pojo.enums.EventType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
 import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
+import com.tencent.supersonic.headless.api.pojo.request.TagBatchCreateReq;
 import com.tencent.supersonic.headless.api.pojo.request.TagReq;
+import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
+import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
 import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
 import com.tencent.supersonic.headless.server.persistence.repository.TagRepository;
+import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
 import com.tencent.supersonic.headless.server.service.CollectService;
+import com.tencent.supersonic.headless.server.service.DimensionService;
+import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
@@ -36,12 +44,14 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
+
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.beans.BeanUtils;
 import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 
 @Service
@@ -52,13 +62,18 @@ public class TagMetaServiceImpl implements TagMetaService {
     private final ModelService modelService;
     private final CollectService collectService;
     private ApplicationEventPublisher eventPublisher;
+    private final DimensionService dimensionService;
+    private final MetricService metricService;
 
     public TagMetaServiceImpl(TagRepository tagRepository, ModelService modelService,
-                              CollectService collectService, ApplicationEventPublisher eventPublisher) {
+                              CollectService collectService, ApplicationEventPublisher eventPublisher,
+                              @Lazy DimensionService dimensionService, @Lazy MetricService metricService) {
         this.tagRepository = tagRepository;
         this.modelService = modelService;
         this.collectService = collectService;
         this.eventPublisher = eventPublisher;
+        this.dimensionService = dimensionService;
+        this.metricService = metricService;
     }
 
     @Override
@@ -171,7 +186,7 @@ public class TagMetaServiceImpl implements TagMetaService {
         }
 
         PageInfo<TagDO> tagDOPageInfo = PageHelper.startPage(tagFilterPage.getCurrent(),
-                        tagFilterPage.getPageSize())
+                tagFilterPage.getPageSize())
                 .doSelectPageInfo(() -> getTags(tagFilter));
         PageInfo<TagResp> pageInfo = new PageInfo<>();
         BeanUtils.copyProperties(tagDOPageInfo, pageInfo);
@@ -185,13 +200,14 @@ public class TagMetaServiceImpl implements TagMetaService {
 
     @Override
     public Boolean batchUpdateStatus(MetaBatchReq metaBatchReq, User user) {
-        if (Objects.isNull(metaBatchReq) || CollectionUtils.isEmpty(metaBatchReq.getIds())
+        if (Objects.isNull(metaBatchReq)) {
-                || Objects.isNull(metaBatchReq.getStatus())) {
             return false;
         }
         TagFilter tagFilter = new TagFilter();
-        tagFilter.setIds(metaBatchReq.getIds());
+        BeanUtils.copyProperties(metaBatchReq, tagFilter);
+        tagFilter.setStatus(null);
         List<TagDO> tagDOList = tagRepository.query(tagFilter);
+        log.info("tagFilter:{},{}", tagFilter.getModelIds(), tagFilter.getBizNames());
         if (CollectionUtils.isEmpty(tagDOList)) {
             return true;
         }
@@ -211,6 +227,72 @@ public class TagMetaServiceImpl implements TagMetaService {
         return true;
     }
 
+    @Override
+    public Integer createBatch(TagBatchCreateReq tagLoadReq, User user) {
+        Long modelId = tagLoadReq.getModelId();
+        int num = 0;
+        if (Objects.isNull(tagLoadReq.getType()) || SchemaElementType.DIMENSION.equals(tagLoadReq.getType())) {
+            List<DimensionResp> dimensions = dimensionService.getDimensionInModelCluster(modelId);
+            num += loadDimTagBatch(tagLoadReq, dimensions, user);
+        }
+        if (Objects.isNull(tagLoadReq.getType()) || SchemaElementType.METRIC.equals(tagLoadReq.getType())) {
+            MetaFilter metaFilter = new MetaFilter();
+            List<Long> modelIds = new ArrayList<>();
+            modelIds.add(modelId);
+            List<MetricResp> metrics = metricService.getMetrics(metaFilter);
+            num += loadMetricTagBatch(tagLoadReq, metrics, user);
+        }
+        log.info("loadTagBatch finished ,tag num:{}", num);
+        return num;
+    }
+
+    private int loadMetricTagBatch(TagBatchCreateReq tagLoadReq, List<MetricResp> metrics, User user) {
+        if (!CollectionUtils.isEmpty(tagLoadReq.getItemIds())) {
+            metrics = metrics.stream().filter(metric -> tagLoadReq.getItemIds().contains(metric.getId()))
+                    .collect(Collectors.toList());
+        }
+        metrics.stream().forEach(metric -> {
+            TagReq tagReq = new TagReq();
+            BeanUtils.copyProperties(metric, tagReq);
+            tagReq.setId(null);
+            tagReq.setBizName(metric.getBizName());
+            tagReq.setTagDefineType(TagDefineType.METRIC);
+            TagDefineParams tagDefineParams = new TagDefineParams();
+            tagDefineParams.setExpr(metric.getBizName());
+            tagDefineParams.setDependencies(new ArrayList<>(Arrays.asList(metric.getId())));
+            // tagReq.setSensitiveLevel(metric.getSensitiveLevel());
+            tagReq.setTagDefineParams(tagDefineParams);
+            create(tagReq, user);
+        });
+        return metrics.size();
+    }
+
+    private Integer loadDimTagBatch(TagBatchCreateReq tagLoadReq, List<DimensionResp> dimensions, User user) {
+        if (!CollectionUtils.isEmpty(tagLoadReq.getItemIds())) {
+            dimensions = dimensions.stream().filter(dim -> tagLoadReq.getItemIds().contains(dim.getId()))
+                    .collect(Collectors.toList());
+        }
+        dimensions.stream().forEach(dim -> {
+            TagReq tagReq = new TagReq();
+            BeanUtils.copyProperties(dim, tagReq);
+            tagReq.setId(null);
+            tagReq.setBizName(dim.getBizName());
+            tagReq.setTagDefineType(TagDefineType.DIMENSION);
+            TagDefineParams tagDefineParams = new TagDefineParams();
+            tagDefineParams.setExpr(dim.getBizName());
+            tagDefineParams.setDependencies(new ArrayList<>(Arrays.asList(dim.getId())));
+            // tagReq.setSensitiveLevel(dim.getSensitiveLevel());
+            tagReq.setTagDefineParams(tagDefineParams);
+            try {
+                create(tagReq, user);
+            } catch (Exception e) {
+                log.info("loadDimTagBatch, e:{}", e);
+            }
+
+        });
+        return dimensions.size();
+    }
+
     private TagDO fillUpdateInfo(TagReq tagReq, TagDO tagDO) {
         if (Objects.nonNull(tagDO) && tagDO.getId() > 0) {
             if (Objects.nonNull(tagReq.getExt()) && !tagReq.getExt().isEmpty()) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java

@@ -1,6 +1,5 @@
 package com.tencent.supersonic.headless.server.service.impl;
 
-
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
@@ -10,6 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.Dim;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
 import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
 import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
@@ -19,6 +19,7 @@ import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.QueryService;
 import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.tencent.supersonic.headless.server.service.TagQueryService;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
@@ -27,19 +28,23 @@ import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER;
 
 @Service
+@Slf4j
 public class TagQueryServiceImpl implements TagQueryService {
 
-    @Value("${item.value.date.before:1}")
+    @Value("${item.value.date.before:3}")
     private Integer dayBefore;
 
     private final String tagValueAlias = "internalTagCount";
+    private final String maxDateAlias = "internalMaxDate";
     private final TagMetaService tagMetaService;
     private final QueryService queryService;
     private final ModelService modelService;
@@ -59,7 +64,7 @@ public class TagQueryServiceImpl implements TagQueryService {
         TagResp tag = tagMetaService.getTag(itemValueReq.getItemId(), user);
         itemValueResp.setName(tag.getName());
         itemValueResp.setBizName(tag.getBizName());
-        correctDateConf(itemValueReq, tag);
+        correctDateConf(itemValueReq, tag, user);
         // tag total count
         Long totalCount = queryTagTotalCount(tag, itemValueReq, user);
         // tag value
@@ -69,7 +74,7 @@ public class TagQueryServiceImpl implements TagQueryService {
         return itemValueResp;
     }
 
-    private void correctDateConf(ItemValueReq itemValueReq, TagResp tag) {
+    private void correctDateConf(ItemValueReq itemValueReq, TagResp tag, User user) throws Exception {
         if (Objects.nonNull(itemValueReq.getDateConf())) {
             return;
         }
@@ -78,10 +83,8 @@ public class TagQueryServiceImpl implements TagQueryService {
         if (CollectionUtils.isEmpty(timeDimension)) {
             return;
         }
-
+        // query date info from db
-        Dim dim = timeDimension.get(0);
+        String endDate = queryTagDateFromDbBySql(timeDimension.get(0), tag, user);
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dim.getDateFormat());
-        String endDate = LocalDate.now().plusDays(-dayBefore).format(formatter);
         DateConf dateConf = new DateConf();
         dateConf.setDateMode(DateConf.DateMode.BETWEEN);
         dateConf.setStartDate(endDate);
@@ -89,6 +92,60 @@ public class TagQueryServiceImpl implements TagQueryService {
         itemValueReq.setDateConf(dateConf);
     }
 
+    private String queryTagDate(Dim dim) {
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dim.getDateFormat());
+        String endDate = LocalDate.now().plusDays(-dayBefore).format(formatter);
+        return endDate;
+    }
+
+    private String queryTagDateFromDbByStruct(Dim dim, TagResp tag, User user) throws Exception {
+        QueryTagReq queryTagReq = new QueryTagReq();
+        queryTagReq.addModelId(tag.getModelId());
+        queryTagReq.setLimit(1L);
+        List<Aggregator> aggregators = new ArrayList<>();
+        aggregators.add(new Aggregator(dim.getBizName(), AggOperatorEnum.MAX, maxDateAlias));
+        queryTagReq.setAggregators(aggregators);
+        queryTagReq.setDateInfo(null);
+        log.info("queryTagDateFromDb, queryTagReq:{}", queryTagReq.toCustomizedString());
+        SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryTagReq, user);
+        if (!CollectionUtils.isEmpty(semanticQueryResp.getResultList())) {
+            Object date = semanticQueryResp.getResultList().get(0).get(maxDateAlias);
+            if (Objects.nonNull(date)) {
+                return date.toString();
+            }
+        }
+        throw new RuntimeException("queryTagTotalCount error");
+
+    }
+
+    private String queryTagDateFromDbBySql(Dim dim, TagResp tag, User user) throws Exception {
+
+        String sqlPattern = "select max(%s)  as %s from tbl where %s is not null";
+        String sql = String.format(sqlPattern, dim.getBizName(), maxDateAlias, tag.getBizName());
+        Set<Long> modelIds = new HashSet<>();
+        modelIds.add(tag.getModelId());
+        QuerySqlReq querySqlReq = new QuerySqlReq();
+        querySqlReq.setSql(sql);
+        querySqlReq.setNeedAuth(false);
+        querySqlReq.setModelIds(modelIds);
+        log.info("queryTagDateFromDbBySql, QuerySqlReq:{}", querySqlReq.toCustomizedString());
+        try {
+            SemanticQueryResp semanticQueryResp = queryService.queryByReq(querySqlReq, user);
+            if (!CollectionUtils.isEmpty(semanticQueryResp.getResultList())) {
+                Object date = semanticQueryResp.getResultList().get(0).get(maxDateAlias);
+                if (Objects.nonNull(date)) {
+                    return date.toString();
+                }
+            }
+        } catch (Exception e) {
+            log.warn("queryTagDateFromDbBySql date info e, e:{}", e);
+        }
+        String dateDefault = queryTagDate(dim);
+        log.info("queryTagDate by default, dateDefault:{}.", dateDefault);
+        return dateDefault;
+
+    }
+
     private Long queryTagTotalCount(TagResp tag, ItemValueReq itemValueReq, User user) throws Exception {
 
         QueryTagReq queryTagReq = new QueryTagReq();
@@ -104,6 +161,8 @@ public class TagQueryServiceImpl implements TagQueryService {
         if (!CollectionUtils.isEmpty(semanticQueryResp.getResultList())) {
             Object total = semanticQueryResp.getResultList().get(0).get(tagValueAlias);
             if (Objects.nonNull(total)) {
+                long tagCount = Long.parseLong(total.toString());
+                log.info("queryTagTotalCount, tagCount:{}, tagId:{}", tagCount, tag.getId());
                 return Long.parseLong(total.toString());
             }
         }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DimensionConverter.java

@@ -17,6 +17,7 @@ import org.springframework.beans.BeanUtils;
 import org.springframework.util.CollectionUtils;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -79,10 +80,14 @@ public class DimensionConverter {
         return dimensionResp;
     }
 
+    public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO) {
+        return convert2DimensionResp(dimensionDO, new HashMap<>());
+    }
+
     public static List<DimensionResp> filterByDataSet(List<DimensionResp> dimensionResps, DataSetResp dataSetResp) {
         return dimensionResps.stream().filter(dimensionResp ->
-                        dataSetResp.getAllDimensions().contains(dimensionResp.getId())
+                dataSetResp.getAllDimensions().contains(dimensionResp.getId())
-                                || dataSetResp.getAllIncludeAllModels().contains(dimensionResp.getModelId()))
+                        || dataSetResp.getAllIncludeAllModels().contains(dimensionResp.getModelId()))
                 .collect(Collectors.toList());
     }
 

headless/server/src/main/resources/mapper/custom/TagCustomMapper.xml

@@ -105,6 +105,13 @@
                 #{id}
             </foreach>
         </if>
+        <if test="bizNames != null and bizNames.size >0">
+            and biz_name in
+            <foreach collection="bizNames" index="index" item="bizName" open="(" close=")"
+                     separator=",">
+                #{bizName}
+            </foreach>
+        </if>
         <if test="createdBy != null">
             and created_by = #{createdBy}
         </if>