Tag market support queryTagValue for value distribution info (#794)

2025-12-10 19:51:00 +00:00 · 2024-03-06 20:46:59 +08:00
parent 3cccac58a1
commit a6428f7dbf
12 changed files with 249 additions and 19 deletions
--- a/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java
+++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java
@@ -35,6 +35,12 @@ public class Aggregator {
        this.args = args;
    }

+    public Aggregator(String column, AggOperatorEnum func, String alias) {
+        this.column = column;
+        this.func = func;
+        this.alias = alias;
+    }
+
    @Override
    public String toString() {
        final StringBuilder sb = new StringBuilder("{");
@@ -46,6 +52,8 @@ public class Aggregator {
                .append(nameCh).append('\"');
        sb.append(",\"args\":")
                .append(args);
+        sb.append(",\"alias\":")
+                .append(alias);
        sb.append('}');
        return sb.toString();
    }
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java
@@ -0,0 +1,19 @@
+package com.tencent.supersonic.headless.api.pojo;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+@Builder
+public class ValueDistribution {
+
+    private Object valueMap;
+
+    private Long totalCount;
+
+    private Long valueCount;
+
+    private Double ratio;
+}
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java
@@ -0,0 +1,22 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
+import lombok.Data;
+import lombok.ToString;
+
+import javax.validation.constraints.NotNull;
+
+@Data
+@ToString
+public class ItemValueReq {
+
+    private SchemaElementType type = SchemaElementType.TAG;
+
+    @NotNull
+    private Long itemId;
+
+    private DateConf dateConf;
+
+    private Long limit = 10L;
+}
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java
@@ -0,0 +1,18 @@
+package com.tencent.supersonic.headless.api.pojo.response;
+
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class ItemValueResp {
+    private SchemaElementType type;
+    private Long itemId;
+    private String name;
+    private String bizName;
+    private List<ValueDistribution> valueDistributionList;
+}
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java
@@ -3,13 +3,18 @@ package com.tencent.supersonic.headless.server.rest;
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.pojo.request.TagReq;
+import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
-import com.tencent.supersonic.headless.server.service.TagService;
+import com.tencent.supersonic.headless.server.service.TagMetaService;
+
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+
+import com.tencent.supersonic.headless.server.service.TagQueryService;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -22,9 +27,12 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping("/api/semantic/tag")
 public class TagController {

-    private final TagService tagService;
-    public TagController(TagService tagService) {
-        this.tagService = tagService;
+    private final TagMetaService tagMetaService;
+    private final TagQueryService tagQueryService;
+    public TagController(TagMetaService tagMetaService,
+                         TagQueryService tagQueryService) {
+        this.tagMetaService = tagMetaService;
+        this.tagQueryService = tagQueryService;
    }

    @PostMapping("/create")
@@ -32,7 +40,7 @@ public class TagController {
            HttpServletRequest request,
            HttpServletResponse response) throws Exception {
        User user = UserHolder.findUser(request, response);
-        return tagService.create(tagReq, user);
+        return tagMetaService.create(tagReq, user);
    }

    @PostMapping("/update")
@@ -40,7 +48,7 @@ public class TagController {
            HttpServletRequest request,
            HttpServletResponse response) throws Exception {
        User user = UserHolder.findUser(request, response);
-        return tagService.update(tagReq, user);
+        return tagMetaService.update(tagReq, user);
    }

    @PostMapping("/batchUpdateStatus")
@@ -48,7 +56,7 @@ public class TagController {
                                     HttpServletRequest request,
                                     HttpServletResponse response) {
        User user = UserHolder.findUser(request, response);
-        return tagService.batchUpdateStatus(metaBatchReq, user);
+        return tagMetaService.batchUpdateStatus(metaBatchReq, user);
    }

    @DeleteMapping("delete/{id}")
@@ -56,7 +64,7 @@ public class TagController {
            HttpServletRequest request,
            HttpServletResponse response) throws Exception {
        User user = UserHolder.findUser(request, response);
-        tagService.delete(id, user);
+        tagMetaService.delete(id, user);
        return true;
    }

@@ -65,7 +73,7 @@ public class TagController {
            HttpServletRequest request,
            HttpServletResponse response) {
        User user = UserHolder.findUser(request, response);
-        return tagService.getTag(id, user);
+        return tagMetaService.getTag(id, user);
    }

    @PostMapping("/queryTag")
@@ -73,7 +81,24 @@ public class TagController {
                                         HttpServletRequest request,
                                         HttpServletResponse response) throws Exception {
        User user = UserHolder.findUser(request, response);
-        return tagService.queryPage(tagFilterPage, user);
+        return tagMetaService.queryPage(tagFilterPage, user);
+    }
+
+
+    /**
+     * 获取标签值分布信息
+     * @param itemValueReq
+     * @param request
+     * @param response
+     * @return
+     * @throws Exception
+     */
+    @PostMapping("/value/distribution")
+    public ItemValueResp queryTagValue(@RequestBody ItemValueReq itemValueReq,
+                                       HttpServletRequest request,
+                                       HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        return tagQueryService.queryTagValue(itemValueReq, user);
    }

 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
@@ -26,5 +26,4 @@ public interface QueryService {

    @ApiHeaderCheck
    ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception;
-
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagMetaService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagMetaService.java
@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
 import java.util.List;

-public interface TagService {
+public interface TagMetaService {

    TagResp create(TagReq tagReq, User user);

--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagQueryService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagQueryService.java
@@ -0,0 +1,10 @@
+package com.tencent.supersonic.headless.server.service;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
+import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp;
+
+public interface TagQueryService {
+
+    ItemValueResp queryTagValue(ItemValueReq itemValueReq, User user) throws Exception;
+}
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
@@ -43,7 +43,7 @@ import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelRelaService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.SchemaService;
-import com.tencent.supersonic.headless.server.service.TagService;
+import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.tencent.supersonic.headless.server.utils.DimensionConverter;
 import com.tencent.supersonic.headless.server.utils.MetricConverter;
 import com.tencent.supersonic.headless.server.utils.StatUtils;
@@ -81,7 +81,7 @@ public class SchemaServiceImpl implements SchemaService {
    private final DomainService domainService;
    private final DataSetService dataSetService;
    private final ModelRelaService modelRelaService;
-    private final TagService tagService;
+    private final TagMetaService tagService;

    public SchemaServiceImpl(ModelService modelService,
            DimensionService dimensionService,
@@ -89,7 +89,7 @@ public class SchemaServiceImpl implements SchemaService {
            DomainService domainService,
            DataSetService dataSetService,
            ModelRelaService modelRelaService,
-            StatUtils statUtils, TagService tagService) {
+            StatUtils statUtils, TagMetaService tagService) {
        this.modelService = modelService;
        this.dimensionService = dimensionService;
        this.metricService = metricService;
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java
@@ -26,7 +26,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
 import com.tencent.supersonic.headless.server.service.CollectService;
 import com.tencent.supersonic.headless.server.service.ModelService;
-import com.tencent.supersonic.headless.server.service.TagService;
+import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -46,15 +46,15 @@ import org.springframework.stereotype.Service;

@Service
@Slf4j
-public class TagServiceImpl implements TagService {
+public class TagMetaServiceImpl implements TagMetaService {

    private final TagRepository tagRepository;
    private final ModelService modelService;
    private final CollectService collectService;
    private ApplicationEventPublisher eventPublisher;

-    public TagServiceImpl(TagRepository tagRepository, ModelService modelService,
-            CollectService collectService, ApplicationEventPublisher eventPublisher) {
+    public TagMetaServiceImpl(TagRepository tagRepository, ModelService modelService,
+                              CollectService collectService, ApplicationEventPublisher eventPublisher) {
        this.tagRepository = tagRepository;
        this.modelService = modelService;
        this.collectService = collectService;
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java
@@ -0,0 +1,127 @@
+package com.tencent.supersonic.headless.server.service.impl;
+
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.Aggregator;
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.pojo.Order;
+import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
+import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
+import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
+import com.tencent.supersonic.headless.api.pojo.response.TagResp;
+import com.tencent.supersonic.headless.server.service.QueryService;
+import com.tencent.supersonic.headless.server.service.TagMetaService;
+import com.tencent.supersonic.headless.server.service.TagQueryService;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER;
+
+@Service
+public class TagQueryServiceImpl implements TagQueryService {
+
+    private final String tagValueAlias = "internalTagCount";
+    private final TagMetaService tagMetaService;
+    private final QueryService queryService;
+
+    public TagQueryServiceImpl(TagMetaService tagMetaService, QueryService queryService) {
+        this.tagMetaService = tagMetaService;
+        this.queryService = queryService;
+    }
+
+    @Override
+    public ItemValueResp queryTagValue(ItemValueReq itemValueReq, User user) throws Exception {
+        ItemValueResp itemValueResp = new ItemValueResp();
+        itemValueResp.setItemId(itemValueReq.getItemId());
+        itemValueResp.setType(SchemaElementType.TAG);
+        TagResp tag = tagMetaService.getTag(itemValueReq.getItemId(), user);
+        itemValueResp.setName(tag.getName());
+        itemValueResp.setBizName(tag.getBizName());
+        // tag total count
+        Long totalCount = queryTagTotalCount(tag, itemValueReq, user);
+        // tag value
+        QueryTagReq queryTagReq = generateTagReq(tag, itemValueReq);
+        SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryTagReq, user);
+        fillTagValueInfo(itemValueResp, semanticQueryResp, totalCount);
+        return itemValueResp;
+    }
+
+    private Long queryTagTotalCount(TagResp tag, ItemValueReq itemValueReq, User user) throws Exception {
+
+        QueryTagReq queryTagReq = new QueryTagReq();
+        queryTagReq.addModelId(tag.getModelId());
+        queryTagReq.setLimit(1L);
+        List<Aggregator> aggregators = new ArrayList<>();
+        aggregators.add(new Aggregator(tag.getBizName(), AggOperatorEnum.COUNT, tagValueAlias));
+        queryTagReq.setAggregators(aggregators);
+        DateConf dateConf = generateDateConf(itemValueReq);
+        queryTagReq.setDateInfo(dateConf);
+
+        SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryTagReq, user);
+        if (!CollectionUtils.isEmpty(semanticQueryResp.getResultList())) {
+            Object total = semanticQueryResp.getResultList().get(0).get(tagValueAlias);
+            if (Objects.nonNull(total)) {
+                return Long.parseLong(total.toString());
+            }
+        }
+
+        return Long.MAX_VALUE;
+    }
+
+    private void fillTagValueInfo(ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp, Long totalCount) {
+        List<ValueDistribution> valueDistributionList = new ArrayList<>();
+        List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
+        if (!CollectionUtils.isEmpty(resultList)) {
+            resultList.stream().forEach(line -> {
+                Object tagValue = line.get(itemValueResp.getBizName());
+                Long tagValueCount = Long.parseLong(line.get(tagValueAlias).toString());
+                valueDistributionList.add(ValueDistribution.builder()
+                        .totalCount(totalCount)
+                        .valueMap(tagValue)
+                        .valueCount(tagValueCount)
+                        .ratio(1.0 * tagValueCount / totalCount).build());
+            });
+        }
+        itemValueResp.setValueDistributionList(valueDistributionList);
+    }
+
+    private QueryTagReq generateTagReq(TagResp tag, ItemValueReq itemValueReq) {
+        QueryTagReq queryTagReq = new QueryTagReq();
+        queryTagReq.addModelId(tag.getModelId());
+        queryTagReq.setGroups(new ArrayList<>(Arrays.asList(tag.getBizName())));
+        queryTagReq.setLimit(itemValueReq.getLimit());
+
+        List<Aggregator> aggregators = new ArrayList<>();
+        aggregators.add(new Aggregator(tag.getBizName(), AggOperatorEnum.COUNT, tagValueAlias));
+        queryTagReq.setAggregators(aggregators);
+
+        List<Order> orders = new ArrayList<>();
+        orders.add(new Order(String.format("count(%s)", tag.getBizName()), DESC_UPPER));
+        queryTagReq.setOrders(orders);
+
+        DateConf dateConf = generateDateConf(itemValueReq);
+        queryTagReq.setDateInfo(dateConf);
+
+        return queryTagReq;
+    }
+
+    private DateConf generateDateConf(ItemValueReq itemValueReq) {
+        DateConf dateConf = itemValueReq.getDateConf();
+        if (Objects.isNull(dateConf)) {
+            dateConf = new DateConf();
+            dateConf.setDateMode(DateConf.DateMode.RECENT);
+            dateConf.setUnit(1);
+        }
+        return dateConf;
+    }
+}
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
@@ -239,6 +239,8 @@ public class DictUtils {
    }

    private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) {
+        // todo tag
+
        String sqlPattern = "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d";
        String bizName = dictItemResp.getBizName();
        String whereStr = generateWhereStr(dictItemResp);