Tag market support queryTagValue for value distribution info (#794)

common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java

@@ -35,6 +35,12 @@ public class Aggregator {
         this.args = args;
     }
 
+    public Aggregator(String column, AggOperatorEnum func, String alias) {
+        this.column = column;
+        this.func = func;
+        this.alias = alias;
+    }
+
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder("{");
@@ -46,6 +52,8 @@ public class Aggregator {
                 .append(nameCh).append('\"');
         sb.append(",\"args\":")
                 .append(args);
+        sb.append(",\"alias\":")
+                .append(alias);
         sb.append('}');
         return sb.toString();
     }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java

@@ -0,0 +1,19 @@
+package com.tencent.supersonic.headless.api.pojo;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+@Builder
+public class ValueDistribution {
+
+    private Object valueMap;
+
+    private Long totalCount;
+
+    private Long valueCount;
+
+    private Double ratio;
+}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java

@@ -0,0 +1,22 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
+import lombok.Data;
+import lombok.ToString;
+
+import javax.validation.constraints.NotNull;
+
+@Data
+@ToString
+public class ItemValueReq {
+
+    private SchemaElementType type = SchemaElementType.TAG;
+
+    @NotNull
+    private Long itemId;
+
+    private DateConf dateConf;
+
+    private Long limit = 10L;
+}

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java

@@ -0,0 +1,18 @@
+package com.tencent.supersonic.headless.api.pojo.response;
+
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class ItemValueResp {
+    private SchemaElementType type;
+    private Long itemId;
+    private String name;
+    private String bizName;
+    private List<ValueDistribution> valueDistributionList;
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java

@@ -3,13 +3,18 @@ package com.tencent.supersonic.headless.server.rest;
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
 import com.tencent.supersonic.headless.api.pojo.request.TagReq;
+import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp;
 import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
-import com.tencent.supersonic.headless.server.service.TagService;
+import com.tencent.supersonic.headless.server.service.TagMetaService;
+
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+
+import com.tencent.supersonic.headless.server.service.TagQueryService;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -22,9 +27,12 @@ import org.springframework.web.bind.annotation.RestController;
 @RequestMapping("/api/semantic/tag")
 public class TagController {
 
-    private final TagService tagService;
+    private final TagMetaService tagMetaService;
-    public TagController(TagService tagService) {
+    private final TagQueryService tagQueryService;
-        this.tagService = tagService;
+    public TagController(TagMetaService tagMetaService,
+                         TagQueryService tagQueryService) {
+        this.tagMetaService = tagMetaService;
+        this.tagQueryService = tagQueryService;
     }
 
     @PostMapping("/create")
@@ -32,7 +40,7 @@ public class TagController {
             HttpServletRequest request,
             HttpServletResponse response) throws Exception {
         User user = UserHolder.findUser(request, response);
-        return tagService.create(tagReq, user);
+        return tagMetaService.create(tagReq, user);
     }
 
     @PostMapping("/update")
@@ -40,7 +48,7 @@ public class TagController {
             HttpServletRequest request,
             HttpServletResponse response) throws Exception {
         User user = UserHolder.findUser(request, response);
-        return tagService.update(tagReq, user);
+        return tagMetaService.update(tagReq, user);
     }
 
     @PostMapping("/batchUpdateStatus")
@@ -48,7 +56,7 @@ public class TagController {
                                      HttpServletRequest request,
                                      HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        return tagService.batchUpdateStatus(metaBatchReq, user);
+        return tagMetaService.batchUpdateStatus(metaBatchReq, user);
     }
 
     @DeleteMapping("delete/{id}")
@@ -56,7 +64,7 @@ public class TagController {
             HttpServletRequest request,
             HttpServletResponse response) throws Exception {
         User user = UserHolder.findUser(request, response);
-        tagService.delete(id, user);
+        tagMetaService.delete(id, user);
         return true;
     }
 
@@ -65,7 +73,7 @@ public class TagController {
             HttpServletRequest request,
             HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        return tagService.getTag(id, user);
+        return tagMetaService.getTag(id, user);
     }
 
     @PostMapping("/queryTag")
@@ -73,7 +81,24 @@ public class TagController {
                                          HttpServletRequest request,
                                          HttpServletResponse response) throws Exception {
         User user = UserHolder.findUser(request, response);
-        return tagService.queryPage(tagFilterPage, user);
+        return tagMetaService.queryPage(tagFilterPage, user);
+    }
+
+
+    /**
+     * 获取标签值分布信息
+     * @param itemValueReq
+     * @param request
+     * @param response
+     * @return
+     * @throws Exception
+     */
+    @PostMapping("/value/distribution")
+    public ItemValueResp queryTagValue(@RequestBody ItemValueReq itemValueReq,
+                                       HttpServletRequest request,
+                                       HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        return tagQueryService.queryTagValue(itemValueReq, user);
     }
 
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java

@@ -26,5 +26,4 @@ public interface QueryService {
 
     @ApiHeaderCheck
     ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception;
-
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagMetaService.java

@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
 import java.util.List;
 
-public interface TagService {
+public interface TagMetaService {
 
     TagResp create(TagReq tagReq, User user);
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagQueryService.java

@@ -0,0 +1,10 @@
+package com.tencent.supersonic.headless.server.service;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
+import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp;
+
+public interface TagQueryService {
+
+    ItemValueResp queryTagValue(ItemValueReq itemValueReq, User user) throws Exception;
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java

@@ -43,7 +43,7 @@ import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelRelaService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.SchemaService;
-import com.tencent.supersonic.headless.server.service.TagService;
+import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.tencent.supersonic.headless.server.utils.DimensionConverter;
 import com.tencent.supersonic.headless.server.utils.MetricConverter;
 import com.tencent.supersonic.headless.server.utils.StatUtils;
@@ -81,7 +81,7 @@ public class SchemaServiceImpl implements SchemaService {
     private final DomainService domainService;
     private final DataSetService dataSetService;
     private final ModelRelaService modelRelaService;
-    private final TagService tagService;
+    private final TagMetaService tagService;
 
     public SchemaServiceImpl(ModelService modelService,
             DimensionService dimensionService,
@@ -89,7 +89,7 @@ public class SchemaServiceImpl implements SchemaService {
             DomainService domainService,
             DataSetService dataSetService,
             ModelRelaService modelRelaService,
-            StatUtils statUtils, TagService tagService) {
+            StatUtils statUtils, TagMetaService tagService) {
         this.modelService = modelService;
         this.dimensionService = dimensionService;
         this.metricService = metricService;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java

@@ -26,7 +26,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter;
 import com.tencent.supersonic.headless.server.pojo.TagFilterPage;
 import com.tencent.supersonic.headless.server.service.CollectService;
 import com.tencent.supersonic.headless.server.service.ModelService;
-import com.tencent.supersonic.headless.server.service.TagService;
+import com.tencent.supersonic.headless.server.service.TagMetaService;
 import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -46,15 +46,15 @@ import org.springframework.stereotype.Service;
 
 @Service
 @Slf4j
-public class TagServiceImpl implements TagService {
+public class TagMetaServiceImpl implements TagMetaService {
 
     private final TagRepository tagRepository;
     private final ModelService modelService;
     private final CollectService collectService;
     private ApplicationEventPublisher eventPublisher;
 
-    public TagServiceImpl(TagRepository tagRepository, ModelService modelService,
+    public TagMetaServiceImpl(TagRepository tagRepository, ModelService modelService,
-            CollectService collectService, ApplicationEventPublisher eventPublisher) {
+                              CollectService collectService, ApplicationEventPublisher eventPublisher) {
         this.tagRepository = tagRepository;
         this.modelService = modelService;
         this.collectService = collectService;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java

@@ -0,0 +1,127 @@
+package com.tencent.supersonic.headless.server.service.impl;
+
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.Aggregator;
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.pojo.Order;
+import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
+import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
+import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
+import com.tencent.supersonic.headless.api.pojo.response.TagResp;
+import com.tencent.supersonic.headless.server.service.QueryService;
+import com.tencent.supersonic.headless.server.service.TagMetaService;
+import com.tencent.supersonic.headless.server.service.TagQueryService;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER;
+
+@Service
+public class TagQueryServiceImpl implements TagQueryService {
+
+    private final String tagValueAlias = "internalTagCount";
+    private final TagMetaService tagMetaService;
+    private final QueryService queryService;
+
+    public TagQueryServiceImpl(TagMetaService tagMetaService, QueryService queryService) {
+        this.tagMetaService = tagMetaService;
+        this.queryService = queryService;
+    }
+
+    @Override
+    public ItemValueResp queryTagValue(ItemValueReq itemValueReq, User user) throws Exception {
+        ItemValueResp itemValueResp = new ItemValueResp();
+        itemValueResp.setItemId(itemValueReq.getItemId());
+        itemValueResp.setType(SchemaElementType.TAG);
+        TagResp tag = tagMetaService.getTag(itemValueReq.getItemId(), user);
+        itemValueResp.setName(tag.getName());
+        itemValueResp.setBizName(tag.getBizName());
+        // tag total count
+        Long totalCount = queryTagTotalCount(tag, itemValueReq, user);
+        // tag value
+        QueryTagReq queryTagReq = generateTagReq(tag, itemValueReq);
+        SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryTagReq, user);
+        fillTagValueInfo(itemValueResp, semanticQueryResp, totalCount);
+        return itemValueResp;
+    }
+
+    private Long queryTagTotalCount(TagResp tag, ItemValueReq itemValueReq, User user) throws Exception {
+
+        QueryTagReq queryTagReq = new QueryTagReq();
+        queryTagReq.addModelId(tag.getModelId());
+        queryTagReq.setLimit(1L);
+        List<Aggregator> aggregators = new ArrayList<>();
+        aggregators.add(new Aggregator(tag.getBizName(), AggOperatorEnum.COUNT, tagValueAlias));
+        queryTagReq.setAggregators(aggregators);
+        DateConf dateConf = generateDateConf(itemValueReq);
+        queryTagReq.setDateInfo(dateConf);
+
+        SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryTagReq, user);
+        if (!CollectionUtils.isEmpty(semanticQueryResp.getResultList())) {
+            Object total = semanticQueryResp.getResultList().get(0).get(tagValueAlias);
+            if (Objects.nonNull(total)) {
+                return Long.parseLong(total.toString());
+            }
+        }
+
+        return Long.MAX_VALUE;
+    }
+
+    private void fillTagValueInfo(ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp, Long totalCount) {
+        List<ValueDistribution> valueDistributionList = new ArrayList<>();
+        List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
+        if (!CollectionUtils.isEmpty(resultList)) {
+            resultList.stream().forEach(line -> {
+                Object tagValue = line.get(itemValueResp.getBizName());
+                Long tagValueCount = Long.parseLong(line.get(tagValueAlias).toString());
+                valueDistributionList.add(ValueDistribution.builder()
+                        .totalCount(totalCount)
+                        .valueMap(tagValue)
+                        .valueCount(tagValueCount)
+                        .ratio(1.0 * tagValueCount / totalCount).build());
+            });
+        }
+        itemValueResp.setValueDistributionList(valueDistributionList);
+    }
+
+    private QueryTagReq generateTagReq(TagResp tag, ItemValueReq itemValueReq) {
+        QueryTagReq queryTagReq = new QueryTagReq();
+        queryTagReq.addModelId(tag.getModelId());
+        queryTagReq.setGroups(new ArrayList<>(Arrays.asList(tag.getBizName())));
+        queryTagReq.setLimit(itemValueReq.getLimit());
+
+        List<Aggregator> aggregators = new ArrayList<>();
+        aggregators.add(new Aggregator(tag.getBizName(), AggOperatorEnum.COUNT, tagValueAlias));
+        queryTagReq.setAggregators(aggregators);
+
+        List<Order> orders = new ArrayList<>();
+        orders.add(new Order(String.format("count(%s)", tag.getBizName()), DESC_UPPER));
+        queryTagReq.setOrders(orders);
+
+        DateConf dateConf = generateDateConf(itemValueReq);
+        queryTagReq.setDateInfo(dateConf);
+
+        return queryTagReq;
+    }
+
+    private DateConf generateDateConf(ItemValueReq itemValueReq) {
+        DateConf dateConf = itemValueReq.getDateConf();
+        if (Objects.isNull(dateConf)) {
+            dateConf = new DateConf();
+            dateConf.setDateMode(DateConf.DateMode.RECENT);
+            dateConf.setUnit(1);
+        }
+        return dateConf;
+    }
+}

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -239,6 +239,8 @@ public class DictUtils {
     }
 
     private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) {
+        // todo tag
+
         String sqlPattern = "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d";
         String bizName = dictItemResp.getBizName();
         String whereStr = generateWhereStr(dictItemResp);