diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java index 1f2da6ddb..5e798ecb9 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java @@ -35,6 +35,12 @@ public class Aggregator { this.args = args; } + public Aggregator(String column, AggOperatorEnum func, String alias) { + this.column = column; + this.func = func; + this.alias = alias; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); @@ -46,6 +52,8 @@ public class Aggregator { .append(nameCh).append('\"'); sb.append(",\"args\":") .append(args); + sb.append(",\"alias\":") + .append(alias); sb.append('}'); return sb.toString(); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java new file mode 100644 index 000000000..58026c85f --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ValueDistribution.java @@ -0,0 +1,19 @@ +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Builder; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +@Builder +public class ValueDistribution { + + private Object valueMap; + + private Long totalCount; + + private Long valueCount; + + private Double ratio; +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java new file mode 100644 index 000000000..bb05a6a11 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ItemValueReq.java @@ -0,0 +1,22 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import lombok.Data; +import lombok.ToString; + +import javax.validation.constraints.NotNull; + +@Data +@ToString +public class ItemValueReq { + + private SchemaElementType type = SchemaElementType.TAG; + + @NotNull + private Long itemId; + + private DateConf dateConf; + + private Long limit = 10L; +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java new file mode 100644 index 000000000..7f27868a5 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ItemValueResp.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.ValueDistribution; +import lombok.Data; +import lombok.ToString; + +import java.util.List; + +@Data +@ToString +public class ItemValueResp { + private SchemaElementType type; + private Long itemId; + private String name; + private String bizName; + private List valueDistributionList; +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java index 1a32a5d6c..e462fb924 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java @@ -3,13 +3,18 @@ package com.tencent.supersonic.headless.server.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.TagReq; +import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp; import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.server.pojo.TagFilterPage; -import com.tencent.supersonic.headless.server.service.TagService; +import com.tencent.supersonic.headless.server.service.TagMetaService; + import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; + +import com.tencent.supersonic.headless.server.service.TagQueryService; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -22,9 +27,12 @@ import org.springframework.web.bind.annotation.RestController; @RequestMapping("/api/semantic/tag") public class TagController { - private final TagService tagService; - public TagController(TagService tagService) { - this.tagService = tagService; + private final TagMetaService tagMetaService; + private final TagQueryService tagQueryService; + public TagController(TagMetaService tagMetaService, + TagQueryService tagQueryService) { + this.tagMetaService = tagMetaService; + this.tagQueryService = tagQueryService; } @PostMapping("/create") @@ -32,7 +40,7 @@ public class TagController { HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - return tagService.create(tagReq, user); + return tagMetaService.create(tagReq, user); } @PostMapping("/update") @@ -40,7 +48,7 @@ public class TagController { HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - return tagService.update(tagReq, user); + return tagMetaService.update(tagReq, user); } @PostMapping("/batchUpdateStatus") @@ -48,7 +56,7 @@ public class TagController { HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - return tagService.batchUpdateStatus(metaBatchReq, user); + return tagMetaService.batchUpdateStatus(metaBatchReq, user); } @DeleteMapping("delete/{id}") @@ -56,7 +64,7 @@ public class TagController { HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - tagService.delete(id, user); + tagMetaService.delete(id, user); return true; } @@ -65,7 +73,7 @@ public class TagController { HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - return tagService.getTag(id, user); + return tagMetaService.getTag(id, user); } @PostMapping("/queryTag") @@ -73,7 +81,24 @@ public class TagController { HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - return tagService.queryPage(tagFilterPage, user); + return tagMetaService.queryPage(tagFilterPage, user); + } + + + /** + * 获取标签值分布信息 + * @param itemValueReq + * @param request + * @param response + * @return + * @throws Exception + */ + @PostMapping("/value/distribution") + public ItemValueResp queryTagValue(@RequestBody ItemValueReq itemValueReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + User user = UserHolder.findUser(request, response); + return tagQueryService.queryTagValue(itemValueReq, user); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java index 664841991..e5375676b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java @@ -26,5 +26,4 @@ public interface QueryService { @ApiHeaderCheck ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception; - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagMetaService.java similarity index 96% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagMetaService.java index 5e92994e1..33ca2fcad 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagMetaService.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.pojo.TagFilterPage; import java.util.List; -public interface TagService { +public interface TagMetaService { TagResp create(TagReq tagReq, User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagQueryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagQueryService.java new file mode 100644 index 000000000..dd930cd66 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagQueryService.java @@ -0,0 +1,10 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq; +import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp; + +public interface TagQueryService { + + ItemValueResp queryTagValue(ItemValueReq itemValueReq, User user) throws Exception; +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index c02ea0e4d..a48672fb0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -43,7 +43,7 @@ import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; -import com.tencent.supersonic.headless.server.service.TagService; +import com.tencent.supersonic.headless.server.service.TagMetaService; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter; import com.tencent.supersonic.headless.server.utils.StatUtils; @@ -81,7 +81,7 @@ public class SchemaServiceImpl implements SchemaService { private final DomainService domainService; private final DataSetService dataSetService; private final ModelRelaService modelRelaService; - private final TagService tagService; + private final TagMetaService tagService; public SchemaServiceImpl(ModelService modelService, DimensionService dimensionService, @@ -89,7 +89,7 @@ public class SchemaServiceImpl implements SchemaService { DomainService domainService, DataSetService dataSetService, ModelRelaService modelRelaService, - StatUtils statUtils, TagService tagService) { + StatUtils statUtils, TagMetaService tagService) { this.modelService = modelService; this.dimensionService = dimensionService; this.metricService = metricService; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java similarity index 97% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java index 2e111178c..d5e28d975 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java @@ -26,7 +26,7 @@ import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.pojo.TagFilterPage; import com.tencent.supersonic.headless.server.service.CollectService; import com.tencent.supersonic.headless.server.service.ModelService; -import com.tencent.supersonic.headless.server.service.TagService; +import com.tencent.supersonic.headless.server.service.TagMetaService; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; import java.util.ArrayList; import java.util.Arrays; @@ -46,15 +46,15 @@ import org.springframework.stereotype.Service; @Service @Slf4j -public class TagServiceImpl implements TagService { +public class TagMetaServiceImpl implements TagMetaService { private final TagRepository tagRepository; private final ModelService modelService; private final CollectService collectService; private ApplicationEventPublisher eventPublisher; - public TagServiceImpl(TagRepository tagRepository, ModelService modelService, - CollectService collectService, ApplicationEventPublisher eventPublisher) { + public TagMetaServiceImpl(TagRepository tagRepository, ModelService modelService, + CollectService collectService, ApplicationEventPublisher eventPublisher) { this.tagRepository = tagRepository; this.modelService = modelService; this.collectService = collectService; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java new file mode 100644 index 000000000..8e9dcaa6e --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java @@ -0,0 +1,127 @@ +package com.tencent.supersonic.headless.server.service.impl; + + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.ValueDistribution; +import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; +import com.tencent.supersonic.headless.api.pojo.response.ItemValueResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; +import com.tencent.supersonic.headless.server.service.QueryService; +import com.tencent.supersonic.headless.server.service.TagMetaService; +import com.tencent.supersonic.headless.server.service.TagQueryService; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER; + +@Service +public class TagQueryServiceImpl implements TagQueryService { + + private final String tagValueAlias = "internalTagCount"; + private final TagMetaService tagMetaService; + private final QueryService queryService; + + public TagQueryServiceImpl(TagMetaService tagMetaService, QueryService queryService) { + this.tagMetaService = tagMetaService; + this.queryService = queryService; + } + + @Override + public ItemValueResp queryTagValue(ItemValueReq itemValueReq, User user) throws Exception { + ItemValueResp itemValueResp = new ItemValueResp(); + itemValueResp.setItemId(itemValueReq.getItemId()); + itemValueResp.setType(SchemaElementType.TAG); + TagResp tag = tagMetaService.getTag(itemValueReq.getItemId(), user); + itemValueResp.setName(tag.getName()); + itemValueResp.setBizName(tag.getBizName()); + // tag total count + Long totalCount = queryTagTotalCount(tag, itemValueReq, user); + // tag value + QueryTagReq queryTagReq = generateTagReq(tag, itemValueReq); + SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryTagReq, user); + fillTagValueInfo(itemValueResp, semanticQueryResp, totalCount); + return itemValueResp; + } + + private Long queryTagTotalCount(TagResp tag, ItemValueReq itemValueReq, User user) throws Exception { + + QueryTagReq queryTagReq = new QueryTagReq(); + queryTagReq.addModelId(tag.getModelId()); + queryTagReq.setLimit(1L); + List aggregators = new ArrayList<>(); + aggregators.add(new Aggregator(tag.getBizName(), AggOperatorEnum.COUNT, tagValueAlias)); + queryTagReq.setAggregators(aggregators); + DateConf dateConf = generateDateConf(itemValueReq); + queryTagReq.setDateInfo(dateConf); + + SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryTagReq, user); + if (!CollectionUtils.isEmpty(semanticQueryResp.getResultList())) { + Object total = semanticQueryResp.getResultList().get(0).get(tagValueAlias); + if (Objects.nonNull(total)) { + return Long.parseLong(total.toString()); + } + } + + return Long.MAX_VALUE; + } + + private void fillTagValueInfo(ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp, Long totalCount) { + List valueDistributionList = new ArrayList<>(); + List> resultList = semanticQueryResp.getResultList(); + if (!CollectionUtils.isEmpty(resultList)) { + resultList.stream().forEach(line -> { + Object tagValue = line.get(itemValueResp.getBizName()); + Long tagValueCount = Long.parseLong(line.get(tagValueAlias).toString()); + valueDistributionList.add(ValueDistribution.builder() + .totalCount(totalCount) + .valueMap(tagValue) + .valueCount(tagValueCount) + .ratio(1.0 * tagValueCount / totalCount).build()); + }); + } + itemValueResp.setValueDistributionList(valueDistributionList); + } + + private QueryTagReq generateTagReq(TagResp tag, ItemValueReq itemValueReq) { + QueryTagReq queryTagReq = new QueryTagReq(); + queryTagReq.addModelId(tag.getModelId()); + queryTagReq.setGroups(new ArrayList<>(Arrays.asList(tag.getBizName()))); + queryTagReq.setLimit(itemValueReq.getLimit()); + + List aggregators = new ArrayList<>(); + aggregators.add(new Aggregator(tag.getBizName(), AggOperatorEnum.COUNT, tagValueAlias)); + queryTagReq.setAggregators(aggregators); + + List orders = new ArrayList<>(); + orders.add(new Order(String.format("count(%s)", tag.getBizName()), DESC_UPPER)); + queryTagReq.setOrders(orders); + + DateConf dateConf = generateDateConf(itemValueReq); + queryTagReq.setDateInfo(dateConf); + + return queryTagReq; + } + + private DateConf generateDateConf(ItemValueReq itemValueReq) { + DateConf dateConf = itemValueReq.getDateConf(); + if (Objects.isNull(dateConf)) { + dateConf = new DateConf(); + dateConf.setDateMode(DateConf.DateMode.RECENT); + dateConf.setUnit(1); + } + return dateConf; + } +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index c12b3a364..9c5ada64d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -239,6 +239,8 @@ public class DictUtils { } private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) { + // todo tag + String sqlPattern = "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d"; String bizName = dictItemResp.getBizName(); String whereStr = generateWhereStr(dictItemResp);