topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add constructTagQueryReq for dict (#807)

Browse Source
This commit is contained in:
daikon
2024-03-12 21:28:02 +08:00
committed by GitHub
parent cd70096c31
commit 26bef3d472
5 changed files with 110 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Param;
import lombok.Data;
@@ -23,6 +24,8 @@ public abstract class SemanticQueryReq {
protected String dataSetName;
protected QueryType queryType;
protected Set<Long> modelIds = new HashSet<>();
protected List<Param> params = new ArrayList<>();

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictItemResp.java
View File

@@ -32,7 +32,7 @@ public class DictItemResp {
@NotNull
private StatusEnum status;
public String getNature() {
public String generateNature() {
return UNDERLINE + modelId + UNDERLINE + itemId + UNDERLINE + type.name().toLowerCase().substring(0, 1)
+ DICT_VALUE;
@@ -41,4 +41,5 @@ public class DictItemResp {
public String fetchDictFileName() {
return String.format("dic_value_%d_%s_%s", modelId, type.name(), itemId);
}
}

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagMetaServiceImpl.java
View File

@@ -231,14 +231,16 @@ public class TagMetaServiceImpl implements TagMetaService {
public Integer createBatch(TagBatchCreateReq tagLoadReq, User user) {
Long modelId = tagLoadReq.getModelId();
int num = 0;
MetaFilter metaFilter = new MetaFilter();
List<Long> modelIds = new ArrayList<>();
modelIds.add(modelId);
metaFilter.setModelIds(modelIds);
if (Objects.isNull(tagLoadReq.getType()) || SchemaElementType.DIMENSION.equals(tagLoadReq.getType())) {
List<DimensionResp> dimensions = dimensionService.getDimensionInModelCluster(modelId);
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
num += loadDimTagBatch(tagLoadReq, dimensions, user);
}
if (Objects.isNull(tagLoadReq.getType()) || SchemaElementType.METRIC.equals(tagLoadReq.getType())) {
MetaFilter metaFilter = new MetaFilter();
List<Long> modelIds = new ArrayList<>();
modelIds.add(modelId);
List<MetricResp> metrics = metricService.getMetrics(metaFilter);
num += loadMetricTagBatch(tagLoadReq, metrics, user);
}
@@ -369,6 +371,7 @@ public class TagMetaServiceImpl implements TagMetaService {
if (CollectionUtils.isNotEmpty(tagDOList)) {
tagDOList.stream().forEach(tagDO -> {
TagResp tagResp = convert(tagDO);
tagResp.setTypeEnum(TypeEnums.TAG);
if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagDO.getId())) {
tagResp.setIsCollect(true);
} else {

72
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
View File

@@ -15,6 +15,7 @@ import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
@@ -25,12 +26,14 @@ import com.tencent.supersonic.headless.api.pojo.ItemValueConfig;
import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -51,11 +54,15 @@ import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@Component
public class DictUtils {
@@ -79,15 +86,18 @@ public class DictUtils {
private final MetricService metricService;
private final QueryService queryService;
private final ModelService modelService;
private final TagMetaService tagMetaService;
public DictUtils(DimensionService dimensionService,
MetricService metricService,
QueryService queryService,
ModelService modelService) {
ModelService modelService,
@Lazy TagMetaService tagMetaService) {
this.dimensionService = dimensionService;
this.metricService = metricService;
this.queryService = queryService;
this.modelService = modelService;
this.tagMetaService = tagMetaService;
}
public String fetchDictFileName(DictItemResp dictItemResp) {
@@ -140,17 +150,22 @@ public class DictUtils {
dictItemResp.setModelId(dimension.getModelId());
dictItemResp.setBizName(dimension.getBizName());
}
if (TypeEnums.TAG.equals(TypeEnums.valueOf(dictConfDO.getType()))) {
TagResp tagResp = tagMetaService.getTag(dictConfDO.getItemId(), User.getFakeUser());
dictItemResp.setModelId(tagResp.getModelId());
dictItemResp.setBizName(tagResp.getBizName());
}
return dictItemResp;
}
public List<String> fetchItemValue(DictItemResp dictItemResp) {
List<String> lines = new ArrayList<>();
QuerySqlReq querySqlReq = constructQueryReq(dictItemResp);
querySqlReq.setNeedAuth(false);
SemanticQueryReq semanticQueryReq = constructQueryReq(dictItemResp);
semanticQueryReq.setNeedAuth(false);
String bizName = dictItemResp.getBizName();
try {
SemanticQueryResp semanticQueryResp = queryService.queryByReq(querySqlReq, null);
SemanticQueryResp semanticQueryResp = queryService.queryByReq(semanticQueryReq, null);
if (Objects.isNull(semanticQueryResp) || CollectionUtils.isEmpty(semanticQueryResp.getResultList())) {
return lines;
}
@@ -173,7 +188,7 @@ public class DictUtils {
mergeMultivaluedValue(valueAndFrequencyPair, dimValue, metric);
}
}
String nature = dictItemResp.getNature();
String nature = dictItemResp.generateNature();
constructDictLines(valueAndFrequencyPair, lines, nature);
addWhiteValueLines(dictItemResp, lines, nature);
} catch (Exception e) {
@@ -227,7 +242,51 @@ public class DictUtils {
}
}
private QuerySqlReq constructQueryReq(DictItemResp dictItemResp) {
private SemanticQueryReq constructQueryReq(DictItemResp dictItemResp) {
if (TypeEnums.DIMENSION.equals(dictItemResp.getType())) {
QuerySqlReq querySqlReq = constructDimQueryReq(dictItemResp);
querySqlReq.setQueryType(QueryType.METRIC);
return querySqlReq;
}
if (TypeEnums.TAG.equals(dictItemResp.getType())) {
QuerySqlReq querySqlReq = constructTagQueryReq(dictItemResp);
querySqlReq.setQueryType(QueryType.TAG);
return querySqlReq;
}
log.warn("constructQueryReq failed");
return null;
}
private QuerySqlReq constructTagQueryReq(DictItemResp dictItemResp) {
String sqlPattern = "select %s, %s from tbl %s group by %s order by %s desc limit %d";
String bizName = dictItemResp.getBizName();
String whereStr = generateWhereStr(dictItemResp);
String where = Strings.isNullOrEmpty(whereStr) ? "" : "WHERE" + whereStr;
ItemValueConfig config = dictItemResp.getConfig();
Long limit = (Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount :
dictItemResp.getConfig().getLimit();
// todo 自定义指标
String metric = "count(1)";
if (Objects.nonNull(dictItemResp.getConfig()) && Objects.nonNull(dictItemResp.getConfig().getMetricId())) {
Long metricId = dictItemResp.getConfig().getMetricId();
MetricResp metricResp = metricService.getMetric(metricId);
String metricBizName = metricResp.getBizName();
metric = String.format("sum(%s)", metricBizName);
}
String sql = String.format(sqlPattern, bizName, metric, where, bizName, metric, limit);
Set<Long> modelIds = new HashSet<>();
modelIds.add(dictItemResp.getModelId());
QuerySqlReq querySqlReq = new QuerySqlReq();
querySqlReq.setSql(sql);
querySqlReq.setNeedAuth(false);
querySqlReq.setModelIds(modelIds);
return querySqlReq;
}
private QuerySqlReq constructDimQueryReq(DictItemResp dictItemResp) {
if (Objects.nonNull(dictItemResp) && Objects.nonNull(dictItemResp.getConfig())
&& Objects.nonNull(dictItemResp.getConfig().getMetricId())) {
// 查询默认指标
@@ -239,7 +298,6 @@ public class DictUtils {
}
private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) {
// todo tag
String sqlPattern = "select %s,count(1) from tbl %s group by %s order by count(1) desc limit %d";
String bizName = dictItemResp.getBizName();

33
launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java
View File

@@ -2,13 +2,19 @@ package com.tencent.supersonic.headless;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.headless.api.pojo.ItemValueConfig;
import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
import com.tencent.supersonic.headless.api.pojo.request.TagReq;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DictConfService;
import com.tencent.supersonic.headless.server.service.DictTaskService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.service.TagQueryService;
import org.junit.Assert;
@@ -28,6 +34,10 @@ public class TagTest extends BaseTest {
private TagMetaService tagMetaService;
@Autowired
private TagQueryService tagQueryService;
@Autowired
private DictConfService dictConfService;
@Autowired
private DictTaskService dictTaskService;
@Test
void testCreateTag() {
@@ -95,4 +105,27 @@ public class TagTest extends BaseTest {
tagMetaService.delete(tag.getId(), User.getFakeUser());
}
@Test
void testTagDict() {
User user = User.getFakeUser();
TagReq tagReq = newTagReq();
TagResp tagResp = tagMetaService.create(tagReq, user);
// add conf
DictItemReq itemValueReq = new DictItemReq();
itemValueReq.setType(TypeEnums.TAG);
itemValueReq.setItemId(tagResp.getId());
itemValueReq.setStatus(StatusEnum.ONLINE);
ItemValueConfig config = new ItemValueConfig();
config.setMetricId(4L);
config.setWhiteList(Arrays.asList("p10", "p20"));
config.setBlackList(Arrays.asList("p1", "p2"));
itemValueReq.setConfig(config);
dictConfService.addDictConf(itemValueReq, user);
// run Task
DictSingleTaskReq taskReq = DictSingleTaskReq.builder().type(TypeEnums.TAG).itemId(tagResp.getId()).build();
dictTaskService.addDictTask(taskReq, user);
tagMetaService.delete(tagResp.getId(), user);
}
}