topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2026-04-29 04:14:20 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

(improvement)(headless) add tag logic (#831)

Browse Source
This commit is contained in:
daikon
2024-03-19 19:40:06 +08:00
committed by GitHub
parent af53812d08
commit bd95552854
23 changed files with 448 additions and 761 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java
View File

@@ -33,7 +33,6 @@ import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.enums.DataType;
@@ -41,14 +40,12 @@ import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.DomainReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
import com.tencent.supersonic.headless.api.pojo.request.TagReq;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -405,60 +402,6 @@ public class ModelDemoDataLoader {
metricService.updateMetric(metricReq, user);
}
private void addTags() {
TagReq tagReq = new TagReq();
tagReq.setModelId(4L);
tagReq.setName("活跃区域");
tagReq.setBizName("act_area");
tagReq.setStatus(StatusEnum.ONLINE.getCode());
tagReq.setTypeEnum(TypeEnums.TAG);
tagReq.setTagDefineType(TagDefineType.DIMENSION);
TagDefineParams tagDefineParams = new TagDefineParams();
tagDefineParams.setExpr("act_area");
tagDefineParams.setDependencies(new ArrayList<>(Arrays.asList(4)));
tagReq.setTagDefineParams(tagDefineParams);
tagMetaService.create(tagReq, user);
TagReq tagReq2 = new TagReq();
tagReq2.setModelId(4L);
tagReq2.setName("风格");
tagReq2.setBizName("genre");
tagReq2.setStatus(StatusEnum.ONLINE.getCode());
tagReq2.setTypeEnum(TypeEnums.TAG);
tagReq2.setTagDefineType(TagDefineType.DIMENSION);
TagDefineParams tagDefineParam2s = new TagDefineParams();
tagDefineParam2s.setExpr("genre");
tagDefineParam2s.setDependencies(new ArrayList<>(Arrays.asList(6)));
tagReq2.setTagDefineParams(tagDefineParam2s);
tagMetaService.create(tagReq2, user);
TagReq tagReq3 = new TagReq();
tagReq3.setModelId(4L);
tagReq3.setName("播放量");
tagReq3.setBizName("js_play_cnt");
tagReq3.setStatus(StatusEnum.ONLINE.getCode());
tagReq3.setTypeEnum(TypeEnums.TAG);
tagReq3.setTagDefineType(TagDefineType.METRIC);
TagDefineParams tagDefineParam3s = new TagDefineParams();
tagDefineParam3s.setExpr("js_play_cnt");
tagDefineParam3s.setDependencies(new ArrayList<>(Arrays.asList(5)));
tagReq3.setTagDefineParams(tagDefineParam3s);
tagMetaService.create(tagReq3, user);
TagReq tagReq4 = new TagReq();
tagReq4.setModelId(4L);
tagReq4.setName("歌手名");
tagReq4.setBizName("singer_name");
tagReq4.setStatus(StatusEnum.ONLINE.getCode());
tagReq4.setTypeEnum(TypeEnums.TAG);
tagReq4.setTagDefineType(TagDefineType.DIMENSION);
TagDefineParams tagDefineParam4s = new TagDefineParams();
tagDefineParam4s.setExpr("singer_name");
tagDefineParam4s.setDependencies(new ArrayList<>(Arrays.asList(7)));
tagReq4.setTagDefineParams(tagDefineParam4s);
tagMetaService.create(tagReq4, user);
}
public void addMetric_uv() throws Exception {
MetricReq metricReq = new MetricReq();
metricReq.setModelId(2L);

36
launchers/standalone/src/main/resources/config.update/sql-update.sql
View File

@@ -194,26 +194,6 @@ alter table s2_view_info rename to s2_canvas;
alter table s2_query_stat_info add column `view_id` bigint(20) DEFAULT NULL after `model_id`;
--20240221
CREATE TABLE s2_tag(
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL ,
`sensitive_level` INT NOT NULL ,
`type` varchar(50) NOT NULL , -- ATOMIC, DERIVED
`define_type` varchar(50) NOT NULL, -- FIELD, DIMENSION
`type_params` LONGVARCHAR DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
`ext` LONGVARCHAR DEFAULT NULL ,
PRIMARY KEY (`id`)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--20240301
CREATE TABLE IF NOT EXISTS `s2_dictionary_conf` (
`id` INT NOT NULL AUTO_INCREMENT,
@@ -270,4 +250,18 @@ CREATE TABLE IF NOT EXISTS `s2_tag_object`
`ext` text DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8 COMMENT ='标签表对象';
DEFAULT CHARSET = utf8 COMMENT ='标签表对象';
alter table s2_model add column `tag_object_id` bigint(20) DEFAULT NULL after domain_id;
CREATE TABLE IF NOT EXISTS s2_tag(
`id` INT NOT NULL AUTO_INCREMENT,
`item_id` INT NOT NULL ,
`type` varchar(255) NOT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
`ext` LONGVARCHAR DEFAULT NULL ,
PRIMARY KEY (`id`)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

10
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -578,20 +578,12 @@ CREATE TABLE IF NOT EXISTS `s2_data_set` (
CREATE TABLE IF NOT EXISTS `s2_tag` (
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL ,
`sensitive_level` INT NOT NULL ,
`item_id` INT NOT NULL ,
`type` varchar(50) NOT NULL , -- ATOMIC, DERIVED
`define_type` varchar(50) NOT NULL, -- FIELD, DIMENSION
`type_params` LONGVARCHAR DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
`ext` LONGVARCHAR DEFAULT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_tag IS 'tag information';

16
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -498,26 +498,18 @@ CREATE TABLE s2_view
`admin_org` varchar(3000) DEFAULT NULL
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `s2_tag`
CREATE TABLE IF NOT EXISTS `s2_tag`
(
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`model_id` bigint(20) DEFAULT NULL,
`name` varchar(255) NOT NULL COMMENT '名称',
`biz_name` varchar(255) NOT NULL COMMENT '英文名称',
`description` varchar(500) DEFAULT NULL COMMENT '描述',
`status` int(10) NOT NULL COMMENT '状态',
`sensitive_level` int(10) NOT NULL COMMENT '敏感级别',
`type` varchar(50) NOT NULL COMMENT '类型(DERIVED,ATOMIC)',
`define_type` varchar(50) DEFAULT NULL, -- FIELD, DIMENSION
`type_params` text NOT NULL COMMENT '类型参数',
`item_id` bigint(20) DEFAULT NULL,
`type` varchar(255) NOT NULL ,
`created_at` datetime NOT NULL COMMENT '创建时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
`updated_at` datetime NULL COMMENT '更新时间',
`updated_by` varchar(100) NULL COMMENT '更新人',
`ext` text DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8 COMMENT ='标签表';
) ENGINE = InnoDB DEFAULT CHARSET = utf8 COMMENT ='标签表';
CREATE TABLE IF NOT EXISTS `s2_tag_object`
(

132
launchers/standalone/src/test/java/com/tencent/supersonic/headless/TagTest.java
View File

@@ -1,132 +0,0 @@
package com.tencent.supersonic.headless;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.headless.api.pojo.ItemValueConfig;
import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemValueReq;
import com.tencent.supersonic.headless.api.pojo.request.TagReq;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DictConfService;
import com.tencent.supersonic.headless.server.service.DictTaskService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.service.TagQueryService;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.ArrayList;
import java.util.Arrays;
public class TagTest extends BaseTest {
private final String bizName = "page";
private final Long modelId = 3L;
private final Integer dimId = 3;
@Autowired
private TagMetaService tagMetaService;
@Autowired
private TagQueryService tagQueryService;
@Autowired
private DictConfService dictConfService;
@Autowired
private DictTaskService dictTaskService;
@Test
void testCreateTag() {
TagReq tagReq = newTagReq();
tagMetaService.create(tagReq, User.getFakeUser());
TagResp tag = queryTagRespByBizName(bizName);
Assert.assertEquals(bizName, tag.getBizName());
tagMetaService.delete(tag.getId(), User.getFakeUser());
}
@Test
void testUpdateTag() {
TagReq tagReq = newTagReq();
TagResp tagResp = tagMetaService.create(tagReq, User.getFakeUser());
Assert.assertEquals(bizName, tagReq.getBizName());
tagReq.setId(tagResp.getId());
tagReq.setName("新页面");
tagMetaService.update(tagReq, User.getFakeUser());
TagResp tag = queryTagRespByBizName(bizName);
Assert.assertEquals("新页面", tag.getName());
tagMetaService.delete(tag.getId(), User.getFakeUser());
}
private TagResp queryTagRespByBizName(String bizName) {
TagFilter tagFilter = new TagFilter();
tagFilter.setBizName(bizName);
TagResp tagRespDb = tagMetaService.getTags(tagFilter).get(0);
return tagRespDb;
}
private TagReq newTagReq() {
TagReq tagReq = new TagReq();
tagReq.setModelId(modelId);
tagReq.setName("页面");
tagReq.setBizName(bizName);
tagReq.setStatus(1);
tagReq.setTypeEnum(TypeEnums.TAG);
tagReq.setTagDefineType(TagDefineType.DIMENSION);
TagDefineParams tagDefineParams = new TagDefineParams();
tagDefineParams.setExpr(bizName);
tagDefineParams.setDependencies(new ArrayList<>(Arrays.asList(dimId)));
tagReq.setTagDefineParams(tagDefineParams);
return tagReq;
}
@Test
void testQueryTag() {
TagReq tagReq = newTagReq();
tagMetaService.create(tagReq, User.getFakeUser());
TagResp tag = queryTagRespByBizName(bizName);
Assert.assertEquals(bizName, tag.getBizName());
tagMetaService.delete(tag.getId(), User.getFakeUser());
}
@Test
void testTagValue() {
TagReq tagReq = newTagReq();
tagMetaService.create(tagReq, User.getFakeUser());
TagResp tag = queryTagRespByBizName(bizName);
ItemValueReq itemValueReq = new ItemValueReq();
itemValueReq.setItemId(tag.getId());
// ItemValueResp itemValueResp = tagQueryService.queryTagValue(itemValueReq, User.getFakeUser());
tagMetaService.delete(tag.getId(), User.getFakeUser());
}
@Test
void testTagDict() {
User user = User.getFakeUser();
TagReq tagReq = newTagReq();
TagResp tagResp = tagMetaService.create(tagReq, user);
// add conf
DictItemReq itemValueReq = new DictItemReq();
itemValueReq.setType(TypeEnums.TAG);
itemValueReq.setItemId(tagResp.getId());
itemValueReq.setStatus(StatusEnum.ONLINE);
ItemValueConfig config = new ItemValueConfig();
config.setMetricId(4L);
config.setWhiteList(Arrays.asList("p10", "p20"));
config.setBlackList(Arrays.asList("p1", "p2"));
itemValueReq.setConfig(config);
dictConfService.addDictConf(itemValueReq, user);
// run Task
DictSingleTaskReq taskReq = DictSingleTaskReq.builder().type(TypeEnums.TAG).itemId(tagResp.getId()).build();
dictTaskService.addDictTask(taskReq, user);
tagMetaService.delete(tagResp.getId(), user);
}
}

10
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -579,20 +579,12 @@ CREATE TABLE IF NOT EXISTS `s2_data_set` (
CREATE TABLE IF NOT EXISTS `s2_tag` (
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL ,
`sensitive_level` INT NOT NULL ,
`item_id` INT NOT NULL ,
`type` varchar(50) NOT NULL , -- ATOMIC, DERIVED
`define_type` varchar(50) NOT NULL, -- FIELD, DIMENSION
`type_params` LONGVARCHAR DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
`updated_by` varchar(100) DEFAULT NULL ,
`ext` LONGVARCHAR DEFAULT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_tag IS 'tag information';