(improvement)(chat) Update ConfigureDemo and dict txt for test (#438)

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-28 00:31:54 +08:00
committed by GitHub
parent 4222d7e2b5
commit 02b9dc6947
39 changed files with 467 additions and 645 deletions

View File

@@ -10,7 +10,7 @@ import lombok.NoArgsConstructor;
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RelateSchemaElement {
public class RelatedSchemaElement {
private Long dimensionId;

View File

@@ -1,14 +1,15 @@
package com.tencent.supersonic.chat.api.pojo;
import com.google.common.base.Objects;
import java.io.Serializable;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.util.List;
@Data
@Getter
@Builder
@@ -24,7 +25,7 @@ public class SchemaElement implements Serializable {
private SchemaElementType type;
private List<String> alias;
private List<SchemaValueMap> schemaValueMaps;
private List<RelateSchemaElement> relateSchemaElements;
private List<RelatedSchemaElement> relatedSchemaElements;
private String defaultAgg;

View File

@@ -1,51 +0,0 @@
package com.tencent.supersonic.chat.parser.llm.time;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LLMTimeEnhancementParse implements SemanticParser {
@Override
public void parse(QueryContext queryContext, ChatContext chatContext) {
log.info("before queryContext:{},chatContext:{}", queryContext, chatContext);
ChatGptHelper chatGptHelper = ContextUtils.getBean(ChatGptHelper.class);
try {
String inferredTime = chatGptHelper.inferredTime(queryContext.getRequest().getQueryText());
if (!queryContext.getCandidateQueries().isEmpty()) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
DateConf dateInfo = query.getParseInfo().getDateInfo();
JSONObject jsonObject = JSON.parseObject(inferredTime);
if (jsonObject.containsKey("date")) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(jsonObject.getString("date"));
dateInfo.setEndDate(jsonObject.getString("date"));
query.getParseInfo().setDateInfo(dateInfo);
} else if (jsonObject.containsKey("start")) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(jsonObject.getString("start"));
dateInfo.setEndDate(jsonObject.getString("end"));
query.getParseInfo().setDateInfo(dateInfo);
}
}
}
} catch (Exception exception) {
log.error("{} parse error,this reason is:{}", LLMTimeEnhancementParse.class.getSimpleName(),
(Object) exception.getStackTrace());
}
log.info("{} after queryContext:{},chatContext:{}",
LLMTimeEnhancementParse.class.getSimpleName(), queryContext, chatContext);
}
}

View File

@@ -4,7 +4,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
@@ -133,9 +133,9 @@ public class MetricCheckPostProcessor implements PostProcessor {
return false;
}
List<String> relateDimensions = metricElements.stream()
.filter(schemaElement -> !CollectionUtils.isEmpty(schemaElement.getRelateSchemaElements()))
.map(schemaElement -> schemaElement.getRelateSchemaElements().stream()
.map(RelateSchemaElement::getDimensionId).collect(Collectors.toList()))
.filter(schemaElement -> !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements()))
.map(schemaElement -> schemaElement.getRelatedSchemaElements().stream()
.map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList()))
.flatMap(Collection::stream)
.map(id -> convertDimensionIdToName(id, semanticSchema))
.filter(Objects::nonNull)
@@ -158,12 +158,12 @@ public class MetricCheckPostProcessor implements PostProcessor {
if (metric == null) {
return Lists.newArrayList();
}
List<RelateSchemaElement> relateSchemaElements = metric.getRelateSchemaElements();
List<RelatedSchemaElement> relateSchemaElements = metric.getRelatedSchemaElements();
if (CollectionUtils.isEmpty(relateSchemaElements)) {
return Lists.newArrayList();
}
return relateSchemaElements.stream()
.filter(RelateSchemaElement::isNecessary).map(RelateSchemaElement::getDimensionId)
.filter(RelatedSchemaElement::isNecessary).map(RelatedSchemaElement::getDimensionId)
.collect(Collectors.toList());
}

View File

@@ -4,10 +4,8 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
@@ -18,10 +16,8 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -42,47 +38,9 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
@Override
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryCtx) {
candidateElementMatches = filterElementMatches(candidateElementMatches);
return super.match(candidateElementMatches, queryCtx);
}
private List<SchemaElementMatch> filterElementMatches(List<SchemaElementMatch> candidateElementMatches) {
List<SchemaElementMatch> filteredMatches = new ArrayList<>();
if (CollectionUtils.isEmpty(candidateElementMatches)
|| Objects.isNull(candidateElementMatches.get(0).getElement().getModel())) {
return candidateElementMatches;
}
Long modelId = candidateElementMatches.get(0).getElement().getModel();
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigResp chatConfig = configService.fetchConfigByModelId(modelId);
List<Long> blackDimIdList = new ArrayList<>();
List<Long> blackMetricIdList = new ArrayList<>();
if (Objects.nonNull(chatConfig.getChatAggConfig())
&& Objects.nonNull(chatConfig.getChatAggConfig().getVisibility())) {
blackDimIdList = chatConfig.getChatAggConfig().getVisibility().getBlackDimIdList();
blackMetricIdList = chatConfig.getChatAggConfig().getVisibility().getBlackMetricIdList();
}
for (SchemaElementMatch schemaElementMatch : candidateElementMatches) {
SchemaElementType type = schemaElementMatch.getElement().getType();
if (SchemaElementType.DIMENSION.equals(type) || SchemaElementType.VALUE.equals(type)) {
if (!blackDimIdList.contains(schemaElementMatch.getElement().getId())) {
filteredMatches.add(schemaElementMatch);
}
} else if (SchemaElementType.METRIC.equals(type)) {
if (!blackMetricIdList.contains(schemaElementMatch.getElement().getId())) {
filteredMatches.add(schemaElementMatch);
}
} else {
filteredMatches.add(schemaElementMatch);
}
}
return filteredMatches;
}
@Override
public void fillParseInfo(ChatContext chatContext) {
super.fillParseInfo(chatContext);

View File

@@ -3,8 +3,6 @@ package com.tencent.supersonic.chat.query.rule.tag;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
@@ -13,10 +11,8 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -37,47 +33,9 @@ public abstract class TagSemanticQuery extends RuleSemanticQuery {
@Override
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryCtx) {
candidateElementMatches = filterElementMatches(candidateElementMatches);
return super.match(candidateElementMatches, queryCtx);
}
private List<SchemaElementMatch> filterElementMatches(List<SchemaElementMatch> candidateElementMatches) {
List<SchemaElementMatch> filteredMatches = new ArrayList<>();
if (CollectionUtils.isEmpty(candidateElementMatches)
|| Objects.isNull(candidateElementMatches.get(0).getElement().getModel())) {
return candidateElementMatches;
}
Long modelId = candidateElementMatches.get(0).getElement().getModel();
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigResp chatConfig = configService.fetchConfigByModelId(modelId);
List<Long> blackDimIdList = new ArrayList<>();
List<Long> blackMetricIdList = new ArrayList<>();
if (Objects.nonNull(chatConfig.getChatDetailConfig())
&& Objects.nonNull(chatConfig.getChatDetailConfig().getVisibility())) {
blackDimIdList = chatConfig.getChatDetailConfig().getVisibility().getBlackDimIdList();
blackMetricIdList = chatConfig.getChatDetailConfig().getVisibility().getBlackMetricIdList();
}
for (SchemaElementMatch schemaElementMatch : candidateElementMatches) {
SchemaElementType type = schemaElementMatch.getElement().getType();
if (SchemaElementType.DIMENSION.equals(type) || SchemaElementType.VALUE.equals(type)) {
if (!blackDimIdList.contains(schemaElementMatch.getElement().getId())) {
filteredMatches.add(schemaElementMatch);
}
} else if (SchemaElementType.METRIC.equals(type)) {
if (!blackMetricIdList.contains(schemaElementMatch.getElement().getId())) {
filteredMatches.add(schemaElementMatch);
}
} else {
filteredMatches.add(schemaElementMatch);
}
}
return filteredMatches;
}
@Override
public void fillParseInfo(ChatContext chatContext) {
super.fillParseInfo(chatContext);

View File

@@ -2,23 +2,13 @@ package com.tencent.supersonic.chat.service.impl;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.RecommendReq;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.RecommendReq;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResp;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.RecommendService;
import com.tencent.supersonic.chat.service.SemanticService;
@@ -28,6 +18,14 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/***
* Recommend Service impl
*/
@@ -59,11 +57,11 @@ public class RecommendServiceImpl implements RecommendService {
if (recommendReq.getMetricId() != null && !CollectionUtils.isEmpty(metricElements)) {
Optional<SchemaElement> metric = metricElements.stream().filter(schemaElement ->
recommendReq.getMetricId().equals(schemaElement.getId())
&& !CollectionUtils.isEmpty(schemaElement.getRelateSchemaElements()))
&& !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements()))
.findFirst();
if (metric.isPresent()) {
drillDownDimensions = metric.get().getRelateSchemaElements().stream()
.map(RelateSchemaElement::getDimensionId).collect(Collectors.toList());
drillDownDimensions = metric.get().getRelatedSchemaElements().stream()
.map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList());
}
}
final List<Long> drillDownDimensionsFinal = drillDownDimensions;
@@ -112,25 +110,7 @@ public class RecommendServiceImpl implements RecommendService {
@Override
public RecommendResp recommendMetricMode(RecommendReq recommendReq, Long limit) {
RecommendResp recommendResponse = recommend(recommendReq, limit);
// filter black Item
if (Objects.isNull(recommendResponse)) {
return recommendResponse;
}
ChatConfigRichResp chatConfigRich = configService.getConfigRichInfo(recommendReq.getModelId());
if (Objects.nonNull(chatConfigRich) && Objects.nonNull(chatConfigRich.getChatAggRichConfig())
&& Objects.nonNull(chatConfigRich.getChatAggRichConfig().getVisibility())) {
List<Long> blackMetricIdList = chatConfigRich.getChatAggRichConfig().getVisibility().getBlackMetricIdList();
List<SchemaElement> metrics = filterBlackItem(recommendResponse.getMetrics(), blackMetricIdList);
recommendResponse.setMetrics(metrics);
List<Long> blackDimIdList = chatConfigRich.getChatAggRichConfig().getVisibility().getBlackDimIdList();
List<SchemaElement> dimensions = filterBlackItem(recommendResponse.getDimensions(), blackDimIdList);
recommendResponse.setDimensions(dimensions);
}
return recommendResponse;
return recommend(recommendReq, limit);
}
@Override

View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.postprocessor;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
@@ -118,10 +118,10 @@ class MetricCheckPostProcessorTest {
ModelSchema modelSchema = new ModelSchema();
Set<SchemaElement> metrics = Sets.newHashSet(
mockElement(1L, "访问次数", SchemaElementType.METRIC,
Lists.newArrayList(RelateSchemaElement.builder().dimensionId(2L).isNecessary(false).build(),
RelateSchemaElement.builder().dimensionId(1L).isNecessary(false).build())),
Lists.newArrayList(RelatedSchemaElement.builder().dimensionId(2L).isNecessary(false).build(),
RelatedSchemaElement.builder().dimensionId(1L).isNecessary(false).build())),
mockElement(2L, "访问用户数", SchemaElementType.METRIC,
Lists.newArrayList(RelateSchemaElement.builder().dimensionId(2L).isNecessary(true).build()))
Lists.newArrayList(RelatedSchemaElement.builder().dimensionId(2L).isNecessary(true).build()))
);
modelSchema.setMetrics(metrics);
modelSchema.setDimensions(mockDimensions());
@@ -148,9 +148,9 @@ class MetricCheckPostProcessorTest {
}
private SchemaElement mockElement(Long id, String name, SchemaElementType type,
List<RelateSchemaElement> relateSchemaElements) {
List<RelatedSchemaElement> relateSchemaElements) {
return SchemaElement.builder().id(id).name(name).type(type)
.relateSchemaElements(relateSchemaElements).build();
.relatedSchemaElements(relateSchemaElements).build();
}
private SemanticParseInfo mockParseInfo(String correctSql) {

View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.knowledge.semantic;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
@@ -50,7 +50,7 @@ public class ModelSchemaBuilder {
.type(SchemaElementType.METRIC)
.useCnt(metric.getUseCnt())
.alias(alias)
.relateSchemaElements(getRelateSchemaElement(metric))
.relatedSchemaElements(getRelateSchemaElement(metric))
.defaultAgg(metric.getDefaultAgg())
.build();
metrics.add(metricToAdd);
@@ -138,13 +138,13 @@ public class ModelSchemaBuilder {
return modelSchema;
}
private static List<RelateSchemaElement> getRelateSchemaElement(MetricSchemaResp metricSchemaResp) {
private static List<RelatedSchemaElement> getRelateSchemaElement(MetricSchemaResp metricSchemaResp) {
RelateDimension relateDimension = metricSchemaResp.getRelateDimension();
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
return Lists.newArrayList();
}
return relateDimension.getDrillDownDimensions().stream().map(dimension -> {
RelateSchemaElement relateSchemaElement = new RelateSchemaElement();
RelatedSchemaElement relateSchemaElement = new RelatedSchemaElement();
BeanUtils.copyProperties(dimension, relateSchemaElement);
return relateSchemaElement;
}).collect(Collectors.toList());

View File

@@ -110,6 +110,45 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
}
public void addDemoChatConfig_1() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(1L);
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
List<Long> dimensionIds0 = Collections.singletonList(1L);
List<Long> metricIds0 = Lists.newArrayList();
chatDefaultConfigDetail.setDimensionIds(dimensionIds0);
chatDefaultConfigDetail.setMetricIds(metricIds0);
chatDefaultConfigDetail.setUnit(7);
chatDefaultConfigDetail.setPeriod("DAY");
chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
ItemVisibility visibility0 = new ItemVisibility();
chatDetailConfig.setVisibility(visibility0);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq();
List<Long> dimensionIds1 = Arrays.asList(1L);
List<Long> metricIds1 = Lists.newArrayList();
chatDefaultConfigAgg.setDimensionIds(dimensionIds1);
chatDefaultConfigAgg.setMetricIds(metricIds1);
chatDefaultConfigAgg.setUnit(7);
chatDefaultConfigAgg.setPeriod("DAY");
chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT);
chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg);
ItemVisibility visibility1 = new ItemVisibility();
chatAggConfig.setVisibility(visibility1);
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setItemId(3L);
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfos.add(knowledgeInfoReq);
chatAggConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatAggConfig(chatAggConfig);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_2() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(2L);
@@ -148,7 +187,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_2() {
public void addDemoChatConfig_3() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(3L);
@@ -187,7 +226,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_3() {
public void addDemoChatConfig_4() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(4L);
@@ -335,6 +374,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
addDemoChatConfig_1();
addDemoChatConfig_2();
addDemoChatConfig_3();
addDemoChatConfig_4();
addPlugin_1();
addAgent1();
addAgent2();

View File

@@ -321,7 +321,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
public void updateMetric() throws Exception {
MetricReq metricReq = new MetricReq();
metricReq.setModelId(1L);
metricReq.setModelId(3L);
metricReq.setId(3L);
metricReq.setName("停留时长");
metricReq.setBizName("stay_hours");

View File

@@ -244,7 +244,7 @@ CREATE TABLE s2_model_rela
create table s2_view_info
(
id INT auto_increment,
model_id INT null,
domain_id INT null,
type varchar(20) null comment 'datasource、dimension、metric',
config LONGVARCHAR null comment 'config detail',
created_at TIMESTAMP null,
@@ -532,3 +532,12 @@ CREATE TABLE s2_sys_parameter
parameters text null
);
CREATE TABLE `s2_collect` (
`id` bigint NOT NULL AUTO_INCREMENT,
`type` varchar(20) NOT NULL,
`username` varchar(20) NOT NULL,
`collect_id` bigint NOT NULL,
`create_time` TIMESTAMP,
`update_time` TIMESTAMP,
PRIMARY KEY (`id`)
);

View File

@@ -363,7 +363,7 @@ CREATE TABLE `s2_semantic_pasre_info` (
CREATE TABLE `s2_view_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`model_id` bigint(20) DEFAULT NULL,
`domain_id` bigint(20) DEFAULT NULL,
`type` varchar(20) DEFAULT NULL COMMENT 'datasource、dimension、metric',
`config` text COMMENT 'config detail',
`created_at` datetime DEFAULT NULL,

View File

@@ -113,4 +113,6 @@ CREATE TABLE s2_model_rela
join_type VARCHAR(255),
join_condition VARCHAR(255),
PRIMARY KEY (`id`)
);
);
alter table s2_view_info change model_id domain_id bigint;

View File

@@ -1,21 +1,20 @@
package com.tencent.supersonic.integration;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.rule.tag.TagFilterQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricTagQuery;
import com.tencent.supersonic.chat.query.rule.tag.TagFilterQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
public class TagQueryTest extends BaseQueryTest {
@@ -30,13 +29,13 @@ public class TagQueryTest extends BaseQueryTest {
expectedResult.setQueryMode(MetricTagQuery.QUERY_MODE);
expectedParseInfo.setAggType(NONE);
QueryFilter dimensionFilter = DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 7L);
QueryFilter dimensionFilter = DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 9L);
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
SchemaElement metric = SchemaElement.builder().name("播放量").build();
expectedParseInfo.getMetrics().add(metric);
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateMode.BETWEEN, 1, period, startDay, endDay));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateMode.RECENT, 7, period, startDay, endDay));
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
@@ -54,9 +53,9 @@ public class TagQueryTest extends BaseQueryTest {
expectedParseInfo.setAggType(NONE);
List<String> list = new ArrayList<>();
list.add("爱情");
list.add("流行");
QueryFilter dimensionFilter = DataUtils.getFilter("genre", FilterOperatorEnum.IN, list, "风格", 6L);
QueryFilter dimensionFilter = DataUtils.getFilter("genre", FilterOperatorEnum.EQUALS,
"流行", "风格", 8L);
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
SchemaElement metric = SchemaElement.builder().name("播放量").build();

View File

@@ -5,15 +5,14 @@ dean _1_2 36
john _1_2 50
jack _1_2 38
admin _1_2 70
周杰伦 _2_7 100
陈奕迅 _2_7 100
林俊杰 _2_7 100
张碧晨 _2_7 100
程响 _2_7 100
Taylor#Swift _2_7 100
中国 _2_4 100
欧美 _2_4 100
流行 _2_6 100
爱情 _2_6 100
激情 _2_6 100
国风 _2_6 100
周杰伦 _4_9 100
陈奕迅 _4_9 100
林俊杰 _4_9 100
张碧晨 _4_9 100
程响 _4_9 100
Taylor#Swift _4_9 100
内地 _4_6 100
欧美 _4_6 100
港台 _4_6 100
流行 _4_8 100
国风 _4_8 100

View File

@@ -1,6 +1,6 @@
p1 _1_3 52
p2 _1_3 47
p3 _1_3 31
p4 _1_3 36
p5 _1_3 50
p6 _1_3 38
p1 _3_4 52
p2 _3_4 47
p3 _3_4 31
p4 _3_4 36
p5 _3_4 50
p6 _3_4 38

View File

@@ -0,0 +1,9 @@
周杰伦 _4_9 9000
周深 _4_9 8000
周传雄 _4_9 7000
周华建 _4_9 6000
陈奕迅 _4_9 8000
林俊杰 _4_9 7000
张碧晨 _4_9 7000
程响 _4_9 7000
Taylor#Swift _4_9 7000

View File

@@ -17,4 +17,5 @@ public class PageSchemaItemReq extends PageBaseReq {
private Integer sensitiveLevel;
private Integer status;
private String key;
private List<Long> ids;
}

View File

@@ -46,6 +46,8 @@ public class MetricResp extends SchemaItem {
private boolean hasAdminRes = false;
private Boolean isCollect;
public void setTag(String tag) {
if (StringUtils.isBlank(tag)) {
tags = Lists.newArrayList();

View File

@@ -0,0 +1,57 @@
package com.tencent.supersonic.semantic.model.application;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.model.domain.CollectService;
import com.tencent.supersonic.semantic.model.domain.dataobject.CollectDO;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.CollectMapper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
@Slf4j
@Service
public class CollectServiceImpl implements CollectService {
@Resource
private CollectMapper collectMapper;
String type = "metric";
@Override
public Boolean createCollectionIndicators(User user, Long id) {
CollectDO collectDO = new CollectDO();
collectDO.setType(type);
collectDO.setUsername(user.getName());
collectDO.setCollectId(id);
collectMapper.insert(collectDO);
return true;
}
@Override
public Boolean deleteCollectionIndicators(User user, Long id) {
QueryWrapper<CollectDO> collectDOQueryWrapper = new QueryWrapper<>();
collectDOQueryWrapper.lambda().eq(CollectDO::getUsername,user.getName());
collectDOQueryWrapper.lambda().eq(CollectDO::getCollectId,id);
collectDOQueryWrapper.lambda().eq(CollectDO::getType,type);
collectMapper.delete(collectDOQueryWrapper);
return true;
}
@Override
public List<CollectDO> getCollectList(String username) {
QueryWrapper<CollectDO> queryWrapper = new QueryWrapper<>();
if (!StringUtils.isEmpty(username)){
queryWrapper.lambda().eq(CollectDO::getUsername,username);
}
return collectMapper.selectList(queryWrapper);
}
}

View File

@@ -9,6 +9,7 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
@@ -25,6 +26,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.model.domain.DatabaseService;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelRelaService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO;
import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
@@ -46,7 +48,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@Slf4j
public class DimensionServiceImpl implements DimensionService {
@@ -60,6 +61,8 @@ public class DimensionServiceImpl implements DimensionService {
private DatabaseService databaseService;
private ModelRelaService modelRelaService;
@Autowired
private ApplicationEventPublisher eventPublisher;
@@ -67,11 +70,13 @@ public class DimensionServiceImpl implements DimensionService {
public DimensionServiceImpl(DimensionRepository dimensionRepository,
ModelService modelService,
ChatGptHelper chatGptHelper,
DatabaseService databaseService) {
DatabaseService databaseService,
ModelRelaService modelRelaService) {
this.modelService = modelService;
this.dimensionRepository = dimensionRepository;
this.chatGptHelper = chatGptHelper;
this.databaseService = databaseService;
this.modelRelaService = modelRelaService;
}
@Override
@@ -213,6 +218,21 @@ public class DimensionServiceImpl implements DimensionService {
return getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
}
@Override
public List<DimensionResp> getDimensionInModelCluster(Long modelId) {
ModelResp modelResp = modelService.getModel(modelId);
List<ModelRela> modelRelas = modelRelaService.getModelRelaList(modelResp.getDomainId());
List<Long> modelIds = new ArrayList<>();
modelIds.add(modelId);
for (ModelRela modelRela : modelRelas) {
modelIds.add(modelRela.getFromModelId());
modelIds.add(modelRela.getToModelId());
}
DimensionFilter dimensionFilter = new DimensionFilter();
dimensionFilter.setModelIds(modelIds);
return getDimensions(dimensionFilter);
}
private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS,
Map<Long, ModelResp> modelRespMap) {
List<DimensionResp> dimensionResps = Lists.newArrayList();
@@ -287,10 +307,17 @@ public class DimensionServiceImpl implements DimensionService {
return dimValueMapsResp;
}
private List<DimensionResp> getDimensionInSameDomain(Long modelId) {
ModelResp modelResp = modelService.getModel(modelId);
Long domainId = modelResp.getDomainId();
List<ModelResp> modelResps = modelService.getModelByDomainIds(Lists.newArrayList(domainId));
List<Long> modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList());
return getDimensions(new MetaFilter(modelIds));
}
private void checkExist(List<DimensionReq> dimensionReqs) {
Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensions(modelId);
List<DimensionResp> dimensionResps = getDimensionInSameDomain(modelId);
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> nameMap = dimensionResps.stream()
@@ -316,6 +343,14 @@ public class DimensionServiceImpl implements DimensionService {
}
}
@Override
public void sendDimensionEventBatch(List<Long> modelIds, EventType eventType) {
DimensionFilter dimensionFilter = new DimensionFilter();
dimensionFilter.setModelIds(modelIds);
List<DimensionDO> dimensionDOS = queryDimension(dimensionFilter);
sendEventBatch(dimensionDOS, eventType);
}
private void sendEventBatch(List<DimensionDO> dimensionDOS, EventType eventType) {
List<DataItem> dataItems = dimensionDOS.stream().map(dimensionDO ->
DataItem.builder().id(dimensionDO.getId()).name(dimensionDO.getName())

View File

@@ -23,9 +23,11 @@ import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.model.domain.CollectService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.dataobject.CollectDO;
import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
@@ -58,18 +60,22 @@ public class MetricServiceImpl implements MetricService {
private ChatGptHelper chatGptHelper;
private CollectService collectService;
private ApplicationEventPublisher eventPublisher;
public MetricServiceImpl(MetricRepository metricRepository,
ModelService modelService,
DomainService domainService,
ChatGptHelper chatGptHelper,
CollectService collectService,
ApplicationEventPublisher eventPublisher) {
this.domainService = domainService;
this.metricRepository = metricRepository;
this.modelService = modelService;
this.chatGptHelper = chatGptHelper;
this.eventPublisher = eventPublisher;
this.collectService = collectService;
}
@Override
@@ -177,7 +183,9 @@ public class MetricServiceImpl implements MetricService {
.doSelectPageInfo(() -> queryMetric(metricFilter));
PageInfo<MetricResp> pageInfo = new PageInfo<>();
BeanUtils.copyProperties(metricDOPageInfo, pageInfo);
List<MetricResp> metricResps = convertList(metricDOPageInfo.getList());
List<CollectDO> collectList = collectService.getCollectList(user.getName());
List<Long> collect = collectList.stream().map(CollectDO::getCollectId).collect(Collectors.toList());
List<MetricResp> metricResps = convertList(metricDOPageInfo.getList(),collect);
fillAdminRes(metricResps, user);
pageInfo.setList(metricResps);
return pageInfo;
@@ -191,7 +199,7 @@ public class MetricServiceImpl implements MetricService {
public List<MetricResp> getMetrics(MetaFilter metaFilter) {
MetricFilter metricFilter = new MetricFilter();
BeanUtils.copyProperties(metaFilter, metricFilter);
return convertList(queryMetric(metricFilter));
return convertList(queryMetric(metricFilter), Lists.newArrayList());
}
private void fillAdminRes(List<MetricResp> metricResps, User user) {
@@ -236,7 +244,7 @@ public class MetricServiceImpl implements MetricService {
if (metricDO == null) {
return null;
}
return MetricConverter.convert2MetricResp(metricDO, new HashMap<>());
return MetricConverter.convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
}
@Override
@@ -333,17 +341,25 @@ public class MetricServiceImpl implements MetricService {
return getMetrics(new MetaFilter(modelIds));
}
private List<MetricResp> convertList(List<MetricDO> metricDOS) {
private List<MetricResp> convertList(List<MetricDO> metricDOS, List<Long> collect) {
List<MetricResp> metricResps = Lists.newArrayList();
Map<Long, ModelResp> modelMap = modelService.getModelMap();
if (!CollectionUtils.isEmpty(metricDOS)) {
metricResps = metricDOS.stream()
.map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap))
.map(metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect))
.collect(Collectors.toList());
}
return metricResps;
}
@Override
public void sendMetricEventBatch(List<Long> modelIds, EventType eventType) {
MetricFilter metricFilter = new MetricFilter();
metricFilter.setModelIds(modelIds);
List<MetricDO> metricDOS = queryMetric(metricFilter);
sendEventBatch(metricDOS, eventType);
}
private void sendEventBatch(List<MetricDO> metricDOS, EventType eventType) {
List<DataItem> dataItems = metricDOS.stream().map(this::getDataItem)
.collect(Collectors.toList());

View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
@@ -130,13 +131,32 @@ public class ModelServiceImpl implements ModelService {
public ModelResp updateModel(ModelReq modelReq, User user) throws Exception {
checkName(modelReq);
ModelDO modelDO = modelRepository.getModelById(modelReq.getId());
int oldStatus = modelDO.getStatus();
ModelConverter.convert(modelDO, modelReq, user);
modelRepository.updateModel(modelDO);
batchCreateDimension(modelDO, user);
batchCreateMetric(modelDO, user);
statusPublish(oldStatus, modelDO);
return ModelConverter.convert(modelDO);
}
private void statusPublish(Integer oldStatus, ModelDO modelDO) {
if (oldStatus.equals(modelDO.getStatus())) {
return;
}
EventType eventType = null;
if (oldStatus.equals(StatusEnum.ONLINE.getCode())
&& modelDO.getStatus().equals(StatusEnum.OFFLINE.getCode())) {
eventType = EventType.DELETE;
} else if (oldStatus.equals(StatusEnum.OFFLINE.getCode())
&& modelDO.getStatus().equals(StatusEnum.ONLINE.getCode())) {
eventType = EventType.ADD;
}
log.info("model:{} status from {} to {}", modelDO.getId(), oldStatus, modelDO.getStatus());
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()), eventType);
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()), eventType);
}
@Override
public List<ModelResp> getModelList(ModelFilter modelFilter) {
return ModelConverter.convertList(modelRepository.getModelList(modelFilter));

View File

@@ -40,8 +40,8 @@ public class ViewInfoServiceImpl {
this.modelService = modelService;
}
public List<ViewInfoDO> getViewInfoList(Long modelId) {
return viewInfoRepository.getViewInfoList(modelId);
public List<ViewInfoDO> getViewInfoList(Long domainId) {
return viewInfoRepository.getViewInfoList(domainId);
}
public List<ModelSchemaRelaResp> getDomainSchema(Long domainId) {

View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.semantic.model.domain;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.model.domain.dataobject.CollectDO;
import java.util.List;
/**
* @author yannsu
*/
public interface CollectService {
Boolean createCollectionIndicators(User user, Long id);
Boolean deleteCollectionIndicators(User user,Long id);
List<CollectDO> getCollectList(String username);
}

View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.domain;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
@@ -32,9 +33,13 @@ public interface DimensionService {
void deleteDimension(Long id, User user);
List<DimensionResp> getDimensionInModelCluster(Long modelId);
List<DataItem> getDataItems(Long modelId);
List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user);
List<DimValueMap> mockDimensionValueAlias(DimensionReq dimensionReq, User user);
void sendDimensionEventBatch(List<Long> modelIds, EventType eventType);
}

View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.domain;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.MetricReq;
@@ -39,4 +40,6 @@ public interface MetricService {
List<DrillDownDimension> getDrillDownDimension(Long metricId);
List<DataItem> getDataItems(Long modelId);
void sendMetricEventBatch(List<Long> modelIds, EventType eventType);
}

View File

@@ -0,0 +1,64 @@
package com.tencent.supersonic.semantic.model.domain.dataobject;
import com.baomidou.mybatisplus.annotation.FieldFill;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Getter;
import lombok.Setter;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* <p>
* 收藏项表
* </p>
*
* @author yannsu
* @since 2023-11-09 03:49:33
*/
@Getter
@Setter
@TableName("s2_collect")
public class CollectDO implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 收藏项ID
*/
@TableId(value = "id", type = IdType.AUTO)
private Long id;
/**
* 收藏的类型
*/
@TableField("type")
private String type;
/**
* 收藏人
*/
@TableField("username")
private String username;
/**
* 收藏ID
*/
@TableField("collect_id")
private Long collectId;
/**
* 创建时间
*/
@TableField(value = "create_time", fill = FieldFill.INSERT)
private LocalDateTime createTime;
/**
* 更新时间
*/
@TableField(value = "update_time", fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@@ -1,15 +1,22 @@
package com.tencent.supersonic.semantic.model.domain.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_view_info")
public class ViewInfoDO {
/**
*/
@TableId(type = IdType.AUTO)
private Long id;
/**
*/
private Long modelId;
private Long domainId;
/**
* datasource、dimension、metric
@@ -36,120 +43,4 @@ public class ViewInfoDO {
* config detail
*/
private String config;
/**
* @return id
*/
public Long getId() {
return id;
}
/**
* @param id
*/
public void setId(Long id) {
this.id = id;
}
/**
* @return model_id
*/
public Long getModelId() {
return modelId;
}
/**
* @param modelId
*/
public void setModelId(Long modelId) {
this.modelId = modelId;
}
/**
* datasource、dimension、metric
* @return type datasource、dimension、metric
*/
public String getType() {
return type;
}
/**
* datasource、dimension、metric
* @param type datasource、dimension、metric
*/
public void setType(String type) {
this.type = type == null ? null : type.trim();
}
/**
* @return created_at
*/
public Date getCreatedAt() {
return createdAt;
}
/**
* @param createdAt
*/
public void setCreatedAt(Date createdAt) {
this.createdAt = createdAt;
}
/**
* @return created_by
*/
public String getCreatedBy() {
return createdBy;
}
/**
* @param createdBy
*/
public void setCreatedBy(String createdBy) {
this.createdBy = createdBy == null ? null : createdBy.trim();
}
/**
* @return updated_at
*/
public Date getUpdatedAt() {
return updatedAt;
}
/**
* @param updatedAt
*/
public void setUpdatedAt(Date updatedAt) {
this.updatedAt = updatedAt;
}
/**
* @return updated_by
*/
public String getUpdatedBy() {
return updatedBy;
}
/**
* @param updatedBy
*/
public void setUpdatedBy(String updatedBy) {
this.updatedBy = updatedBy == null ? null : updatedBy.trim();
}
/**
* config detail
* @return config config detail
*/
public String getConfig() {
return config;
}
/**
* config detail
* @param config config detail
*/
public void setConfig(String config) {
this.config = config == null ? null : config.trim();
}
}

View File

@@ -56,7 +56,7 @@ public class MetricConverter {
return measureYamlTpl;
}
public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap) {
public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap, List<Long> collect) {
MetricResp metricResp = new MetricResp();
BeanUtils.copyProperties(metricDO, metricResp);
metricResp.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(), MetricTypeParams.class));
@@ -66,6 +66,11 @@ public class MetricConverter {
metricResp.setModelName(modelResp.getName());
metricResp.setDomainId(modelResp.getDomainId());
}
if (collect != null && collect.contains(metricDO.getId())){
metricResp.setIsCollect(true);
}else {
metricResp.setIsCollect(false);
}
metricResp.setTag(metricDO.getTags());
metricResp.setRelateDimension(JSONObject.parseObject(metricDO.getRelateDimensions(),
RelateDimension.class));

View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.semantic.model.infrastructure.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.semantic.model.domain.dataobject.CollectDO;
import org.apache.ibatis.annotations.Mapper;
/**
* <p>
* 收藏项表 Mapper 接口
* </p>
*
* @author yannsu
* @since 2023-11-09 03:49:33
*/
@Mapper
public interface CollectMapper extends BaseMapper<CollectDO> {
}

View File

@@ -1,70 +1,10 @@
package com.tencent.supersonic.semantic.model.infrastructure.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO;
import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDOExample;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface ViewInfoDOMapper {
/**
*
* @mbg.generated
*/
long countByExample(ViewInfoDOExample example);
public interface ViewInfoDOMapper extends BaseMapper<ViewInfoDO> {
/**
*
* @mbg.generated
*/
int deleteByPrimaryKey(Long id);
/**
*
* @mbg.generated
*/
int insert(ViewInfoDO record);
/**
*
* @mbg.generated
*/
int insertSelective(ViewInfoDO record);
/**
*
* @mbg.generated
*/
List<ViewInfoDO> selectByExampleWithBLOBs(ViewInfoDOExample example);
/**
*
* @mbg.generated
*/
List<ViewInfoDO> selectByExample(ViewInfoDOExample example);
/**
*
* @mbg.generated
*/
ViewInfoDO selectByPrimaryKey(Long id);
/**
*
* @mbg.generated
*/
int updateByPrimaryKeySelective(ViewInfoDO record);
/**
*
* @mbg.generated
*/
int updateByPrimaryKeyWithBLOBs(ViewInfoDO record);
/**
*
* @mbg.generated
*/
int updateByPrimaryKey(ViewInfoDO record);
}

View File

@@ -1,13 +1,11 @@
package com.tencent.supersonic.semantic.model.infrastructure.repository;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO;
import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDOExample;
import com.tencent.supersonic.semantic.model.domain.repository.ViewInfoRepository;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.ViewInfoDOMapper;
import java.util.List;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
public class ViewInfoRepositoryImpl implements ViewInfoRepository {
@@ -21,21 +19,21 @@ public class ViewInfoRepositoryImpl implements ViewInfoRepository {
@Override
public List<ViewInfoDO> getViewInfoList(Long domainId) {
ViewInfoDOExample viewInfoDOExample = new ViewInfoDOExample();
viewInfoDOExample.createCriteria().andModelIdEqualTo(domainId);
return viewInfoDOMapper.selectByExampleWithBLOBs(viewInfoDOExample);
QueryWrapper<ViewInfoDO> wrapper = new QueryWrapper<>();
wrapper.lambda().eq(ViewInfoDO::getDomainId, domainId);
return viewInfoDOMapper.selectList(wrapper);
}
@Override
public ViewInfoDO getViewInfoById(Long id) {
return viewInfoDOMapper.selectByPrimaryKey(id);
return viewInfoDOMapper.selectById(id);
}
@Override
public void deleteViewInfo(Long id) {
viewInfoDOMapper.deleteByPrimaryKey(id);
viewInfoDOMapper.deleteById(id);
}
@@ -47,7 +45,7 @@ public class ViewInfoRepositoryImpl implements ViewInfoRepository {
@Override
public void updateViewInfo(ViewInfoDO viewInfoDO) {
viewInfoDOMapper.updateByPrimaryKeyWithBLOBs(viewInfoDO);
viewInfoDOMapper.updateById(viewInfoDO);
}

View File

@@ -0,0 +1,48 @@
package com.tencent.supersonic.semantic.model.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.semantic.model.domain.CollectService;
import com.tencent.supersonic.semantic.model.domain.dataobject.CollectDO;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/***
* 创建收藏指标的逻辑
*/
@RestController
@RequestMapping("/api/semantic/collect")
public class CollectController {
private CollectService collectService;
public CollectController(CollectService collectService) {
this.collectService = collectService;
}
@PostMapping("/createCollectionIndicators")
public boolean createCollectionIndicators(@RequestBody CollectDO collectDO,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return collectService.createCollectionIndicators(user, collectDO.getId());
}
@DeleteMapping("/deleteCollectionIndicators/{id}")
public boolean deleteCollectionIndicators(@PathVariable Long id,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return collectService.deleteCollectionIndicators(user,id);
}
}

View File

@@ -11,11 +11,6 @@ import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import org.springframework.web.bind.annotation.DeleteMapping;
@@ -26,6 +21,10 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
@RestController
@RequestMapping("/api/semantic/dimension")
@@ -95,6 +94,11 @@ public class DimensionController {
return dimensionService.getDimensions(dimensionFilter);
}
@GetMapping("/getDimensionInModelCluster/{modelId}")
public List<DimensionResp> getDimensionInModelCluster(@PathVariable("modelId") Long modelId) {
return dimensionService.getDimensionInModelCluster(modelId);
}
@GetMapping("/{modelId}/{dimensionName}")
public DimensionResp getDimensionDescByNameAndId(@PathVariable("modelId") Long modelId,

View File

@@ -37,9 +37,9 @@ public class ViewInfoController {
return viewInfoServiceImpl.createOrUpdateViewInfo(viewInfoReq, user);
}
@GetMapping("/getViewInfoList/{modelId}")
public List<ViewInfoDO> getViewInfoList(@PathVariable("modelId") Long modelId) {
return viewInfoServiceImpl.getViewInfoList(modelId);
@GetMapping("/getViewInfoList/{domainId}")
public List<ViewInfoDO> getViewInfoList(@PathVariable("domainId") Long domainId) {
return viewInfoServiceImpl.getViewInfoList(domainId);
}
@DeleteMapping("/deleteViewInfo/{id}")

View File

@@ -1,213 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.tencent.supersonic.semantic.model.infrastructure.mapper.ViewInfoDOMapper">
<resultMap id="BaseResultMap" type="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO">
<id column="id" jdbcType="BIGINT" property="id" />
<result column="model_id" jdbcType="BIGINT" property="modelId" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt" />
<result column="created_by" jdbcType="VARCHAR" property="createdBy" />
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO">
<result column="config" jdbcType="LONGVARCHAR" property="config" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, model_id, type, created_at, created_by, updated_at, updated_by
</sql>
<sql id="Blob_Column_List">
config
</sql>
<select id="selectByExampleWithBLOBs" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDOExample" resultMap="ResultMapWithBLOBs">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from s2_view_info
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="selectByExample" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDOExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
from s2_view_info
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
<if test="limitStart != null and limitStart>=0">
limit #{limitStart} , #{limitEnd}
</if>
</select>
<select id="selectByPrimaryKey" parameterType="java.lang.Long" resultMap="ResultMapWithBLOBs">
select
<include refid="Base_Column_List" />
,
<include refid="Blob_Column_List" />
from s2_view_info
where id = #{id,jdbcType=BIGINT}
</select>
<delete id="deleteByPrimaryKey" parameterType="java.lang.Long">
delete from s2_view_info
where id = #{id,jdbcType=BIGINT}
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO">
insert into s2_view_info (id, model_id, type,
created_at, created_by, updated_at,
updated_by, config)
values (#{id,jdbcType=BIGINT}, #{modelId,jdbcType=BIGINT}, #{type,jdbcType=VARCHAR},
#{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP},
#{updatedBy,jdbcType=VARCHAR}, #{config,jdbcType=LONGVARCHAR})
</insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO">
insert into s2_view_info
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="modelId != null">
model_id,
</if>
<if test="type != null">
type,
</if>
<if test="createdAt != null">
created_at,
</if>
<if test="createdBy != null">
created_by,
</if>
<if test="updatedAt != null">
updated_at,
</if>
<if test="updatedBy != null">
updated_by,
</if>
<if test="config != null">
config,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=BIGINT},
</if>
<if test="modelId != null">
#{modelId,jdbcType=BIGINT},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
#{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
#{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
#{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
#{updatedBy,jdbcType=VARCHAR},
</if>
<if test="config != null">
#{config,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDOExample" resultType="java.lang.Long">
select count(*) from s2_view_info
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</select>
<update id="updateByPrimaryKeySelective" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO">
update s2_view_info
<set>
<if test="modelId != null">
model_id = #{modelId,jdbcType=BIGINT},
</if>
<if test="type != null">
type = #{type,jdbcType=VARCHAR},
</if>
<if test="createdAt != null">
created_at = #{createdAt,jdbcType=TIMESTAMP},
</if>
<if test="createdBy != null">
created_by = #{createdBy,jdbcType=VARCHAR},
</if>
<if test="updatedAt != null">
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
</if>
<if test="updatedBy != null">
updated_by = #{updatedBy,jdbcType=VARCHAR},
</if>
<if test="config != null">
config = #{config,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKeyWithBLOBs" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO">
update s2_view_info
set model_id = #{modelId,jdbcType=BIGINT},
type = #{type,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
config = #{config,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO">
update s2_view_info
set model_id = #{modelId,jdbcType=BIGINT},
type = #{type,jdbcType=VARCHAR},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
</mapper>

View File

@@ -42,6 +42,7 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -65,7 +66,7 @@ public class DownloadServiceImpl implements DownloadService {
List<List<String>> data = new ArrayList<>();
List<List<String>> header = org.assertj.core.util.Lists.newArrayList();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
header.add(org.assertj.core.util.Lists.newArrayList(column.getName()));
header.add(Lists.newArrayList(column.getName()));
}
for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
List<String> rowData = new ArrayList<>();
@@ -212,6 +213,8 @@ public class DownloadServiceImpl implements DownloadService {
private QueryResultWithSchemaResp getQueryResult(List<DimSchemaResp> dimensionResps, MetricResp metricResp,
DateConf dateConf, User user) throws Exception {
Set<Long> modelIds = dimensionResps.stream().map(DimSchemaResp::getModelId).collect(Collectors.toSet());
modelIds.add(metricResp.getModelId());
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setGroups(dimensionResps.stream().map(DimSchemaResp::getBizName).collect(Collectors.toList()));
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
@@ -219,7 +222,7 @@ public class DownloadServiceImpl implements DownloadService {
aggregator.setColumn(metricResp.getBizName());
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
queryStructReq.setDateInfo(dateConf);
queryStructReq.setModelId(metricResp.getModelId());
queryStructReq.setModelIds(modelIds);
queryStructReq.setLimit(10000L);
return queryService.queryByStructWithAuth(queryStructReq, user);
}