topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:28:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Feature/model data embedding for chat and support status for metric and dimension (#311)

Browse Source 
* (improvement)(semantic) add offline status for metric and dimension

* (improvement)(chat) add metric recall

---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-02 18:44:58 +08:00
committed by GitHub
parent f4e3922f47
commit ad20380283
89 changed files with 1572 additions and 896 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java
View File

@@ -28,6 +28,8 @@ public class SchemaElement implements Serializable {
private String defaultAgg;
private int order;
@Override
public boolean equals(Object o) {
if (this == o) {

3
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java
View File

@@ -53,6 +53,9 @@ public class SemanticParseInfo {
@Override
public int compare(SchemaElement o1, SchemaElement o2) {
if (o1.getOrder() != o2.getOrder()) {
return o1.getOrder() - o2.getOrder();
}
int len1 = o1.getName().length();
int len2 = o2.getName().length();
if (len1 != len2) {

1
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingBasedParser.java
View File

@@ -7,6 +7,7 @@ import com.tencent.supersonic.chat.parser.plugin.PluginParser;
import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.Comparator;
import java.util.List;

2
chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.agent.Agent;
import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.PluginTool;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval;
import com.tencent.supersonic.chat.plugin.event.PluginAddEvent;

76
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SimilarMetricParseResponder.java Normal file
View File

@@ -0,0 +1,76 @@
package com.tencent.supersonic.chat.responder.parse;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.embedding.EmbeddingUtils;
import com.tencent.supersonic.common.util.embedding.Retrieval;
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
import com.tencent.supersonic.semantic.model.domain.listener.MetaEmbeddingListener;
import org.springframework.util.CollectionUtils;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
public class SimilarMetricParseResponder implements ParseResponder {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext, List<ChatParseDO> chatParseDOS) {
if (CollectionUtils.isEmpty(parseResp.getSelectedParses())) {
return;
}
fillSimilarMetric(parseResp.getSelectedParses().iterator().next());
}
private void fillSimilarMetric(SemanticParseInfo parseInfo) {
if (!QueryManager.isMetricQuery(parseInfo.getQueryMode())
|| CollectionUtils.isEmpty(parseInfo.getMetrics())) {
return;
}
List<String> metricNames = parseInfo.getMetrics().stream()
.map(SchemaElement::getName).collect(Collectors.toList());
Map<String, String> filterCondition = new HashMap<>();
filterCondition.put("modelId", parseInfo.getModelId().toString());
filterCondition.put("type", SchemaElementType.METRIC.name());
RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(metricNames)
.filterCondition(filterCondition).queryEmbeddings(null).build();
EmbeddingUtils embeddingUtils = ContextUtils.getBean(EmbeddingUtils.class);
List<RetrieveQueryResult> retrieveQueryResults = embeddingUtils.retrieveQuery(
MetaEmbeddingListener.COLLECTION_NAME, retrieveQuery, 10);
if (CollectionUtils.isEmpty(retrieveQueryResults)) {
return;
}
List<Retrieval> retrievals = retrieveQueryResults.stream()
.flatMap(retrieveQueryResult -> retrieveQueryResult.getRetrieval().stream())
.sorted(Comparator.comparingDouble(Retrieval::getDistance).reversed())
.distinct().collect(Collectors.toList());
Set<Long> metricIds = parseInfo.getMetrics().stream().map(SchemaElement::getId).collect(Collectors.toSet());
int metricOrder = 0;
for (SchemaElement metric : parseInfo.getMetrics()) {
metric.setOrder(metricOrder++);
}
for (Retrieval retrieval : retrievals) {
if (!metricIds.contains(retrieval.getId())) {
SchemaElement schemaElement = JSONObject.parseObject(JSONObject.toJSONString(retrieval.getMetadata()),
SchemaElement.class);
if (retrieval.getMetadata().containsKey("modelId")) {
schemaElement.setModel(Long.parseLong(retrieval.getMetadata().get("modelId")));
}
schemaElement.setOrder(metricOrder++);
parseInfo.getMetrics().add(schemaElement);
}
}
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java
View File

@@ -18,7 +18,6 @@ import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
import com.tencent.supersonic.chat.persistence.repository.ChatContextRepository;
import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository;
import com.tencent.supersonic.chat.persistence.repository.ChatRepository;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Comparator;
@@ -29,7 +28,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.tencent.supersonic.chat.service.ChatService;
import com.tencent.supersonic.chat.utils.SolvedQueryManager;
import com.tencent.supersonic.common.util.JsonUtil;

14
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
@@ -38,10 +39,12 @@ import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -143,16 +146,19 @@ public class ConfigServiceImpl implements ConfigService {
List<Long> filterMetricIdList = blackMetricIdList.stream().distinct().collect(Collectors.toList());
ItemNameVisibilityInfo itemNameVisibility = new ItemNameVisibilityInfo();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
if (!CollectionUtils.isEmpty(blackDimIdList)) {
List<DimensionResp> dimensionRespList = dimensionService.getDimensions(modelId);
List<DimensionResp> dimensionRespList = dimensionService.getDimensions(metaFilter);
List<String> blackDimNameList = dimensionRespList.stream().filter(o -> filterDimIdList.contains(o.getId()))
.map(o -> o.getName()).collect(Collectors.toList());
.map(SchemaItem::getName).collect(Collectors.toList());
itemNameVisibility.setBlackDimNameList(blackDimNameList);
}
if (!CollectionUtils.isEmpty(blackMetricIdList)) {
List<MetricResp> metricRespList = metricService.getMetrics(modelId);
List<MetricResp> metricRespList = metricService.getMetrics(metaFilter);
List<String> blackMetricList = metricRespList.stream().filter(o -> filterMetricIdList.contains(o.getId()))
.map(o -> o.getName()).collect(Collectors.toList());
.map(SchemaItem::getName).collect(Collectors.toList());
itemNameVisibility.setBlackMetricNameList(blackMetricList);
}
return itemNameVisibility;

2
chat/core/src/main/java/com/tencent/supersonic/chat/utils/SolvedQueryManager.java
View File

@@ -5,7 +5,7 @@ import com.alibaba.fastjson.serializer.SerializerFeature;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval;
import lombok.extern.slf4j.Slf4j;

11
chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.test.context;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.when;
@@ -26,6 +26,9 @@ import com.tencent.supersonic.semantic.model.domain.MetricService;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import org.mockito.Mockito;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -81,11 +84,11 @@ public class MockBeansConfiguration {
}
public static void dimensionDescBuild(DimensionService dimensionService, List<DimensionResp> dimensionDescs) {
when(dimensionService.getDimensions(anyList())).thenReturn(dimensionDescs);
when(dimensionService.getDimensions(any(DimensionFilter.class))).thenReturn(dimensionDescs);
}
public static void metricDescBuild(MetricService dimensionService, List<MetricResp> metricDescs) {
when(dimensionService.getMetrics(anyList())).thenReturn(metricDescs);
public static void metricDescBuild(MetricService metricService, List<MetricResp> metricDescs) {
when(metricService.getMetrics(any(MetaFilter.class))).thenReturn(metricDescs);
}
public static DimSchemaResp getDimensionDesc(Long id, String bizName, String name) {

27
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataAddListener.java
View File

@@ -1,27 +0,0 @@
package com.tencent.supersonic.knowledge.listener;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DataAddEvent;
import com.tencent.supersonic.knowledge.dictionary.DictWord;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class DataAddListener implements ApplicationListener<DataAddEvent> {
@Override
public void onApplicationEvent(DataAddEvent event) {
DictWord dictWord = new DictWord();
dictWord.setWord(event.getName());
String sign = DictWordType.NATURE_SPILT;
String nature = sign + event.getModelId() + sign + event.getId() + event.getType();
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);
log.info("dataAddListener begins to add data:{}", dictWord);
HanlpHelper.addToCustomDictionary(dictWord);
}
}

27
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataDeleteListener.java
View File

@@ -1,27 +0,0 @@
package com.tencent.supersonic.knowledge.listener;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DataDeleteEvent;
import com.tencent.supersonic.knowledge.dictionary.DictWord;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class DataDeleteListener implements ApplicationListener<DataDeleteEvent> {
@Override
public void onApplicationEvent(DataDeleteEvent event) {
DictWord dictWord = new DictWord();
dictWord.setWord(event.getName());
String sign = DictWordType.NATURE_SPILT;
String nature = sign + event.getModelId() + sign + event.getId() + event.getType();
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);
log.info("dataDeleteListener begins to delete data:{}", dictWord);
HanlpHelper.removeFromCustomDictionary(dictWord);
}
}

29
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DataUpdateListener.java
View File

@@ -1,29 +0,0 @@
package com.tencent.supersonic.knowledge.listener;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
import com.tencent.supersonic.knowledge.dictionary.DictWord;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class DataUpdateListener implements ApplicationListener<DataUpdateEvent> {
@Override
public void onApplicationEvent(DataUpdateEvent event) {
DictWord dictWord = new DictWord();
dictWord.setWord(event.getName());
String sign = DictWordType.NATURE_SPILT;
String nature = sign + event.getModelId() + sign + event.getId() + event.getType();
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);
log.info("dataUpdateListener begins to update data:{}", dictWord);
HanlpHelper.removeFromCustomDictionary(dictWord);
dictWord.setWord(event.getNewName());
HanlpHelper.addToCustomDictionary(dictWord);
}
}

43
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/DictUpdateListener.java Normal file
View File

@@ -0,0 +1,43 @@
package com.tencent.supersonic.knowledge.listener;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.knowledge.dictionary.DictWord;
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
public class DictUpdateListener implements ApplicationListener<DataEvent> {
@Override
public void onApplicationEvent(DataEvent dataEvent) {
if (CollectionUtils.isEmpty(dataEvent.getDataItems())) {
return;
}
dataEvent.getDataItems().forEach(dataItem -> {
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());
String sign = DictWordType.NATURE_SPILT;
String nature = sign + dataItem.getModelId() + sign + dataItem.getId()
+ sign + dataItem.getType().getName();
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);
if (EventType.ADD.equals(dataEvent.getEventType())) {
HanlpHelper.addToCustomDictionary(dictWord);
} else if (EventType.DELETE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
} else if (EventType.UPDATE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
dictWord.setWord(dataItem.getNewName());
HanlpHelper.addToCustomDictionary(dictWord);
}
});
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/embedding/EmbeddingConfig.java → common/src/main/java/com/tencent/supersonic/common/config/EmbeddingConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.plugin.embedding;
package com.tencent.supersonic.common.config;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;

9
common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import org.springframework.context.ApplicationEvent;
public class DataAddEvent extends ApplicationEvent {
@@ -7,9 +8,9 @@ public class DataAddEvent extends ApplicationEvent {
private String name;
private Long modelId;
private Long id;
private String type;
private TypeEnums type;
public DataAddEvent(Object source, String name, Long modelId, Long id, String type) {
public DataAddEvent(Object source, String name, Long modelId, Long id, TypeEnums type) {
super(source);
this.name = name;
this.modelId = modelId;
@@ -41,11 +42,11 @@ public class DataAddEvent extends ApplicationEvent {
this.modelId = modelId;
}
public void setType(String type) {
public void setType(TypeEnums type) {
this.type = type;
}
public String getType() {
public TypeEnums getType() {
return type;
}
}

9
common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import org.springframework.context.ApplicationEvent;
public class DataDeleteEvent extends ApplicationEvent {
@@ -7,9 +8,9 @@ public class DataDeleteEvent extends ApplicationEvent {
private String name;
private Long modelId;
private Long id;
private String type;
private TypeEnums type;
public DataDeleteEvent(Object source, String name, Long modelId, Long id, String type) {
public DataDeleteEvent(Object source, String name, Long modelId, Long id, TypeEnums type) {
super(source);
this.name = name;
this.modelId = modelId;
@@ -41,11 +42,11 @@ public class DataDeleteEvent extends ApplicationEvent {
this.modelId = modelId;
}
public void setType(String type) {
public void setType(TypeEnums type) {
this.type = type;
}
public String getType() {
public TypeEnums getType() {
return type;
}
}

28
common/src/main/java/com/tencent/supersonic/common/pojo/DataEvent.java Normal file
View File

@@ -0,0 +1,28 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.EventType;
import org.springframework.context.ApplicationEvent;
import java.util.List;
public class DataEvent extends ApplicationEvent {
private List<DataItem> dataItems;
private EventType eventType;
public DataEvent(Object source, List<DataItem> dataItems, EventType eventType) {
super(source);
this.dataItems = dataItems;
this.eventType = eventType;
}
public List<DataItem> getDataItems() {
return dataItems;
}
public EventType getEventType() {
return eventType;
}
}

26
common/src/main/java/com/tencent/supersonic/common/pojo/DataItem.java Normal file
View File

@@ -0,0 +1,26 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class DataItem {
private Long id;
private String bizName;
private String name;
private String newName;
private TypeEnums type;
private Long modelId;
public String getNewName() {
return newName == null ? name : newName;
}
}

9
common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import org.springframework.context.ApplicationEvent;
public class DataUpdateEvent extends ApplicationEvent {
@@ -8,9 +9,9 @@ public class DataUpdateEvent extends ApplicationEvent {
private String newName;
private Long modelId;
private Long id;
private String type;
private TypeEnums type;
public DataUpdateEvent(Object source, String name, String newName, Long modelId, Long id, String type) {
public DataUpdateEvent(Object source, String name, String newName, Long modelId, Long id, TypeEnums type) {
super(source);
this.name = name;
this.newName = newName;
@@ -51,11 +52,11 @@ public class DataUpdateEvent extends ApplicationEvent {
this.modelId = modelId;
}
public void setType(String type) {
public void setType(TypeEnums type) {
this.type = type;
}
public String getType() {
public TypeEnums getType() {
return type;
}
}

9
common/src/main/java/com/tencent/supersonic/common/pojo/enums/EventType.java Normal file
View File

@@ -0,0 +1,9 @@
package com.tencent.supersonic.common.pojo.enums;
public enum EventType {
ADD,
UPDATE,
DELETE
}

17
common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingCollection.java Normal file
View File

@@ -0,0 +1,17 @@
package com.tencent.supersonic.common.util.embedding;
import lombok.Data;
import java.util.Map;
@Data
public class EmbeddingCollection {
private String id;
private String name;
private Map<String, String> metaData;
}

20
common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingQuery.java Normal file
View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.common.util.embedding;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
public class EmbeddingQuery {
private String queryId;
private String query;
private Map<String, String> metadata;
private List<Double> queryEmbedding;
}

107
common/src/main/java/com/tencent/supersonic/common/util/embedding/EmbeddingUtils.java Normal file
View File

@@ -0,0 +1,107 @@
package com.tencent.supersonic.common.util.embedding;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
@Slf4j
@Component
public class EmbeddingUtils {
@Autowired
private EmbeddingConfig embeddingConfig;
private RestTemplate restTemplate = new RestTemplate();
public void addCollection(String collectionName) {
List<String> collections = getCollectionList();
if (collections.contains(collectionName)) {
return;
}
String url = String.format("%s/create_collection?collection_name=%s",
embeddingConfig.getUrl(), collectionName);
doRequest(url, null, HttpMethod.GET);
}
public void addQuery(String collectionName, List<EmbeddingQuery> queries) {
if (CollectionUtils.isEmpty(queries)) {
return;
}
String url = String.format("%s/add_query?collection_name=%s",
embeddingConfig.getUrl(), collectionName);
doRequest(url, JSONObject.toJSONString(queries, SerializerFeature.WriteMapNullValue), HttpMethod.POST);
}
public void deleteQuery(String collectionName, List<EmbeddingQuery> queries) {
if (CollectionUtils.isEmpty(queries)) {
return;
}
List<String> queryIds = queries.stream().map(EmbeddingQuery::getQueryId).collect(Collectors.toList());
String url = String.format("%s/delete_query_by_ids?collection_name=%s",
embeddingConfig.getUrl(), collectionName);
doRequest(url, JSONObject.toJSONString(queryIds), HttpMethod.POST);
}
public List<RetrieveQueryResult> retrieveQuery(String collectionName, RetrieveQuery retrieveQuery, int num) {
String url = String.format("%s/retrieve_query?collection_name=%s&n_results=%s",
embeddingConfig.getUrl(), collectionName, num);
ResponseEntity<String> responseEntity = doRequest(url, JSONObject.toJSONString(retrieveQuery,
SerializerFeature.WriteMapNullValue), HttpMethod.POST);
if (!responseEntity.hasBody()) {
return Lists.newArrayList();
}
return JSONObject.parseArray(responseEntity.getBody(), RetrieveQueryResult.class);
}
private List<String> getCollectionList() {
String url = embeddingConfig.getUrl() + "/list_collections";
ResponseEntity<String> responseEntity = doRequest(url, null, HttpMethod.GET);
if (!responseEntity.hasBody()) {
return Lists.newArrayList();
}
List<EmbeddingCollection> embeddingCollections = JSONObject.parseArray(responseEntity.getBody(),
EmbeddingCollection.class);
return embeddingCollections.stream().map(EmbeddingCollection::getName).collect(Collectors.toList());
}
private ResponseEntity doRequest(String url, String jsonBody, HttpMethod httpMethod) {
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setLocation(URI.create(url));
URI requestUrl = UriComponentsBuilder
.fromHttpUrl(url).build().encode().toUri();
HttpEntity<String> entity = new HttpEntity<>(headers);
if (jsonBody != null) {
log.info("[embedding] request body :{}", jsonBody);
entity = new HttpEntity<>(jsonBody, headers);
}
ResponseEntity<String> responseEntity = restTemplate.exchange(requestUrl,
httpMethod, entity, new ParameterizedTypeReference<String>() {});
log.info("[embedding] url :{} result body:{}", url, responseEntity);
return responseEntity;
} catch (Throwable e) {
log.warn("connect to embedding service failed, url:{}", url);
}
return ResponseEntity.of(Optional.empty());
}
}

18
common/src/main/java/com/tencent/supersonic/common/util/embedding/Retrieval.java Normal file
View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.common.util.embedding;
import lombok.Data;
import java.util.Map;
@Data
public class Retrieval {
private Long id;
private double distance;
private String query;
private Map<String, String> metadata;
}

20
common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQuery.java Normal file
View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.common.util.embedding;
import lombok.Builder;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
@Builder
public class RetrieveQuery {
private List<String> queryTextsList;
private Map<String, String> filterCondition;
private List<List<Double>> queryEmbeddings;
}

15
common/src/main/java/com/tencent/supersonic/common/util/embedding/RetrieveQueryResult.java Normal file
View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.common.util.embedding;
import lombok.Data;
import java.util.List;
@Data
public class RetrieveQueryResult {
private String query;
private List<Retrieval> retrieval;
}

3
launchers/semantic/src/main/resources/db/semantic-schema-h2.sql
View File

@@ -4,6 +4,8 @@ CREATE TABLE IF NOT EXISTS `s2_model` (
`biz_name` varchar(255) DEFAULT NULL , -- internal name
`domain_id` INT DEFAULT '0' , -- parent domain ID
`alias` varchar(255) DEFAULT NULL , -- internal name
`status` INT DEFAULT NULL, -- 0 is off the shelf, 1 is normal
`description` varchar(500) DEFAULT NULL ,
`created_at` TIMESTAMP DEFAULT NULL ,
`created_by` varchar(100) DEFAULT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
@@ -66,6 +68,7 @@ CREATE TABLE IF NOT EXISTS `s2_datasource` (
`description` varchar(500) DEFAULT NULL ,
`database_id` INT NOT NULL ,
`datasource_detail` LONGVARCHAR NOT NULL ,
`status` int(11) DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP NOT NULL ,

3
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -41,7 +41,8 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
com.tencent.supersonic.chat.responder.parse.ParseResponder=\
com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder
com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder, \
com.tencent.supersonic.chat.responder.parse.SimilarMetricParseResponder
com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder

7
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -119,6 +119,8 @@ CREATE TABLE IF NOT EXISTS `s2_model` (
`biz_name` varchar(255) DEFAULT NULL , -- internal name
`domain_id` INT DEFAULT '0' , -- parent domain ID
`alias` varchar(255) DEFAULT NULL , -- internal name
`status` INT DEFAULT NULL,
`description` varchar(500) DEFAULT NULL ,
`created_at` TIMESTAMP DEFAULT NULL ,
`created_by` varchar(100) DEFAULT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
@@ -160,6 +162,7 @@ CREATE TABLE IF NOT EXISTS `s2_datasource` (
`description` varchar(500) DEFAULT NULL ,
`database_id` INT NOT NULL ,
`datasource_detail` LONGVARCHAR NOT NULL ,
`status` int(11) DEFAULT NULL ,
`depends` varchar(500) DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
@@ -182,7 +185,7 @@ CREATE TABLE IF NOT EXISTS `s2_metric` (
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`status` INT NOT NULL , -- status, 0 is normal, 1 is off the shelf, 2 is deleted
`status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal
`sensitive_level` INT NOT NULL ,
`type` varchar(50) NOT NULL , -- type proxy,expr
`type_params` LONGVARCHAR DEFAULT NULL ,
@@ -207,7 +210,7 @@ CREATE TABLE IF NOT EXISTS `s2_dimension` (
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) NOT NULL ,
`status` INT NOT NULL , -- status, 0 is normal, 1 is off the shelf, 2 is deleted
`status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal
`sensitive_level` INT DEFAULT NULL ,
`data_type` varchar(50) DEFAULT NULL , -- type date,array,varchar
`type` varchar(50) NOT NULL , -- type categorical,time

9
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -141,6 +141,7 @@ CREATE TABLE `s2_datasource` (
`description` varchar(500) DEFAULT NULL COMMENT '数据源描述',
`database_id` bigint(20) NOT NULL COMMENT '数据库实例ID',
`datasource_detail` mediumtext NOT NULL COMMENT '数据源配置',
`status` int(11) DEFAULT NULL ,
`depends` text DEFAULT NULL COMMENT '上游依赖标识',
`created_at` datetime NOT NULL COMMENT '创建时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
@@ -202,7 +203,7 @@ CREATE TABLE `s2_dimension` (
`name` varchar(255) NOT NULL COMMENT '维度名称',
`biz_name` varchar(255) NOT NULL COMMENT '字段名称',
`description` varchar(500) NOT NULL COMMENT '描述',
`status` int(10) NOT NULL COMMENT '维度状态,0正常,1下架,2删除',
`status` int(10) NOT NULL COMMENT '维度状态,0正常,1下架',
`sensitive_level` int(10) DEFAULT NULL COMMENT '敏感级别',
`type` varchar(50) NOT NULL COMMENT '维度类型 categorical,time',
`type_params` text COMMENT '类型参数',
@@ -245,9 +246,9 @@ CREATE TABLE `s2_metric` (
`name` varchar(255) NOT NULL COMMENT '指标名称',
`biz_name` varchar(255) NOT NULL COMMENT '字段名称',
`description` varchar(500) DEFAULT NULL COMMENT '描述',
`status` int(10) NOT NULL COMMENT '指标状态,0正常,1下架,2删除',
`status` int(10) NOT NULL COMMENT '指标状态,0未启用,1启用',
`sensitive_level` int(10) NOT NULL COMMENT '敏感级别',
`type` varchar(50) NOT NULL COMMENT '指标类型 proxy,expr',
`type` varchar(50) NOT NULL COMMENT '指标类型',
`type_params` text NOT NULL COMMENT '类型参数',
`created_at` datetime NOT NULL COMMENT '创建时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
@@ -268,6 +269,8 @@ CREATE TABLE `s2_model` (
`biz_name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
`domain_id` bigint(20) DEFAULT NULL,
`alias` varchar(200) COLLATE utf8_unicode_ci DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`description` varchar(500) DEFAULT NULL,
`viewer` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL,
`view_org` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL,
`admin` varchar(500) COLLATE utf8_unicode_ci DEFAULT NULL,

9
launchers/standalone/src/main/resources/db/sql-update.sql
View File

@@ -68,3 +68,12 @@ alter table s2_datasource add column depends text COMMENT '上游依赖标识' a
--20231018
UPDATE `s2_agent` SET `config` = replace (`config`,'DSL','LLM_S2QL') WHERE `config` LIKE '%DSL%';
--20231023
alter table s2_model add column status int null after alias;
alter table s2_model add column description varchar(500) null after status;
alter table s2_datasource add column status int null after database_id;
update s2_model set status = 1;
update s2_datasource set status = 1;
update s2_metric set status = 1;
update s2_dimension set status = 1;

2
launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java
View File

@@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.StandaloneLauncher;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.query.llm.interpret.LLmAnswerResp;
import com.tencent.supersonic.chat.service.AgentService;

2
launchers/standalone/src/test/java/com/tencent/supersonic/integration/MockConfiguration.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.integration;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingResp;
import com.tencent.supersonic.chat.parser.plugin.embedding.RecallRetrieval;
import com.tencent.supersonic.chat.plugin.PluginManager;

36
launchers/standalone/src/test/java/com/tencent/supersonic/integration/model/MetricServiceImplTest.java Normal file
View File

@@ -0,0 +1,36 @@
package com.tencent.supersonic.integration.model;
import com.google.common.collect.Lists;
import com.tencent.supersonic.StandaloneLauncher;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.enums.MetricTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import java.util.List;
@SpringBootTest(classes = StandaloneLauncher.class)
@ActiveProfiles("local")
public class MetricServiceImplTest {
@Autowired
protected MetricService metricService;
@Test
void getMetrics() {
MetricFilter metricFilter = new MetricFilter();
metricFilter.setType(MetricTypeEnum.ATOMIC.name());
metricFilter.setModelIds(Lists.newArrayList(1L));
metricFilter.setSensitiveLevel(SensitiveLevelEnum.LOW.ordinal());
List<MetricResp> metricResps = metricService.getMetrics(metricFilter);
Assertions.assertTrue(metricResps.stream().noneMatch(metricResp -> metricResp.getModelId().equals(2L)));
Assertions.assertTrue(metricResps.stream().noneMatch(metricResp ->
metricResp.getSensitiveLevel().equals(SensitiveLevelEnum.HIGH.ordinal())));
}
}

2
launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingConfig;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.service.QueryService;

3
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -134,6 +134,8 @@ CREATE TABLE IF NOT EXISTS `s2_model` (
`biz_name` varchar(255) DEFAULT NULL , -- internal name
`domain_id` INT DEFAULT '0' , -- parent domain ID
`alias` varchar(255) DEFAULT NULL , -- alias name
`status` INT DEFAULT NULL ,
`description` varchar(500) DEFAULT NULL ,
`created_at` TIMESTAMP DEFAULT NULL ,
`created_by` varchar(100) DEFAULT NULL ,
`updated_at` TIMESTAMP DEFAULT NULL ,
@@ -176,6 +178,7 @@ CREATE TABLE IF NOT EXISTS `s2_datasource` (
`database_id` INT NOT NULL ,
`depends` varchar(500) DEFAULT NULL ,
`datasource_detail` LONGVARCHAR NOT NULL ,
`status` int(11) DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP NOT NULL ,

3
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/SchemaItem.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.semantic.api.model.pojo;
import com.google.common.base.Objects;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,7 +24,7 @@ public class SchemaItem extends RecordInfo {
private String description;
private Integer status = StatusEnum.ONLINE.getCode();
private Integer status;
private TypeEnums typeEnum;

13
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetaBatchReq.java Normal file
View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.semantic.api.model.request;
import lombok.Data;
import java.util.List;
@Data
public class MetaBatchReq {
private List<Long> ids;
private Integer status;
}

9
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/MetricBaseReq.java
View File

@@ -4,6 +4,8 @@ import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.common.pojo.DataFormat;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.List;
@@ -22,4 +24,11 @@ public class MetricBaseReq extends SchemaItem {
private RelateDimension relateDimension;
public String getTag() {
if (CollectionUtils.isEmpty(tags)) {
return "";
}
return StringUtils.join(tags, ",");
}
}

18
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/ModelReq.java
View File

@@ -30,4 +30,22 @@ public class ModelReq extends SchemaItem {
private Entity entity;
private List<DrillDownDimension> drillDownDimensions;
public String getViewer() {
return String.join(",", viewers);
}
public String getViewOrg() {
return String.join(",", viewOrgs);
}
public String getAdmin() {
return String.join(",", admins);
}
public String getAdminOrg() {
return String.join(",", adminOrgs);
}
}

6
semantic/materialization/src/main/java/com/tencent/supersonic/semantic/materialization/domain/utils/MaterializationZipperUtils.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.semantic.materialization.domain.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.materialization.enums.ElementFrequencyEnum;
@@ -9,6 +10,7 @@ import com.tencent.supersonic.semantic.api.materialization.response.Materializat
import com.tencent.supersonic.semantic.api.materialization.response.MaterializationResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -60,7 +62,9 @@ public class MaterializationZipperUtils implements MaterializationUtils {
return "";
}
StringJoiner joiner = new StringJoiner(",");
Map<Long, DimensionResp> dimIdAndDim = dimensionService.getDimensions(materializationResp.getModelId())
DimensionFilter dimensionFilter = new DimensionFilter();
dimensionFilter.setModelIds(Lists.newArrayList(materializationResp.getModelId()));
Map<Long, DimensionResp> dimIdAndDim = dimensionService.getDimensions(dimensionFilter)
.stream().collect(Collectors.toMap(DimensionResp::getId, value -> value, (v1, v2) -> v2));
materializationElementRespList.stream()
.filter(element -> TypeEnums.DIMENSION.equals(element.getType()) && ElementFrequencyEnum.LOW.equals(

11
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java
View File

@@ -1,5 +1,7 @@
package com.tencent.supersonic.semantic.model.application;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
@@ -22,6 +24,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
@@ -84,7 +88,9 @@ public class CatalogImpl implements Catalog {
@Override
public List<DimensionResp> getDimensions(Long modelId) {
return dimensionService.getDimensions(modelId);
MetaFilter metricFilter = new MetaFilter(Lists.newArrayList(modelId));
metricFilter.setStatus(StatusEnum.ONLINE.getCode());
return dimensionService.getDimensions(metricFilter);
}
@Override
@@ -94,7 +100,8 @@ public class CatalogImpl implements Catalog {
@Override
public List<MetricResp> getMetrics(Long modelId) {
return metricService.getMetrics(modelId);
MetaFilter metricFilter = new MetaFilter(Lists.newArrayList(modelId));
return metricService.getMetrics(metricFilter);
}
@Override

57
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.model.application;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
@@ -36,6 +36,7 @@ import com.tencent.supersonic.semantic.model.domain.manager.DatasourceYamlManage
import com.tencent.supersonic.semantic.model.domain.manager.DimensionYamlManager;
import com.tencent.supersonic.semantic.model.domain.manager.MetricYamlManager;
import com.tencent.supersonic.semantic.model.domain.pojo.Datasource;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.model.domain.repository.DatasourceRepository;
import com.tencent.supersonic.semantic.model.domain.repository.DateInfoRepository;
import com.tencent.supersonic.semantic.model.domain.utils.DatasourceConverter;
@@ -74,7 +75,6 @@ public class DatasourceServiceImpl implements DatasourceService {
private DateInfoRepository dateInfoRepository;
public DatasourceServiceImpl(DatasourceRepository datasourceRepository,
DatabaseService databaseService,
@Lazy DimensionService dimensionService,
@@ -89,36 +89,25 @@ public class DatasourceServiceImpl implements DatasourceService {
@Override
@Transactional
public DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception {
preCheck(datasourceReq);
public Datasource createDatasource(DatasourceReq datasourceReq, User user) throws Exception {
checkName(datasourceReq);
checkExist(datasourceReq);
Datasource datasource = DatasourceConverter.convert(datasourceReq);
log.info("[create datasource] object:{}", JSONObject.toJSONString(datasource));
saveDatasource(datasource, user);
Optional<DatasourceResp> datasourceDescOptional = getDatasource(datasourceReq.getModelId(),
datasourceReq.getBizName());
if (!datasourceDescOptional.isPresent()) {
throw new RuntimeException("创建数据源失败");
}
DatasourceResp datasourceResp = datasourceDescOptional.get();
datasource.setId(datasourceResp.getId());
batchCreateDimension(datasource, user);
batchCreateMetric(datasource, user);
return datasourceResp;
return datasource;
}
@Override
@Transactional
public DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception {
preCheck(datasourceReq);
public Datasource updateDatasource(DatasourceReq datasourceReq, User user) throws Exception {
checkName(datasourceReq);
Datasource datasource = DatasourceConverter.convert(datasourceReq);
log.info("[update datasource] object:{}", JSONObject.toJSONString(datasource));
updateDatasource(datasource, user);
batchCreateDimension(datasource, user);
batchCreateMetric(datasource, user);
DatasourceDO datasourceDO = updateDatasource(datasource, user);
return DatasourceConverter.convert(datasourceDO);
return datasource;
}
private DatasourceDO updateDatasource(Datasource datasource, User user) {
@@ -168,7 +157,6 @@ public class DatasourceServiceImpl implements DatasourceService {
return measureResps;
}
private void batchCreateDimension(Datasource datasource, User user) throws Exception {
List<DimensionReq> dimensionReqs = DatasourceConverter.convertDimensionList(datasource);
dimensionService.createDimensionBatch(dimensionReqs, user);
@@ -179,7 +167,6 @@ public class DatasourceServiceImpl implements DatasourceService {
metricService.createMetricBatch(exprMetricReqs, user);
}
private Optional<DatasourceResp> getDatasource(Long modelId, String bizName) {
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
if (CollectionUtils.isEmpty(datasourceResps)) {
@@ -196,13 +183,11 @@ public class DatasourceServiceImpl implements DatasourceService {
//保存并获取自增ID
private void saveDatasource(Datasource datasource, User user) {
DatasourceDO datasourceDO = DatasourceConverter.convert(datasource, user);
log.info("[save datasource] datasourceDO:{}", JSONObject.toJSONString(datasourceDO));
datasourceRepository.createDatasource(datasourceDO);
datasource.setId(datasourceDO.getId());
}
private void preCheck(DatasourceReq datasourceReq) {
private void checkName(DatasourceReq datasourceReq) {
if (NameCheckUtils.containsSpecialCharacters(datasourceReq.getName())) {
String message = String.format("数据源名称[%s]包含特殊字符, 请修改", datasourceReq.getName());
throw new InvalidArgumentException(message);
@@ -297,18 +282,23 @@ public class DatasourceServiceImpl implements DatasourceService {
@Override
public void deleteDatasource(Long id) {
public void deleteDatasource(Long id, User user) {
DatasourceDO datasourceDO = datasourceRepository.getDatasourceById(id);
if (datasourceDO == null) {
return;
}
checkDelete(datasourceDO.getModelId(), id);
datasourceRepository.deleteDatasource(id);
checkDelete(id);
datasourceDO.setStatus(StatusEnum.DELETED.getCode());
datasourceDO.setUpdatedAt(new Date());
datasourceDO.setUpdatedBy(user.getName());
datasourceRepository.updateDatasource(datasourceDO);
}
private void checkDelete(Long modelId, Long datasourceId) {
List<MetricResp> metricResps = metricService.getMetrics(modelId, datasourceId);
List<DimensionResp> dimensionResps = dimensionService.getDimensionsByDatasource(datasourceId);
private void checkDelete(Long datasourceId) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setDatasourceId(datasourceId);
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(dimensionResps)) {
throw new RuntimeException("存在基于该数据源创建的指标和维度, 暂不能删除, 请确认");
}
@@ -420,11 +410,12 @@ public class DatasourceServiceImpl implements DatasourceService {
List<DatasourceYamlTpl> datasourceYamlTplList, List<MetricYamlTpl> metricYamlTplList) {
for (Long modelId : modelIds) {
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);
List<MetricResp> metricResps = metricService.getMetrics(modelId);
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId));
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
metricYamlTplList.addAll(MetricYamlManager.convert2YamlObj(MetricConverter.metricInfo2Metric(metricResps)));
Long databaseId = datasourceResps.iterator().next().getDatabaseId();
DatabaseResp databaseResp = databaseService.getDatabase(databaseId);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(modelId);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
for (DatasourceResp datasourceResp : datasourceResps) {
datasourceYamlTplList.add(DatasourceYamlManager.convert2YamlObj(
DatasourceConverter.datasourceInfo2Datasource(datasourceResp), databaseResp));

268
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java
View File

@@ -7,37 +7,37 @@ import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataAddEvent;
import com.tencent.supersonic.common.pojo.DataDeleteEvent;
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.model.domain.DatabaseService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.model.domain.repository.DimensionRepository;
import com.tencent.supersonic.semantic.model.domain.utils.DimensionConverter;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.pojo.Dimension;
import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
@@ -63,7 +63,7 @@ public class DimensionServiceImpl implements DimensionService {
private DatabaseService databaseService;
@Autowired
private ApplicationEventPublisher applicationEventPublisher;
private ApplicationEventPublisher eventPublisher;
public DimensionServiceImpl(DimensionRepository dimensionRepository,
@@ -76,23 +76,17 @@ public class DimensionServiceImpl implements DimensionService {
this.datasourceService = datasourceService;
this.chatGptHelper = chatGptHelper;
this.databaseService = databaseService;
}
@Override
public void createDimension(DimensionReq dimensionReq, User user) {
checkExist(Lists.newArrayList(dimensionReq));
Dimension dimension = DimensionConverter.convert(dimensionReq);
log.info("[create dimension] object:{}", JSONObject.toJSONString(dimension));
dimension.createdBy(user.getName());
saveDimension(dimension);
String type = DictWordType.DIMENSION.getType();
DimensionResp dimensionResp = getDimension(dimension.getBizName(), dimension.getModelId());
applicationEventPublisher.publishEvent(
new DataAddEvent(this, dimension.getName(), dimension.getModelId(), dimensionResp.getId(), type));
dimensionReq.createdBy(user.getName());
DimensionDO dimensionDO = DimensionConverter.convert2DimensionDO(dimensionReq);
dimensionRepository.createDimension(dimensionDO);
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD);
}
@Override
public void createDimensionBatch(List<DimensionReq> dimensionReqs, User user) {
if (CollectionUtils.isEmpty(dimensionReqs)) {
@@ -100,44 +94,79 @@ public class DimensionServiceImpl implements DimensionService {
}
Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensions(modelId);
Map<String, DimensionResp> dimensionRespMap = dimensionResps.stream()
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
List<Dimension> dimensions = dimensionReqs.stream().map(DimensionConverter::convert)
Map<String, DimensionResp> nameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
List<DimensionReq> dimensionToInsert = dimensionReqs.stream()
.filter(dimension -> !bizNameMap.containsKey(dimension.getBizName())
&& !nameMap.containsKey(dimension.getName()))
.collect(Collectors.toList());
List<Dimension> dimensionToInsert = dimensions.stream()
.filter(dimension -> !dimensionRespMap.containsKey(dimension.getBizName()))
if (CollectionUtils.isEmpty(dimensionToInsert)) {
return;
}
List<DimensionDO> dimensionDOS = dimensionToInsert.stream().peek(dimension ->
dimension.createdBy(user.getName()))
.map(DimensionConverter::convert2DimensionDO)
.collect(Collectors.toList());
log.info("[create dimension] object:{}", JSONObject.toJSONString(dimensions));
saveDimensionBatch(dimensionToInsert, user);
dimensionRepository.createDimensionBatch(dimensionDOS);
sendEventBatch(dimensionDOS, EventType.ADD);
}
@Override
public void updateDimension(DimensionReq dimensionReq, User user) {
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
Dimension dimension = DimensionConverter.convert(dimensionReq);
dimension.updatedBy(user.getName());
log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
updateDimension(dimension);
DimensionResp dimensionResp = getDimension(dimensionReq.getId());
//动态更新字典
String type = DictWordType.DIMENSION.getType();
if (dimensionResp != null) {
applicationEventPublisher.publishEvent(
new DataUpdateEvent(this, dimensionResp.getName(),
dimensionReq.getName(),
dimension.getModelId(),
dimensionResp.getId(), type));
checkExist(Lists.newArrayList(dimensionReq));
DimensionDO dimensionDO = dimensionRepository.getDimensionById(dimensionReq.getId());
dimensionReq.updatedBy(user.getName());
String oldName = dimensionDO.getName();
DimensionConverter.convert(dimensionDO, dimensionReq);
dimensionRepository.updateDimension(dimensionDO);
if (!oldName.equals(dimensionDO.getName())) {
sendEvent(DataItem.builder().modelId(dimensionDO.getModelId()).newName(dimensionReq.getName())
.name(oldName).type(TypeEnums.DIMENSION)
.id(dimensionDO.getId()).build(), EventType.UPDATE);
}
}
protected void updateDimension(Dimension dimension) {
DimensionDO dimensionDO = dimensionRepository.getDimensionById(dimension.getId());
dimensionRepository.updateDimension(DimensionConverter.convert(dimensionDO, dimension));
@Override
public void batchUpdateStatus(MetaBatchReq metaBatchReq, User user) {
if (CollectionUtils.isEmpty(metaBatchReq.getIds())) {
return;
}
DimensionFilter dimensionFilter = new DimensionFilter();
dimensionFilter.setIds(metaBatchReq.getIds());
List<DimensionDO> dimensionDOS = dimensionRepository.getDimension(dimensionFilter);
if (CollectionUtils.isEmpty(dimensionDOS)) {
return;
}
dimensionDOS = dimensionDOS.stream()
.peek(dimensionDO -> {
dimensionDO.setStatus(metaBatchReq.getStatus());
dimensionDO.setUpdatedAt(new Date());
dimensionDO.setUpdatedBy(user.getName());
})
.collect(Collectors.toList());
dimensionRepository.batchUpdateStatus(dimensionDOS);
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
sendEventBatch(dimensionDOS, EventType.DELETE);
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
sendEventBatch(dimensionDOS, EventType.ADD);
}
}
@Override
public void deleteDimension(Long id, User user) {
DimensionDO dimensionDO = dimensionRepository.getDimensionById(id);
if (dimensionDO == null) {
throw new RuntimeException(String.format("the dimension %s not exist", id));
}
dimensionDO.setStatus(StatusEnum.DELETED.getCode());
dimensionDO.setUpdatedAt(new Date());
dimensionDO.setUpdatedBy(user.getName());
dimensionRepository.updateDimension(dimensionDO);
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.DELETE);
}
@Override
public DimensionResp getDimension(String bizName, Long modelId) {
@@ -177,48 +206,15 @@ public class DimensionServiceImpl implements DimensionService {
return dimensionRepository.getDimension(dimensionFilter);
}
@Override
public List<DimensionResp> getDimensions(List<Long> ids) {
List<DimensionResp> dimensionResps = Lists.newArrayList();
List<DimensionDO> dimensionDOS = dimensionRepository.getDimensionListByIds(ids);
Map<Long, String> modelFullPathMap = modelService.getModelFullPathMap();
if (!CollectionUtils.isEmpty(dimensionDOS)) {
dimensionResps = dimensionDOS.stream()
.map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelFullPathMap,
new HashMap<>()))
.collect(Collectors.toList());
}
return dimensionResps;
public List<DimensionResp> getDimensions(MetaFilter metaFilter) {
DimensionFilter dimensionFilter = new DimensionFilter();
BeanUtils.copyProperties(metaFilter, dimensionFilter);
return convertList(dimensionRepository.getDimension(dimensionFilter), datasourceService.getDatasourceMap());
}
@Override
public List<DimensionResp> getDimensions(Long modelId) {
return convertList(getDimensionDOS(modelId), datasourceService.getDatasourceMap());
}
@Override
public List<DimensionResp> getDimensions() {
return convertList(getDimensionDOS(), datasourceService.getDatasourceMap());
}
@Override
public List<DimensionResp> getDimensionsByModelIds(List<Long> modelIds) {
List<DimensionDO> dimensionDOS = dimensionRepository.getDimensionListOfmodelIds(modelIds);
return convertList(dimensionDOS, datasourceService.getDatasourceMap());
}
@Override
public List<DimensionResp> getDimensionsByDatasource(Long datasourceId) {
List<DimensionResp> dimensionResps = Lists.newArrayList();
List<DimensionDO> dimensionDOS = dimensionRepository.getDimensionListOfDatasource(datasourceId);
if (!CollectionUtils.isEmpty(dimensionDOS)) {
dimensionResps = dimensionDOS.stream()
.map(dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, new HashMap<>(),
new HashMap<>()))
.collect(Collectors.toList());
}
return dimensionResps;
private List<DimensionResp> getDimensions(Long modelId) {
return getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
}
private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS,
@@ -234,75 +230,6 @@ public class DimensionServiceImpl implements DimensionService {
return dimensionResps;
}
@Override
public List<DimensionResp> getHighSensitiveDimension(Long modelId) {
List<DimensionResp> dimensionResps = getDimensions(modelId);
if (CollectionUtils.isEmpty(dimensionResps)) {
return dimensionResps;
}
return dimensionResps.stream()
.filter(dimensionResp -> SensitiveLevelEnum.HIGH.getCode().equals(dimensionResp.getSensitiveLevel()))
.collect(Collectors.toList());
}
protected List<DimensionDO> getDimensionDOS(Long modelId) {
return dimensionRepository.getDimensionListOfmodel(modelId);
}
protected List<DimensionDO> getDimensionDOS() {
return dimensionRepository.getDimensionList();
}
@Override
public List<DimensionResp> getAllHighSensitiveDimension() {
List<DimensionResp> dimensionResps = Lists.newArrayList();
List<DimensionDO> dimensionDOS = dimensionRepository.getAllDimensionList();
if (CollectionUtils.isEmpty(dimensionDOS)) {
return dimensionResps;
}
return convertList(dimensionDOS.stream()
.filter(dimensionDO -> SensitiveLevelEnum.HIGH.getCode().equals(dimensionDO.getSensitiveLevel()))
.collect(Collectors.toList()), new HashMap<>());
}
public void saveDimension(Dimension dimension) {
DimensionDO dimensionDO = DimensionConverter.convert2DimensionDO(dimension);
log.info("[save dimension] dimensionDO:{}", JSONObject.toJSONString(dimensionDO));
dimensionRepository.createDimension(dimensionDO);
dimension.setId(dimensionDO.getId());
}
private void saveDimensionBatch(List<Dimension> dimensions, User user) {
if (CollectionUtils.isEmpty(dimensions)) {
return;
}
dimensions = dimensions.stream().peek(dimension -> dimension.createdBy(user.getName()))
.collect(Collectors.toList());
List<DimensionDO> dimensionDOS = dimensions.stream()
.map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
log.info("[save dimension] dimensionDO:{}", JSONObject.toJSONString(dimensionDOS));
dimensionRepository.createDimensionBatch(dimensionDOS);
}
@Override
public void deleteDimension(Long id) {
DimensionDO dimensionDO = dimensionRepository.getDimensionById(id);
if (dimensionDO == null) {
throw new RuntimeException(String.format("the dimension %s not exist", id));
}
dimensionRepository.deleteDimension(id);
//动态更新字典
String type = DictWordType.DIMENSION.getType();
applicationEventPublisher.publishEvent(
new DataDeleteEvent(this, dimensionDO.getName(), dimensionDO.getModelId(), dimensionDO.getId(), type));
}
@Override
public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
String mockAlias = chatGptHelper.mockAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(),
@@ -368,20 +295,43 @@ public class DimensionServiceImpl implements DimensionService {
private void checkExist(List<DimensionReq> dimensionReqs) {
Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensions(modelId);
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> nameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
for (DimensionReq dimensionReq : dimensionReqs) {
if (NameCheckUtils.containsSpecialCharacters(dimensionReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
for (DimensionResp dimensionResp : dimensionResps) {
if (dimensionResp.getName().equalsIgnoreCase(dimensionReq.getName())) {
throw new RuntimeException(String.format("存在相同的维度名 :%s", dimensionReq.getName()));
if (bizNameMap.containsKey(dimensionReq.getBizName())) {
DimensionResp dimensionResp = bizNameMap.get(dimensionReq.getBizName());
if (!dimensionResp.getId().equals(dimensionReq.getId())) {
throw new RuntimeException(String.format("该模型下存在相同的维度字段名:%s 创建人:%s",
dimensionReq.getBizName(), dimensionResp.getCreatedBy()));
}
if (dimensionResp.getBizName().equalsIgnoreCase(dimensionReq.getBizName())) {
throw new RuntimeException(
String.format("存在相同的维度名: %s", dimensionReq.getBizName()));
}
if (nameMap.containsKey(dimensionReq.getName())) {
DimensionResp dimensionResp = nameMap.get(dimensionReq.getName());
if (!dimensionResp.getId().equals(dimensionReq.getId())) {
throw new RuntimeException(String.format("该模型下存在相同的维度名:%s 创建人:%s",
dimensionReq.getName(), dimensionResp.getCreatedBy()));
}
}
}
}
private void sendEventBatch(List<DimensionDO> dimensionDOS, EventType eventType) {
List<DataItem> dataItems = dimensionDOS.stream().map(dimensionDO ->
DataItem.builder().id(dimensionDO.getId()).name(dimensionDO.getName())
.modelId(dimensionDO.getModelId()).type(TypeEnums.DIMENSION).build())
.collect(Collectors.toList());
eventPublisher.publishEvent(new DataEvent(this, dataItems, eventType));
}
private void sendEvent(DataItem dataItem, EventType eventType) {
eventPublisher.publishEvent(new DataEvent(this,
Lists.newArrayList(dataItem), eventType));
}
}

313
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java
View File

@@ -6,31 +6,32 @@ import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataAddEvent;
import com.tencent.supersonic.common.pojo.DataDeleteEvent;
import com.tencent.supersonic.common.pojo.DataUpdateEvent;
import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.MetricReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository;
import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.pojo.Metric;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -60,7 +61,7 @@ public class MetricServiceImpl implements MetricService {
private ChatGptHelper chatGptHelper;
@Autowired
private ApplicationEventPublisher applicationEventPublisher;
private ApplicationEventPublisher eventPublisher;
public MetricServiceImpl(MetricRepository metricRepository,
ModelService modelService,
@@ -73,17 +74,13 @@ public class MetricServiceImpl implements MetricService {
}
@Override
public void creatExprMetric(MetricReq metricReq, User user) {
check(Lists.newArrayList(metricReq));
Metric metric = MetricConverter.convert(metricReq);
metric.createdBy(user.getName());
log.info("[create metric] object:{}", JSONObject.toJSONString(metric));
saveMetric(metric);
//动态更新字典
String type = DictWordType.METRIC.getType();
MetricResp metricResp = getMetric(metric.getModelId(), metric.getBizName());
applicationEventPublisher.publishEvent(
new DataAddEvent(this, metric.getName(), metric.getModelId(), metricResp.getId(), type));
public void createMetric(MetricReq metricReq, User user) {
checkExist(Lists.newArrayList(metricReq));
checkParam(metricReq);
metricReq.createdBy(user.getName());
MetricDO metricDO = MetricConverter.convert2MetricDO(metricReq);
metricRepository.createMetric(metricDO);
sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD);
}
@Override
@@ -91,47 +88,79 @@ public class MetricServiceImpl implements MetricService {
if (CollectionUtils.isEmpty(metricReqs)) {
return;
}
List<Metric> metrics = metricReqs.stream().map(MetricConverter::convert).collect(Collectors.toList());
Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricResps = getMetricByModelId(modelId);
Map<String, MetricResp> metricRespMap = metricResps.stream()
List<MetricResp> metricResps = getMetricInSameDomain(modelId);
Map<String, MetricResp> bizNameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
List<Metric> metricToInsert = metrics.stream()
.filter(metric -> !metricRespMap.containsKey(metric.getBizName())).collect(Collectors.toList());
log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert));
saveMetricBatch(metricToInsert, user);
Map<String, MetricResp> nameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
List<MetricReq> metricToInsert = metricReqs.stream()
.filter(metric -> !bizNameMap.containsKey(metric.getBizName())
&& !nameMap.containsKey(metric.getName())).collect(Collectors.toList());
if (CollectionUtils.isEmpty(metricToInsert)) {
return;
}
List<MetricDO> metricDOS = metricToInsert.stream().peek(metric -> metric.createdBy(user.getName()))
.map(MetricConverter::convert2MetricDO).collect(Collectors.toList());
metricRepository.createMetricBatch(metricDOS);
sendEventBatch(metricDOS, EventType.ADD);
}
@Override
public List<MetricResp> getMetrics(Long modelId) {
return convertList(metricRepository.getMetricList(modelId));
public void updateExprMetric(MetricReq metricReq, User user) {
checkParam(metricReq);
checkExist(Lists.newArrayList(metricReq));
metricReq.updatedBy(user.getName());
MetricDO metricDO = metricRepository.getMetricById(metricReq.getId());
String oldName = metricDO.getName();
MetricConverter.convert(metricDO, metricReq);
metricRepository.updateMetric(metricDO);
if (!oldName.equals(metricDO.getName())) {
DataItem dataItem = getDataItem(metricDO);
dataItem.setName(oldName);
dataItem.setNewName(metricDO.getName());
sendEvent(getDataItem(metricDO), EventType.UPDATE);
}
}
@Override
public List<MetricResp> getMetrics() {
return convertList(metricRepository.getMetricList());
public void batchUpdateStatus(MetaBatchReq metaBatchReq, User user) {
if (CollectionUtils.isEmpty(metaBatchReq.getIds())) {
return;
}
MetricFilter metricFilter = new MetricFilter();
metricFilter.setIds(metaBatchReq.getIds());
List<MetricDO> metricDOS = metricRepository.getMetric(metricFilter);
if (CollectionUtils.isEmpty(metricDOS)) {
return;
}
metricDOS = metricDOS.stream()
.peek(metricDO -> {
metricDO.setStatus(metaBatchReq.getStatus());
metricDO.setUpdatedAt(new Date());
metricDO.setUpdatedBy(user.getName());
})
.collect(Collectors.toList());
metricRepository.batchUpdateStatus(metricDOS);
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
sendEventBatch(metricDOS, EventType.DELETE);
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
sendEventBatch(metricDOS, EventType.ADD);
}
}
@Override
public List<MetricResp> getMetrics(Long modelId, Long datasourceId) {
List<MetricResp> metricResps = convertList(metricRepository.getMetricList(modelId));
return metricResps.stream().filter(metricResp -> {
Set<Long> datasourceIdSet = metricResp.getTypeParams().getMeasures().stream()
.map(Measure::getDatasourceId)
.filter(Objects::nonNull).collect(Collectors.toSet());
return !CollectionUtils.isEmpty(datasourceIdSet) && datasourceIdSet.contains(datasourceId);
}).collect(Collectors.toList());
public void deleteMetric(Long id, User user) {
MetricDO metricDO = metricRepository.getMetricById(id);
if (metricDO == null) {
throw new RuntimeException(String.format("the metric %s not exist", id));
}
public List<MetricResp> getMetrics(List<Long> ids) {
List<MetricDO> metricDOS = metricRepository.getMetricListByIds(ids);
return convertList(metricDOS);
}
@Override
public List<MetricResp> getMetricsByModelIds(List<Long> modelIds) {
List<MetricDO> metricDOS = metricRepository.getMetricList(modelIds);
return convertList(metricDOS);
metricDO.setStatus(StatusEnum.DELETED.getCode());
metricDO.setUpdatedAt(new Date());
metricDO.setUpdatedBy(user.getName());
metricRepository.updateMetric(metricDO);
sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE);
}
@Override
@@ -155,10 +184,30 @@ public class MetricServiceImpl implements MetricService {
return pageInfo;
}
private List<MetricDO> queryMetric(MetricFilter metricFilter) {
protected List<MetricDO> queryMetric(MetricFilter metricFilter) {
return metricRepository.getMetric(metricFilter);
}
@Override
public List<MetricResp> getMetrics(MetaFilter metaFilter) {
MetricFilter metricFilter = new MetricFilter();
BeanUtils.copyProperties(metaFilter, metricFilter);
List<MetricResp> metricResps = convertList(queryMetric(metricFilter));
if (metricFilter.getDatasourceId() != null) {
return filterByDatasource(metricFilter.getDatasourceId(), metricResps);
}
return metricResps;
}
private List<MetricResp> filterByDatasource(Long datasourceId, List<MetricResp> metricResps) {
return metricResps.stream().filter(metricResp -> {
Set<Long> datasourceIdSet = metricResp.getTypeParams().getMeasures().stream()
.map(Measure::getDatasourceId)
.filter(Objects::nonNull).collect(Collectors.toSet());
return !CollectionUtils.isEmpty(datasourceIdSet)
&& datasourceIdSet.contains(datasourceId);
}).collect(Collectors.toList());
}
private void fillAdminRes(List<MetricResp> metricResps, User user) {
List<ModelResp> modelResps = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
@@ -171,22 +220,19 @@ public class MetricServiceImpl implements MetricService {
metricResp.setHasAdminRes(true);
}
}
}
@Override
public MetricResp getMetric(Long modelId, String bizName) {
List<MetricResp> metricResps = getMetricByModelId(modelId);
MetricFilter metricFilter = new MetricFilter();
metricFilter.setBizName(bizName);
metricFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metricFilter);
MetricResp metricResp = null;
if (CollectionUtils.isEmpty(metricResps)) {
return metricResp;
}
for (MetricResp metric : metricResps) {
if (metric.getBizName().equalsIgnoreCase(bizName)) {
metricResp = metric;
}
}
return metricResp;
return metricResps.get(0);
}
private MetricResp getMetric(Long id) {
@@ -197,87 +243,6 @@ public class MetricServiceImpl implements MetricService {
return MetricConverter.convert2MetricResp(metricDO, new HashMap<>());
}
@Override
public void updateExprMetric(MetricReq metricReq, User user) {
preCheckMetric(metricReq);
Metric metric = MetricConverter.convert(metricReq);
metric.updatedBy(user.getName());
log.info("[update metric] object:{}", JSONObject.toJSONString(metric));
List<MetricResp> metricRespList = getMetrics(metricReq.getModelId()).stream().filter(
o -> o.getId().equals(metricReq.getId())).collect(Collectors.toList());
updateMetric(metric);
//动态更新字典
String type = DictWordType.METRIC.getType();
//MetricResp metricResp = getMetric(metric.getModelId(), metric.getBizName());
if (!CollectionUtils.isEmpty(metricRespList)) {
log.info("metricRespList size:{}", metricRespList.size());
log.info("name:{}", metricRespList.get(0).getName());
applicationEventPublisher.publishEvent(
new DataUpdateEvent(this, metricRespList.get(0).getName(),
metricReq.getName(),
metric.getModelId(),
metricRespList.get(0).getId(), type));
}
}
public void saveMetric(Metric metric) {
MetricDO metricDO = MetricConverter.convert2MetricDO(metric);
log.info("[save metric] metricDO:{}", JSONObject.toJSONString(metricDO));
metricRepository.createMetric(metricDO);
metric.setId(metricDO.getId());
}
protected void updateMetric(Metric metric) {
MetricDO metricDO = metricRepository.getMetricById(metric.getId());
metricRepository.updateMetric(MetricConverter.convert(metricDO, metric));
}
public List<MetricResp> getMetricByModelId(Long modelId) {
return convertList(getMetricDOByModelId(modelId));
}
protected List<MetricDO> getMetricDOByModelId(Long modelId) {
List<MetricDO> metricDOS = metricRepository.getAllMetricList();
return metricDOS.stream().filter(metricDO -> Objects.equals(metricDO.getModelId(), modelId))
.collect(Collectors.toList());
}
@Override
public List<MetricResp> getHighSensitiveMetric(Long modelId) {
List<MetricResp> metricResps = getMetricByModelId(modelId);
if (CollectionUtils.isEmpty(metricResps)) {
return metricResps;
}
return metricResps.stream()
.filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
.collect(Collectors.toList());
}
@Override
public List<MetricResp> getAllHighSensitiveMetric() {
List<MetricResp> metricResps = Lists.newArrayList();
List<MetricDO> metricDOS = metricRepository.getAllMetricList();
if (CollectionUtils.isEmpty(metricDOS)) {
return metricResps;
}
return convertList(metricDOS.stream()
.filter(metricResp -> SensitiveLevelEnum.HIGH.getCode().equals(metricResp.getSensitiveLevel()))
.collect(Collectors.toList()));
}
@Override
public void deleteMetric(Long id) {
MetricDO metricDO = metricRepository.getMetricById(id);
if (metricDO == null) {
throw new RuntimeException(String.format("the metric %s not exist", id));
}
metricRepository.deleteMetric(id);
//动态更新字典
String type = DictWordType.METRIC.getType();
applicationEventPublisher.publishEvent(
new DataDeleteEvent(this, metricDO.getName(), metricDO.getModelId(), metricDO.getId(), type));
}
@Override
public List<String> mockAlias(MetricReq metricReq, String mockType, User user) {
@@ -289,7 +254,7 @@ public class MetricServiceImpl implements MetricService {
@Override
public Set<String> getMetricTags() {
List<MetricResp> metricResps = getMetrics();
List<MetricResp> metricResps = getMetrics(new MetaFilter());
if (CollectionUtils.isEmpty(metricResps)) {
return new HashSet<>();
}
@@ -311,17 +276,7 @@ public class MetricServiceImpl implements MetricService {
return modelResp.getDrillDownDimensions();
}
private void saveMetricBatch(List<Metric> metrics, User user) {
if (CollectionUtils.isEmpty(metrics)) {
return;
}
List<MetricDO> metricDOS = metrics.stream().peek(metric -> metric.createdBy(user.getName()))
.map(MetricConverter::convert2MetricDO).collect(Collectors.toList());
log.info("[save metric] metrics:{}", JSONObject.toJSONString(metricDOS));
metricRepository.createMetricBatch(metricDOS);
}
private void preCheckMetric(MetricReq metricReq) {
private void checkParam(MetricReq metricReq) {
MetricTypeParams typeParams = metricReq.getTypeParams();
List<Measure> measures = typeParams.getMeasures();
if (CollectionUtils.isEmpty(measures)) {
@@ -335,20 +290,40 @@ public class MetricServiceImpl implements MetricService {
}
}
private void check(List<MetricReq> exprMetricReqList) {
Long modelId = exprMetricReqList.get(0).getModelId();
List<MetricResp> metricResps = getMetrics(modelId);
for (MetricReq exprMetricReq : exprMetricReqList) {
for (MetricResp metricResp : metricResps) {
if (metricResp.getName().equalsIgnoreCase(exprMetricReq.getName())) {
throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getName()));
private void checkExist(List<MetricReq> metricReqs) {
Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricResps = getMetricInSameDomain(modelId);
Map<String, MetricResp> bizNameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
for (MetricReq metricReq : metricReqs) {
if (NameCheckUtils.containsSpecialCharacters(metricReq.getName())) {
throw new InvalidArgumentException("名称包含特殊字符, 请修改");
}
if (metricResp.getBizName().equalsIgnoreCase(exprMetricReq.getBizName())) {
throw new RuntimeException(String.format("存在相同的指标名:%s", metricResp.getBizName()));
}
preCheckMetric(exprMetricReq);
if (bizNameMap.containsKey(metricReq.getBizName())) {
MetricResp metricResp = bizNameMap.get(metricReq.getBizName());
if (!metricResp.getId().equals(metricReq.getId())) {
throw new RuntimeException(String.format("该主题域下存在相同的指标字段名:%s 创建人:%s",
metricReq.getBizName(), metricResp.getCreatedBy()));
}
}
if (nameMap.containsKey(metricReq.getName())) {
MetricResp metricResp = nameMap.get(metricReq.getName());
if (!metricResp.getId().equals(metricReq.getId())) {
throw new RuntimeException(String.format("该主题域下存在相同的指标名:%s 创建人:%s",
metricReq.getName(), metricResp.getCreatedBy()));
}
}
}
}
private List<MetricResp> getMetricInSameDomain(Long modelId) {
ModelResp modelResp = modelService.getModel(modelId);
Long domainId = modelResp.getDomainId();
List<ModelResp> modelResps = modelService.getModelByDomainIds(Lists.newArrayList(domainId));
List<Long> modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList());
return getMetrics(new MetaFilter(modelIds));
}
private List<MetricResp> convertList(List<MetricDO> metricDOS) {
@@ -362,5 +337,21 @@ public class MetricServiceImpl implements MetricService {
return metricResps;
}
private void sendEventBatch(List<MetricDO> metricDOS, EventType eventType) {
List<DataItem> dataItems = metricDOS.stream().map(this::getDataItem)
.collect(Collectors.toList());
eventPublisher.publishEvent(new DataEvent(this, dataItems, eventType));
}
private void sendEvent(DataItem dataItem, EventType eventType) {
eventPublisher.publishEvent(new DataEvent(this,
Lists.newArrayList(dataItem), eventType));
}
private DataItem getDataItem(MetricDO metricDO) {
return DataItem.builder().id(metricDO.getId()).name(metricDO.getName())
.bizName(metricDO.getBizName())
.modelId(metricDO.getModelId()).type(TypeEnums.METRIC).build();
}
}

61
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.semantic.model.application;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
@@ -28,7 +28,9 @@ import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO;
import com.tencent.supersonic.semantic.model.domain.pojo.Model;
import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
import com.tencent.supersonic.semantic.model.domain.repository.ModelRepository;
import com.tencent.supersonic.semantic.model.domain.utils.ModelConvert;
import java.util.ArrayList;
@@ -56,7 +58,6 @@ public class ModelServiceImpl implements ModelService {
private final DomainService domainService;
private final UserService userService;
private final DatabaseService databaseService;
private final Catalog catalog;
public ModelServiceImpl(ModelRepository modelRepository, @Lazy MetricService metricService,
@@ -76,30 +77,33 @@ public class ModelServiceImpl implements ModelService {
@Override
public void createModel(ModelReq modelReq, User user) {
log.info("[create model] req : {}", JSONObject.toJSONString(modelReq));
Model model = ModelConvert.convert(modelReq);
log.info("[create model] object:{}", JSONObject.toJSONString(modelReq));
saveModel(model, user);
modelReq.createdBy(user.getName());
ModelDO modelDO = ModelConvert.convert(modelReq);
modelRepository.createModel(modelDO);
}
@Override
public void updateModel(ModelReq modelReq, User user) {
ModelDO modelDO = getModelDO(modelReq.getId());
modelDO.setUpdatedAt(new Date());
modelDO.setUpdatedBy(user.getName());
modelReq.updatedBy(user.getName());
BeanMapper.mapper(modelReq, modelDO);
modelDO.setAdmin(String.join(",", modelReq.getAdmins()));
modelDO.setAdminOrg(String.join(",", modelReq.getAdminOrgs()));
modelDO.setViewer(String.join(",", modelReq.getViewers()));
modelDO.setViewOrg(String.join(",", modelReq.getViewOrgs()));
if (modelReq.getEntity() != null) {
modelDO.setEntity(JsonUtil.toString(modelReq.getEntity()));
}
if (modelReq.getDrillDownDimensions() != null) {
modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions()));
}
modelRepository.updateModel(modelDO);
}
@Override
public void deleteModel(Long id) {
public void deleteModel(Long id, User user) {
checkDelete(id);
modelRepository.deleteModel(id);
ModelDO modelDO = getModelDO(id);
modelDO.setStatus(StatusEnum.DELETED.getCode());
modelDO.setUpdatedAt(new Date());
modelDO.setUpdatedBy(user.getName());
modelRepository.updateModel(modelDO);
}
@Override
@@ -178,8 +182,9 @@ public class ModelServiceImpl implements ModelService {
}
private void checkDelete(Long id) {
List<MetricResp> metricResps = metricService.getMetrics(id);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(id);
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(id));
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
List<DatasourceResp> datasourceResps = datasourceService.getDatasourceList(id);
if (!CollectionUtils.isEmpty(metricResps) || !CollectionUtils.isEmpty(datasourceResps)
|| !CollectionUtils.isEmpty(dimensionResps)) {
@@ -187,12 +192,6 @@ public class ModelServiceImpl implements ModelService {
}
}
private void saveModel(Model model, User user) {
ModelDO modelDO = ModelConvert.convert(model, user);
modelRepository.createModel(modelDO);
model.setId(modelDO.getId());
}
private List<ModelResp> convertList(List<ModelDO> modelDOS) {
List<ModelResp> modelResps = Lists.newArrayList();
if (CollectionUtils.isEmpty(modelDOS)) {
@@ -209,9 +208,9 @@ public class ModelServiceImpl implements ModelService {
if (CollectionUtils.isEmpty(modelResps)) {
return modelResps;
}
Map<Long, List<MetricResp>> metricMap = metricService.getMetrics().stream()
Map<Long, List<MetricResp>> metricMap = metricService.getMetrics(new MetricFilter()).stream()
.collect(Collectors.groupingBy(MetricResp::getModelId));
Map<Long, List<DimensionResp>> dimensionMap = dimensionService.getDimensions().stream()
Map<Long, List<DimensionResp>> dimensionMap = dimensionService.getDimensions(new DimensionFilter()).stream()
.collect(Collectors.groupingBy(DimensionResp::getModelId));
modelResps.forEach(modelResp -> {
modelResp.setDimensionCnt(dimensionMap.getOrDefault(modelResp.getId(), Lists.newArrayList()).size());
@@ -280,15 +279,17 @@ public class ModelServiceImpl implements ModelService {
if (CollectionUtils.isEmpty(modelIds)) {
modelIds = generateModelIdsReq(modelSchemaFilterReq);
}
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetricsByModelIds(modelIds)
MetaFilter metaFilter = new MetaFilter(modelIds);
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetrics(metaFilter)
.stream().collect(Collectors.groupingBy(MetricResp::getModelId));
Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensionsByModelIds(modelIds)
Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensions(metaFilter)
.stream().collect(Collectors.groupingBy(DimensionResp::getModelId));
Map<Long, List<MeasureResp>> measureRespsMap = datasourceService.getMeasureListOfModel(modelIds)
.stream().collect(Collectors.groupingBy(MeasureResp::getModelId));
for (Long modelId : modelIds) {
ModelResp modelResp = getModelMap().get(modelId);
if (modelResp == null) {
if (modelResp == null || !StatusEnum.ONLINE.getCode().equals(modelResp.getStatus())) {
continue;
}
List<MeasureResp> measureResps = measureRespsMap.getOrDefault(modelId, Lists.newArrayList());
@@ -318,7 +319,7 @@ public class ModelServiceImpl implements ModelService {
private List<MetricSchemaResp> generateMetricSchema(Long modelId, ModelResp modelResp) {
List<MetricSchemaResp> metricSchemaDescList = new ArrayList<>();
List<MetricResp> metricResps = metricService.getMetrics(modelId);
List<MetricResp> metricResps = metricService.getMetrics(new MetaFilter(Lists.newArrayList(modelId)));
List<MeasureResp> measureResps = datasourceService.getMeasureListOfModel(modelId);
metricResps.stream().forEach(metricResp ->
metricSchemaDescList.add(convert(metricResp, metricResps, measureResps, modelResp)));
@@ -326,7 +327,7 @@ public class ModelServiceImpl implements ModelService {
}
private List<DimSchemaResp> generateDimSchema(Long modelId) {
List<DimensionResp> dimDescList = dimensionService.getDimensions(modelId);
List<DimensionResp> dimDescList = dimensionService.getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
return dimDescList.stream().map(this::convert).collect(Collectors.toList());
}

8
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ViewInfoServiceImpl.java
View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaRelaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.model.domain.dataobject.ViewInfoDO;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.model.domain.repository.ViewInfoRepository;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
@@ -48,8 +49,11 @@ public class ViewInfoServiceImpl {
for (DatasourceResp datasourceResp : datasourceResps) {
ModelSchemaRelaResp domainSchemaRelaResp = new ModelSchemaRelaResp();
Long datasourceId = datasourceResp.getId();
List<MetricResp> metricResps = metricService.getMetrics(modelId, datasourceId);
List<DimensionResp> dimensionResps = dimensionService.getDimensionsByDatasource(datasourceId);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
metaFilter.setDatasourceId(datasourceId);
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
domainSchemaRelaResp.setDatasource(datasourceResp);
domainSchemaRelaResp.setDimensions(dimensionResps);
domainSchemaRelaResp.setMetrics(metricResps);

7
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java
View File

@@ -12,15 +12,16 @@ import com.tencent.supersonic.semantic.api.model.response.DatasourceRelaResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.model.domain.pojo.Datasource;
import java.util.List;
import java.util.Map;
import java.util.Set;
public interface DatasourceService {
DatasourceResp createDatasource(DatasourceReq datasourceReq, User user) throws Exception;
Datasource createDatasource(DatasourceReq datasourceReq, User user) throws Exception;
DatasourceResp updateDatasource(DatasourceReq datasourceReq, User user) throws Exception;
Datasource updateDatasource(DatasourceReq datasourceReq, User user) throws Exception;
List<DatasourceResp> getDatasourceListNoMeasurePrefix(Long modelId);
@@ -32,7 +33,7 @@ public interface DatasourceService {
Map<Long, DatasourceResp> getDatasourceMap();
void deleteDatasource(Long id) throws Exception;
void deleteDatasource(Long id, User user);
DatasourceRelaResp createOrUpdateDatasourceRela(DatasourceRelaReq datasourceRelaReq, User user);

19
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java
View File

@@ -4,39 +4,32 @@ import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import java.util.List;
public interface DimensionService {
List<DimensionResp> getDimensions(List<Long> ids);
List<DimensionResp> getDimensions(Long domainId);
List<DimensionResp> getDimensions();
List<DimensionResp> getDimensions(MetaFilter metaFilter);
DimensionResp getDimension(Long id);
DimensionResp getDimension(String bizName, Long modelId);
List<DimensionResp> getDimensionsByModelIds(List<Long> modelIds);
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
void createDimension(DimensionReq dimensionReq, User user) throws Exception;
void createDimensionBatch(List<DimensionReq> dimensionReqs, User user) throws Exception;
List<DimensionResp> getDimensionsByDatasource(Long datasourceId);
void updateDimension(DimensionReq dimensionReq, User user) throws Exception;
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq);
List<DimensionResp> getHighSensitiveDimension(Long domainId);
List<DimensionResp> getAllHighSensitiveDimension();
void deleteDimension(Long id) throws Exception;
void deleteDimension(Long id, User user);
List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user);

29
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java
View File

@@ -3,41 +3,32 @@ package com.tencent.supersonic.semantic.model.domain;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.MetricReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import java.util.List;
import java.util.Set;
public interface MetricService {
List<MetricResp> getMetrics(List<Long> ids);
List<MetricResp> getMetrics(Long modelId);
List<MetricResp> getMetrics();
List<MetricResp> getMetrics(Long modelId, Long datasourceId);
void creatExprMetric(MetricReq metricReq, User user) throws Exception;
void createMetric(MetricReq metricReq, User user) throws Exception;
void createMetricBatch(List<MetricReq> metricReqs, User user) throws Exception;
List<MetricResp> getMetricsByModelIds(List<Long> modelIds);
void updateExprMetric(MetricReq metricReq, User user) throws Exception;
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
void deleteMetric(Long id, User user) throws Exception;
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
List<MetricResp> getMetrics(MetaFilter metaFilter);
MetricResp getMetric(Long modelId, String bizName);
List<MetricResp> getHighSensitiveMetric(Long modelId);
void updateExprMetric(MetricReq metricReq, User user) throws Exception;
List<MetricResp> getAllHighSensitiveMetric();
void deleteMetric(Long id) throws Exception;
List<String> mockAlias(MetricReq metricReq, String mockType, User user);
Set<String> getMetricTags();

3
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/ModelService.java
View File

@@ -7,7 +7,6 @@ import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import java.util.List;
import java.util.Map;
@@ -29,7 +28,7 @@ public interface ModelService {
void createModel(ModelReq modelReq, User user);
void deleteModel(Long model);
void deleteModel(Long id, User user);
Map<Long, ModelResp> getModelMap();

63
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDO.java
View File

@@ -33,6 +33,11 @@ public class DatasourceDO {
*/
private Long databaseId;
/**
*
*/
private Integer status;
/**
* 创建时间
*/
@@ -58,10 +63,13 @@ public class DatasourceDO {
*/
private String datasourceDetail;
/**
* 上游依赖标识
*/
private String depends;
/**
*
* @return id
*/
public Long getId() {
@@ -69,6 +77,7 @@ public class DatasourceDO {
}
/**
*
* @param id
*/
public void setId(Long id) {
@@ -77,7 +86,6 @@ public class DatasourceDO {
/**
* 主题域ID
*
* @return model_id 主题域ID
*/
public Long getModelId() {
@@ -86,7 +94,6 @@ public class DatasourceDO {
/**
* 主题域ID
*
* @param modelId 主题域ID
*/
public void setModelId(Long modelId) {
@@ -95,7 +102,6 @@ public class DatasourceDO {
/**
* 数据源名称
*
* @return name 数据源名称
*/
public String getName() {
@@ -104,7 +110,6 @@ public class DatasourceDO {
/**
* 数据源名称
*
* @param name 数据源名称
*/
public void setName(String name) {
@@ -113,7 +118,6 @@ public class DatasourceDO {
/**
* 内部名称
*
* @return biz_name 内部名称
*/
public String getBizName() {
@@ -122,7 +126,6 @@ public class DatasourceDO {
/**
* 内部名称
*
* @param bizName 内部名称
*/
public void setBizName(String bizName) {
@@ -131,7 +134,6 @@ public class DatasourceDO {
/**
* 数据源描述
*
* @return description 数据源描述
*/
public String getDescription() {
@@ -140,7 +142,6 @@ public class DatasourceDO {
/**
* 数据源描述
*
* @param description 数据源描述
*/
public void setDescription(String description) {
@@ -149,7 +150,6 @@ public class DatasourceDO {
/**
* 数据库实例ID
*
* @return database_id 数据库实例ID
*/
public Long getDatabaseId() {
@@ -158,7 +158,6 @@ public class DatasourceDO {
/**
* 数据库实例ID
*
* @param databaseId 数据库实例ID
*/
public void setDatabaseId(Long databaseId) {
@@ -166,8 +165,23 @@ public class DatasourceDO {
}
/**
* 创建时间
*
* @return status
*/
public Integer getStatus() {
return status;
}
/**
*
* @param status
*/
public void setStatus(Integer status) {
this.status = status;
}
/**
* 创建时间
* @return created_at 创建时间
*/
public Date getCreatedAt() {
@@ -176,7 +190,6 @@ public class DatasourceDO {
/**
* 创建时间
*
* @param createdAt 创建时间
*/
public void setCreatedAt(Date createdAt) {
@@ -185,7 +198,6 @@ public class DatasourceDO {
/**
* 创建人
*
* @return created_by 创建人
*/
public String getCreatedBy() {
@@ -194,7 +206,6 @@ public class DatasourceDO {
/**
* 创建人
*
* @param createdBy 创建人
*/
public void setCreatedBy(String createdBy) {
@@ -203,7 +214,6 @@ public class DatasourceDO {
/**
* 更新时间
*
* @return updated_at 更新时间
*/
public Date getUpdatedAt() {
@@ -212,7 +222,6 @@ public class DatasourceDO {
/**
* 更新时间
*
* @param updatedAt 更新时间
*/
public void setUpdatedAt(Date updatedAt) {
@@ -221,7 +230,6 @@ public class DatasourceDO {
/**
* 更新人
*
* @return updated_by 更新人
*/
public String getUpdatedBy() {
@@ -230,7 +238,6 @@ public class DatasourceDO {
/**
* 更新人
*
* @param updatedBy 更新人
*/
public void setUpdatedBy(String updatedBy) {
@@ -239,7 +246,6 @@ public class DatasourceDO {
/**
* 数据源配置
*
* @return datasource_detail 数据源配置
*/
public String getDatasourceDetail() {
@@ -248,18 +254,25 @@ public class DatasourceDO {
/**
* 数据源配置
*
* @param datasourceDetail 数据源配置
*/
public void setDatasourceDetail(String datasourceDetail) {
this.datasourceDetail = datasourceDetail == null ? null : datasourceDetail.trim();
}
public void setDepends(String depends) {
this.depends = depends;
}
/**
* 上游依赖标识
* @return depends 上游依赖标识
*/
public String getDepends() {
return depends;
}
/**
* 上游依赖标识
* @param depends 上游依赖标识
*/
public void setDepends(String depends) {
this.depends = depends == null ? null : depends.trim();
}
}

139
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/DatasourceDOExample.java
View File

@@ -31,6 +31,7 @@ public class DatasourceDOExample {
protected Integer limitEnd;
/**
*
* @mbg.generated
*/
public DatasourceDOExample() {
@@ -38,6 +39,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public void setOrderByClause(String orderByClause) {
@@ -45,6 +47,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public String getOrderByClause() {
@@ -52,6 +55,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public void setDistinct(boolean distinct) {
@@ -59,6 +63,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public boolean isDistinct() {
@@ -66,6 +71,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public List<Criteria> getOredCriteria() {
@@ -73,6 +79,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public void or(Criteria criteria) {
@@ -80,6 +87,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public Criteria or() {
@@ -89,6 +97,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public Criteria createCriteria() {
@@ -100,6 +109,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
protected Criteria createCriteriaInternal() {
@@ -108,6 +118,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public void clear() {
@@ -117,6 +128,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public void setLimitStart(Integer limitStart) {
@@ -124,6 +136,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public Integer getLimitStart() {
@@ -131,6 +144,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public void setLimitEnd(Integer limitEnd) {
@@ -138,6 +152,7 @@ public class DatasourceDOExample {
}
/**
*
* @mbg.generated
*/
public Integer getLimitEnd() {
@@ -578,6 +593,66 @@ public class DatasourceDOExample {
return (Criteria) this;
}
public Criteria andStatusIsNull() {
addCriterion("status is null");
return (Criteria) this;
}
public Criteria andStatusIsNotNull() {
addCriterion("status is not null");
return (Criteria) this;
}
public Criteria andStatusEqualTo(Integer value) {
addCriterion("status =", value, "status");
return (Criteria) this;
}
public Criteria andStatusNotEqualTo(Integer value) {
addCriterion("status <>", value, "status");
return (Criteria) this;
}
public Criteria andStatusGreaterThan(Integer value) {
addCriterion("status >", value, "status");
return (Criteria) this;
}
public Criteria andStatusGreaterThanOrEqualTo(Integer value) {
addCriterion("status >=", value, "status");
return (Criteria) this;
}
public Criteria andStatusLessThan(Integer value) {
addCriterion("status <", value, "status");
return (Criteria) this;
}
public Criteria andStatusLessThanOrEqualTo(Integer value) {
addCriterion("status <=", value, "status");
return (Criteria) this;
}
public Criteria andStatusIn(List<Integer> values) {
addCriterion("status in", values, "status");
return (Criteria) this;
}
public Criteria andStatusNotIn(List<Integer> values) {
addCriterion("status not in", values, "status");
return (Criteria) this;
}
public Criteria andStatusBetween(Integer value1, Integer value2) {
addCriterion("status between", value1, value2, "status");
return (Criteria) this;
}
public Criteria andStatusNotBetween(Integer value1, Integer value2) {
addCriterion("status not between", value1, value2, "status");
return (Criteria) this;
}
public Criteria andCreatedAtIsNull() {
addCriterion("created_at is null");
return (Criteria) this;
@@ -869,6 +944,38 @@ public class DatasourceDOExample {
private String typeHandler;
public String getCondition() {
return condition;
}
public Object getValue() {
return value;
}
public Object getSecondValue() {
return secondValue;
}
public boolean isNoValue() {
return noValue;
}
public boolean isSingleValue() {
return singleValue;
}
public boolean isBetweenValue() {
return betweenValue;
}
public boolean isListValue() {
return listValue;
}
public String getTypeHandler() {
return typeHandler;
}
protected Criterion(String condition) {
super();
this.condition = condition;
@@ -904,37 +1011,5 @@ public class DatasourceDOExample {
protected Criterion(String condition, Object value, Object secondValue) {
this(condition, value, secondValue, null);
}
public String getCondition() {
return condition;
}
public Object getValue() {
return value;
}
public Object getSecondValue() {
return secondValue;
}
public boolean isNoValue() {
return noValue;
}
public boolean isSingleValue() {
return singleValue;
}
public boolean isBetweenValue() {
return betweenValue;
}
public boolean isListValue() {
return listValue;
}
public String getTypeHandler() {
return typeHandler;
}
}
}

42
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDO.java
View File

@@ -28,6 +28,11 @@ public class ModelDO {
*/
private String alias;
/**
*
*/
private String description;
/**
*
*/
@@ -78,6 +83,11 @@ public class ModelDO {
*/
private String drillDownDimensions;
/**
*
*/
private Integer status;
/**
*
*/
@@ -163,6 +173,22 @@ public class ModelDO {
this.alias = alias == null ? null : alias.trim();
}
/**
*
* @return description
*/
public String getDescription() {
return description;
}
/**
*
* @param description
*/
public void setDescription(String description) {
this.description = description == null ? null : description.trim();
}
/**
*
* @return viewer
@@ -323,6 +349,22 @@ public class ModelDO {
this.drillDownDimensions = drillDownDimensions == null ? null : drillDownDimensions.trim();
}
/**
*
* @return status
*/
public Integer getStatus() {
return status;
}
/**
*
* @param status
*/
public void setStatus(Integer status) {
this.status = status;
}
/**
*
* @return entity

130
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/dataobject/ModelDOExample.java
View File

@@ -533,6 +533,76 @@ public class ModelDOExample {
return (Criteria) this;
}
public Criteria andDescriptionIsNull() {
addCriterion("description is null");
return (Criteria) this;
}
public Criteria andDescriptionIsNotNull() {
addCriterion("description is not null");
return (Criteria) this;
}
public Criteria andDescriptionEqualTo(String value) {
addCriterion("description =", value, "description");
return (Criteria) this;
}
public Criteria andDescriptionNotEqualTo(String value) {
addCriterion("description <>", value, "description");
return (Criteria) this;
}
public Criteria andDescriptionGreaterThan(String value) {
addCriterion("description >", value, "description");
return (Criteria) this;
}
public Criteria andDescriptionGreaterThanOrEqualTo(String value) {
addCriterion("description >=", value, "description");
return (Criteria) this;
}
public Criteria andDescriptionLessThan(String value) {
addCriterion("description <", value, "description");
return (Criteria) this;
}
public Criteria andDescriptionLessThanOrEqualTo(String value) {
addCriterion("description <=", value, "description");
return (Criteria) this;
}
public Criteria andDescriptionLike(String value) {
addCriterion("description like", value, "description");
return (Criteria) this;
}
public Criteria andDescriptionNotLike(String value) {
addCriterion("description not like", value, "description");
return (Criteria) this;
}
public Criteria andDescriptionIn(List<String> values) {
addCriterion("description in", values, "description");
return (Criteria) this;
}
public Criteria andDescriptionNotIn(List<String> values) {
addCriterion("description not in", values, "description");
return (Criteria) this;
}
public Criteria andDescriptionBetween(String value1, String value2) {
addCriterion("description between", value1, value2, "description");
return (Criteria) this;
}
public Criteria andDescriptionNotBetween(String value1, String value2) {
addCriterion("description not between", value1, value2, "description");
return (Criteria) this;
}
public Criteria andViewerIsNull() {
addCriterion("viewer is null");
return (Criteria) this;
@@ -1202,6 +1272,66 @@ public class ModelDOExample {
addCriterion("drill_down_dimensions not between", value1, value2, "drillDownDimensions");
return (Criteria) this;
}
public Criteria andStatusIsNull() {
addCriterion("status is null");
return (Criteria) this;
}
public Criteria andStatusIsNotNull() {
addCriterion("status is not null");
return (Criteria) this;
}
public Criteria andStatusEqualTo(Integer value) {
addCriterion("status =", value, "status");
return (Criteria) this;
}
public Criteria andStatusNotEqualTo(Integer value) {
addCriterion("status <>", value, "status");
return (Criteria) this;
}
public Criteria andStatusGreaterThan(Integer value) {
addCriterion("status >", value, "status");
return (Criteria) this;
}
public Criteria andStatusGreaterThanOrEqualTo(Integer value) {
addCriterion("status >=", value, "status");
return (Criteria) this;
}
public Criteria andStatusLessThan(Integer value) {
addCriterion("status <", value, "status");
return (Criteria) this;
}
public Criteria andStatusLessThanOrEqualTo(Integer value) {
addCriterion("status <=", value, "status");
return (Criteria) this;
}
public Criteria andStatusIn(List<Integer> values) {
addCriterion("status in", values, "status");
return (Criteria) this;
}
public Criteria andStatusNotIn(List<Integer> values) {
addCriterion("status not in", values, "status");
return (Criteria) this;
}
public Criteria andStatusBetween(Integer value1, Integer value2) {
addCriterion("status between", value1, value2, "status");
return (Criteria) this;
}
public Criteria andStatusNotBetween(Integer value1, Integer value2) {
addCriterion("status not between", value1, value2, "status");
return (Criteria) this;
}
}
/**

56
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/listener/MetaEmbeddingListener.java Normal file
View File

@@ -0,0 +1,56 @@
package com.tencent.supersonic.semantic.model.domain.listener;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.embedding.EmbeddingQuery;
import com.tencent.supersonic.common.util.embedding.EmbeddingUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Component
@Slf4j
public class MetaEmbeddingListener implements ApplicationListener<DataEvent> {
public static final String COLLECTION_NAME = "meta_collection";
@Autowired
private EmbeddingUtils embeddingUtils;
@Override
public void onApplicationEvent(DataEvent event) {
if (CollectionUtils.isEmpty(event.getDataItems())) {
return;
}
List<EmbeddingQuery> embeddingQueries = event.getDataItems()
.stream().filter(dataItem -> dataItem.getType().equals(TypeEnums.METRIC)).map(dataItem -> {
EmbeddingQuery embeddingQuery = new EmbeddingQuery();
embeddingQuery.setQueryId(dataItem.getId().toString());
embeddingQuery.setQuery(dataItem.getName());
Map meta = JSONObject.parseObject(JSONObject.toJSONString(dataItem), Map.class);
embeddingQuery.setMetadata(meta);
embeddingQuery.setQueryEmbedding(null);
return embeddingQuery;
}).collect(Collectors.toList());
if (CollectionUtils.isEmpty(embeddingQueries)) {
return;
}
embeddingUtils.addCollection(COLLECTION_NAME);
if (event.getEventType().equals(EventType.ADD)) {
embeddingUtils.addQuery(COLLECTION_NAME, embeddingQueries);
} else if (event.getEventType().equals(EventType.DELETE)) {
embeddingUtils.deleteQuery(COLLECTION_NAME, embeddingQueries);
} else if (event.getEventType().equals(EventType.UPDATE)) {
embeddingUtils.deleteQuery(COLLECTION_NAME, embeddingQueries);
embeddingUtils.addQuery(COLLECTION_NAME, embeddingQueries);
}
}
}

9
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java
View File

@@ -1,10 +1,12 @@
package com.tencent.supersonic.semantic.model.domain.pojo;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@NoArgsConstructor
public class MetaFilter {
private String id;
@@ -23,4 +25,11 @@ public class MetaFilter {
private String key;
private List<Long> ids;
private Long datasourceId;
public MetaFilter(List<Long> modelIds) {
this.modelIds = modelIds;
}
}

9
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/Metric.java
View File

@@ -5,8 +5,6 @@ import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.List;
@Data
@@ -30,11 +28,4 @@ public class Metric extends SchemaItem {
private RelateDimension relateDimension;
public String getTag() {
if (CollectionUtils.isEmpty(tags)) {
return "";
}
return StringUtils.join(tags, ",");
}
}

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DatasourceRepository.java
View File

@@ -19,8 +19,6 @@ public interface DatasourceRepository {
DatasourceDO getDatasourceById(Long id);
void deleteDatasource(Long id);
void createDatasourceRela(DatasourceRelaDO datasourceRelaDO);
void updateDatasourceRela(DatasourceRelaDO datasourceRelaDO);

15
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java
View File

@@ -13,22 +13,9 @@ public interface DimensionRepository {
void updateDimension(DimensionDO dimensionDO);
List<DimensionDO> getDimensionListOfDatasource(Long datasourceId);
List<DimensionDO> getDimensionListOfmodel(Long domainId);
List<DimensionDO> getDimensionListOfmodelIds(List<Long> modelIds);
List<DimensionDO> getDimensionList();
List<DimensionDO> getDimensionListByIds(List<Long> ids);
void batchUpdateStatus(List<DimensionDO> dimensionDOS);
DimensionDO getDimensionById(Long id);
List<DimensionDO> getAllDimensionList();
List<DimensionDO> getDimension(DimensionFilter dimensionFilter);
void deleteDimension(Long id);
}

13
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.semantic.model.domain.repository;
import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO;
import java.util.List;
@@ -15,19 +14,9 @@ public interface MetricRepository {
void updateMetric(MetricDO metricDO);
List<MetricDO> getMetricList(Long domainId);
List<MetricDO> getMetricList(List<Long> modelIds);
List<MetricDO> getMetricList();
List<MetricDO> getMetricListByIds(List<Long> ids);
void batchUpdateStatus(List<MetricDO> metricDOS);
MetricDO getMetricById(Long id);
List<MetricDO> getAllMetricList();
List<MetricDO> getMetric(MetricFilter metricFilter);
void deleteMetric(Long id);
}

6
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java
View File

@@ -35,8 +35,8 @@ public class DatasourceConverter {
public static Datasource convert(DatasourceReq datasourceReq) {
Datasource datasource = new Datasource();
DatasourceDetail datasourceDetail = new DatasourceDetail();
BeanUtils.copyProperties(datasourceReq, datasource);
BeanUtils.copyProperties(datasourceReq, datasourceDetail);
BeanMapper.mapper(datasourceReq, datasource);
BeanMapper.mapper(datasourceReq, datasourceDetail);
List<Measure> measures = datasourceDetail.getMeasures();
for (Measure measure : measures) {
if (StringUtils.isBlank(measure.getExpr())) {
@@ -71,7 +71,7 @@ public class DatasourceConverter {
public static DatasourceDO convert(Datasource datasource, User user) {
DatasourceDO datasourceDO = new DatasourceDO();
BeanUtils.copyProperties(datasource, datasourceDO);
BeanMapper.mapper(datasource, datasourceDO);
datasourceDO.setDatasourceDetail(JSONObject.toJSONString(datasource.getDatasourceDetail()));
datasourceDO.setUpdatedBy(user.getName());
datasourceDO.setUpdatedAt(new Date());

39
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.model.domain.utils;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
@@ -23,38 +24,38 @@ import org.springframework.util.CollectionUtils;
public class DimensionConverter {
public static Dimension convert(DimensionReq dimensionReq) {
Dimension dimension = new Dimension();
BeanUtils.copyProperties(dimensionReq, dimension);
return dimension;
public static DimensionDO convert(DimensionDO dimensionDO, DimensionReq dimensionReq) {
BeanMapper.mapper(dimensionReq, dimensionDO);
if (dimensionReq.getDefaultValues() != null) {
dimensionDO.setDefaultValues(JSONObject.toJSONString(dimensionReq.getDefaultValues()));
}
public static DimensionDO convert(DimensionDO dimensionDO, Dimension dimension) {
BeanMapper.mapper(dimension, dimensionDO);
dimensionDO.setDefaultValues(JSONObject.toJSONString(dimension.getDefaultValues()));
if (!CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimension.getDimValueMaps()));
if (!CollectionUtils.isEmpty(dimensionReq.getDimValueMaps())) {
dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimensionReq.getDimValueMaps()));
} else {
dimensionDO.setDimValueMaps(JSONObject.toJSONString(new ArrayList<>()));
}
if (Objects.nonNull(dimension.getDataType())) {
dimensionDO.setDataType(dimension.getDataType().getType());
if (Objects.nonNull(dimensionReq.getDataType())) {
dimensionDO.setDataType(dimensionReq.getDataType().getType());
}
return dimensionDO;
}
public static DimensionDO convert2DimensionDO(Dimension dimension) {
public static DimensionDO convert2DimensionDO(DimensionReq dimensionReq) {
DimensionDO dimensionDO = new DimensionDO();
BeanUtils.copyProperties(dimension, dimensionDO);
dimensionDO.setDefaultValues(JSONObject.toJSONString(dimension.getDefaultValues()));
dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimension.getDimValueMaps()));
if (Objects.nonNull(dimension.getDataType())) {
dimensionDO.setDataType(dimension.getDataType().getType());
BeanMapper.mapper(dimensionReq, dimensionDO);
if (dimensionReq.getDefaultValues() != null) {
dimensionDO.setDefaultValues(JSONObject.toJSONString(dimensionReq.getDefaultValues()));
}
if (dimensionReq.getDimValueMaps() != null) {
dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimensionReq.getDimValueMaps()));
}
if (Objects.nonNull(dimensionReq.getDataType())) {
dimensionDO.setDataType(dimensionReq.getDataType().getType());
}
dimensionDO.setStatus(StatusEnum.ONLINE.getCode());
return dimensionDO;
}
public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO,
Map<Long, String> fullPathMap,
Map<Long, DatasourceResp> datasourceRespMap) {

50
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/MetricConverter.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.model.domain.utils;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
@@ -24,24 +25,28 @@ import org.springframework.beans.BeanUtils;
public class MetricConverter {
public static Metric convert(MetricReq metricReq) {
Metric metric = new Metric();
BeanUtils.copyProperties(metricReq, metric);
metric.setType(metricReq.getMetricType().name());
metric.setTypeParams(metricReq.getTypeParams());
return metric;
public static MetricDO convert2MetricDO(MetricReq metricReq) {
MetricDO metricDO = new MetricDO();
BeanMapper.mapper(metricReq, metricDO);
metricDO.setType(metricReq.getMetricType().name());
metricDO.setTypeParams(JSONObject.toJSONString(metricReq.getTypeParams()));
metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat()));
metricDO.setTags(metricReq.getTag());
metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension()));
metricDO.setStatus(StatusEnum.ONLINE.getCode());
return metricDO;
}
public static MetricDO convert(MetricDO metricDO, Metric metric) {
BeanMapper.mapper(metric, metricDO);
metricDO.setTypeParams(JSONObject.toJSONString(metric.getTypeParams()));
if (metric.getDataFormat() != null) {
metricDO.setDataFormat(JSONObject.toJSONString(metric.getDataFormat()));
public static MetricDO convert(MetricDO metricDO, MetricReq metricReq) {
BeanMapper.mapper(metricReq, metricDO);
metricDO.setTypeParams(JSONObject.toJSONString(metricReq.getTypeParams()));
if (metricReq.getDataFormat() != null) {
metricDO.setDataFormat(JSONObject.toJSONString(metricReq.getDataFormat()));
}
if (metric.getRelateDimension() != null) {
metricDO.setRelateDimensions(JSONObject.toJSONString(metric.getRelateDimension()));
if (metricReq.getRelateDimension() != null) {
metricDO.setRelateDimensions(JSONObject.toJSONString(metricReq.getRelateDimension()));
}
metricDO.setTags(metric.getTag());
metricDO.setTags(metricReq.getTag());
return metricDO;
}
@@ -51,17 +56,6 @@ public class MetricConverter {
return measureYamlTpl;
}
public static MetricDO convert2MetricDO(Metric metric) {
MetricDO metricDO = new MetricDO();
BeanUtils.copyProperties(metric, metricDO);
metricDO.setTypeParams(JSONObject.toJSONString(metric.getTypeParams()));
metricDO.setDataFormat(JSONObject.toJSONString(metric.getDataFormat()));
metricDO.setTags(metric.getTag());
metricDO.setRelateDimensions(JSONObject.toJSONString(metric.getRelateDimension()));
return metricDO;
}
public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap) {
MetricResp metricResp = new MetricResp();
BeanUtils.copyProperties(metricDO, metricResp);
@@ -77,12 +71,6 @@ public class MetricConverter {
RelateDimension.class));
return metricResp;
}
public static Metric convert2Metric(MetricDO metricDO) {
Metric metric = new Metric();
BeanUtils.copyProperties(metricDO, metric);
metric.setTypeParams(JSONObject.parseObject(metricDO.getTypeParams(), MetricTypeParams.class));
return metric;
}
public static MetricYamlTpl convert2MetricYamlTpl(Metric metric) {
MetricYamlTpl metricYamlTpl = new MetricYamlTpl();

28
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/ModelConvert.java
View File

@@ -2,8 +2,8 @@ package com.tencent.supersonic.semantic.model.domain.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
import com.tencent.supersonic.semantic.api.model.pojo.Entity;
@@ -11,35 +11,19 @@ import com.tencent.supersonic.semantic.api.model.request.ModelReq;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO;
import com.tencent.supersonic.semantic.model.domain.pojo.Model;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;
public class ModelConvert {
public static Model convert(ModelReq modelReq) {
Model model = new Model();
BeanUtils.copyProperties(modelReq, model);
model.setStatus(StatusEnum.ONLINE.getCode());
return model;
}
public static ModelDO convert(Model model, User user) {
public static ModelDO convert(ModelReq modelReq) {
ModelDO modelDO = new ModelDO();
BeanUtils.copyProperties(model, modelDO);
modelDO.setCreatedBy(user.getName());
modelDO.setUpdatedBy(user.getName());
modelDO.setCreatedAt(new Date());
modelDO.setUpdatedAt(new Date());
modelDO.setAdmin(String.join(",", model.getAdmins()));
modelDO.setAdminOrg(String.join(",", model.getAdminOrgs()));
modelDO.setViewer(String.join(",", model.getViewers()));
modelDO.setViewOrg(String.join(",", model.getViewOrgs()));
modelDO.setEntity(JsonUtil.toString(model.getEntity()));
modelDO.setDrillDownDimensions(JsonUtil.toString(model.getDrillDownDimensions()));
BeanMapper.mapper(modelReq, modelDO);
modelDO.setEntity(JsonUtil.toString(modelReq.getEntity()));
modelDO.setDrillDownDimensions(JsonUtil.toString(modelReq.getDrillDownDimensions()));
modelDO.setStatus(StatusEnum.ONLINE.getCode());
return modelDO;
}

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/DimensionDOCustomMapper.java
View File

@@ -13,5 +13,7 @@ public interface DimensionDOCustomMapper {
void batchUpdate(List<DimensionDO> dimensionDOS);
void batchUpdateStatus(List<DimensionDO> dimensionDOS);
List<DimensionDO> query(DimensionFilter dimensionFilter);
}

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java
View File

@@ -13,6 +13,8 @@ public interface MetricDOCustomMapper {
void batchUpdate(List<MetricDO> metricDOS);
void batchUpdateStatus(List<MetricDO> metricDOS);
List<MetricDO> query(MetricFilter metricFilter);
}

17
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DatasourceRepositoryImpl.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.semantic.model.infrastructure.repository;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO;
import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDOExample;
import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceRelaDO;
@@ -7,9 +8,7 @@ import com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceRelaDOE
import com.tencent.supersonic.semantic.model.domain.repository.DatasourceRepository;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.DatasourceDOMapper;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.DatasourceRelaDOMapper;
import java.util.List;
import org.springframework.stereotype.Component;
@@ -35,26 +34,29 @@ public class DatasourceRepositoryImpl implements DatasourceRepository {
@Override
public void updateDatasource(DatasourceDO datasourceDO) {
datasourceMapper.updateByPrimaryKeyWithBLOBs(datasourceDO);
datasourceMapper.updateByPrimaryKeySelective(datasourceDO);
}
@Override
public List<DatasourceDO> getDatasourceList() {
DatasourceDOExample datasourceExample = new DatasourceDOExample();
datasourceExample.createCriteria().andStatusNotEqualTo(StatusEnum.DELETED.getCode());
return datasourceMapper.selectByExampleWithBLOBs(datasourceExample);
}
@Override
public List<DatasourceDO> getDatasourceList(Long modelId) {
DatasourceDOExample datasourceExample = new DatasourceDOExample();
datasourceExample.createCriteria().andModelIdEqualTo(modelId);
datasourceExample.createCriteria().andModelIdEqualTo(modelId)
.andStatusNotEqualTo(StatusEnum.DELETED.getCode());
return datasourceMapper.selectByExampleWithBLOBs(datasourceExample);
}
@Override
public List<DatasourceDO> getDatasourceByDatabase(Long databaseId) {
DatasourceDOExample datasourceExample = new DatasourceDOExample();
datasourceExample.createCriteria().andDatabaseIdEqualTo(databaseId);
datasourceExample.createCriteria().andDatabaseIdEqualTo(databaseId)
.andStatusNotEqualTo(StatusEnum.DELETED.getCode());
return datasourceMapper.selectByExampleWithBLOBs(datasourceExample);
}
@@ -63,11 +65,6 @@ public class DatasourceRepositoryImpl implements DatasourceRepository {
return datasourceMapper.selectByPrimaryKey(id);
}
@Override
public void deleteDatasource(Long id) {
datasourceMapper.deleteByPrimaryKey(id);
}
@Override
public void createDatasourceRela(DatasourceRelaDO datasourceRelaDO) {
datasourceRelaDOMapper.insert(datasourceRelaDO);

57
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java
View File

@@ -1,17 +1,13 @@
package com.tencent.supersonic.semantic.model.infrastructure.repository;
import com.google.common.collect.Lists;
import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO;
import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDOExample;
import com.tencent.supersonic.semantic.model.domain.repository.DimensionRepository;
import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOCustomMapper;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOMapper;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
@Service
public class DimensionRepositoryImpl implements DimensionRepository {
@@ -19,14 +15,12 @@ public class DimensionRepositoryImpl implements DimensionRepository {
private DimensionDOCustomMapper dimensionDOCustomMapper;
public DimensionRepositoryImpl(DimensionDOMapper dimensionDOMapper,
DimensionDOCustomMapper dimensionDOCustomMapper) {
this.dimensionDOMapper = dimensionDOMapper;
this.dimensionDOCustomMapper = dimensionDOCustomMapper;
}
@Override
public void createDimension(DimensionDO dimensionDO) {
dimensionDOMapper.insert(dimensionDO);
@@ -39,44 +33,12 @@ public class DimensionRepositoryImpl implements DimensionRepository {
@Override
public void updateDimension(DimensionDO dimensionDO) {
dimensionDOMapper.updateByPrimaryKeyWithBLOBs(dimensionDO);
dimensionDOMapper.updateByPrimaryKeySelective(dimensionDO);
}
@Override
public List<DimensionDO> getDimensionListOfDatasource(Long datasourceId) {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
dimensionDOExample.createCriteria().andDatasourceIdEqualTo(datasourceId);
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}
@Override
public List<DimensionDO> getDimensionListOfmodel(Long modelId) {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
dimensionDOExample.createCriteria().andModelIdEqualTo(modelId);
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}
@Override
public List<DimensionDO> getDimensionListOfmodelIds(List<Long> modelIds) {
if (CollectionUtils.isEmpty(modelIds)) {
return Lists.newArrayList();
}
DimensionDOExample dimensionDOExample = new DimensionDOExample();
dimensionDOExample.createCriteria().andModelIdIn(modelIds);
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}
@Override
public List<DimensionDO> getDimensionList() {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}
@Override
public List<DimensionDO> getDimensionListByIds(List<Long> ids) {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
dimensionDOExample.createCriteria().andIdIn(ids);
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
public void batchUpdateStatus(List<DimensionDO> dimensionDOS) {
dimensionDOCustomMapper.batchUpdateStatus(dimensionDOS);
}
@Override
@@ -84,22 +46,9 @@ public class DimensionRepositoryImpl implements DimensionRepository {
return dimensionDOMapper.selectByPrimaryKey(id);
}
@Override
public List<DimensionDO> getAllDimensionList() {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}
@Override
public List<DimensionDO> getDimension(DimensionFilter dimensionFilter) {
return dimensionDOCustomMapper.query(dimensionFilter);
}
@Override
public void deleteDimension(Long id) {
dimensionDOMapper.deleteByPrimaryKey(id);
}
}

45
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java
View File

@@ -1,14 +1,11 @@
package com.tencent.supersonic.semantic.model.infrastructure.repository;
import com.google.common.collect.Lists;
import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO;
import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDOExample;
import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.MetricDOCustomMapper;
import com.tencent.supersonic.semantic.model.infrastructure.mapper.MetricDOMapper;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Component;
@@ -39,37 +36,12 @@ public class MetricRepositoryImpl implements MetricRepository {
@Override
public void updateMetric(MetricDO metricDO) {
metricDOMapper.updateByPrimaryKeyWithBLOBs(metricDO);
metricDOMapper.updateByPrimaryKeySelective(metricDO);
}
@Override
public List<MetricDO> getMetricList(Long modelId) {
MetricDOExample metricDOExample = new MetricDOExample();
metricDOExample.createCriteria().andModelIdEqualTo(modelId);
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
}
@Override
public List<MetricDO> getMetricList(List<Long> modelIds) {
if (CollectionUtils.isEmpty(modelIds)) {
return Lists.newArrayList();
}
MetricDOExample metricDOExample = new MetricDOExample();
metricDOExample.createCriteria().andModelIdIn(modelIds);
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
}
@Override
public List<MetricDO> getMetricList() {
MetricDOExample metricDOExample = new MetricDOExample();
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
}
@Override
public List<MetricDO> getMetricListByIds(List<Long> ids) {
MetricDOExample metricDOExample = new MetricDOExample();
metricDOExample.createCriteria().andIdIn(ids);
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
public void batchUpdateStatus(List<MetricDO> metricDOS) {
metricDOCustomMapper.batchUpdateStatus(metricDOS);
}
@Override
@@ -77,20 +49,9 @@ public class MetricRepositoryImpl implements MetricRepository {
return metricDOMapper.selectByPrimaryKey(id);
}
@Override
public List<MetricDO> getAllMetricList() {
return metricDOMapper.selectByExampleWithBLOBs(new MetricDOExample());
}
@Override
public List<MetricDO> getMetric(MetricFilter metricFilter) {
return metricDOCustomMapper.query(metricFilter);
}
@Override
public void deleteMetric(Long id) {
metricDOMapper.deleteByPrimaryKey(id);
}
}

11
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/ModelRepositoryImpl.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.semantic.model.infrastructure.repository;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO;
import com.tencent.supersonic.semantic.model.domain.dataobject.ModelDOExample;
import com.tencent.supersonic.semantic.model.domain.repository.ModelRepository;
@@ -24,17 +25,21 @@ public class ModelRepositoryImpl implements ModelRepository {
@Override
public void updateModel(ModelDO modelDO) {
modelDOMapper.updateByPrimaryKeyWithBLOBs(modelDO);
modelDOMapper.updateByPrimaryKeySelective(modelDO);
}
@Override
public void deleteModel(Long id) {
modelDOMapper.deleteByPrimaryKey(id);
ModelDO modelDO = modelDOMapper.selectByPrimaryKey(id);
modelDO.setStatus(StatusEnum.DELETED.getCode());
modelDOMapper.updateByPrimaryKey(modelDO);
}
@Override
public List<ModelDO> getModelList() {
return modelDOMapper.selectByExampleWithBLOBs(new ModelDOExample());
ModelDOExample modelDOExample = new ModelDOExample();
modelDOExample.createCriteria().andStatusNotEqualTo(StatusEnum.DELETED.getCode());
return modelDOMapper.selectByExampleWithBLOBs(modelDOExample);
}
@Override

19
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DatasourceController.java
View File

@@ -11,6 +11,7 @@ import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.semantic.model.domain.pojo.Datasource;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -24,25 +25,22 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping("/api/semantic/datasource")
public class DatasourceController {
private DatasourceService datasourceService;
public DatasourceController(DatasourceService datasourceService) {
this.datasourceService = datasourceService;
}
@PostMapping("/createDatasource")
public DatasourceResp createDatasource(@RequestBody DatasourceReq datasourceReq,
public Datasource createDatasource(@RequestBody DatasourceReq datasourceReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return datasourceService.createDatasource(datasourceReq, user);
}
@PostMapping("/updateDatasource")
public DatasourceResp updateDatasource(@RequestBody DatasourceReq datasourceReq,
public Datasource updateDatasource(@RequestBody DatasourceReq datasourceReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
@@ -59,10 +57,12 @@ public class DatasourceController {
return datasourceService.getMeasureListOfModel(modelId);
}
@DeleteMapping("deleteDatasource/{id}")
public void deleteDatasource(@PathVariable("id") Long id) throws Exception {
datasourceService.deleteDatasource(id);
public boolean deleteDatasource(@PathVariable("id") Long id,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
datasourceService.deleteDatasource(id, user);
return true;
}
/**
@@ -71,8 +71,7 @@ public class DatasourceController {
*/
@PostMapping("/createOrUpdateDatasourceRela")
public DatasourceRelaResp createOrUpdateDatasourceRela(@RequestBody DatasourceRelaReq datasourceRelaReq,
HttpServletRequest request,
HttpServletResponse response) {
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return datasourceService.createOrUpdateDatasourceRela(datasourceRelaReq, user);
}

29
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DimensionController.java
View File

@@ -1,10 +1,13 @@
package com.tencent.supersonic.semantic.model.rest;
import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
@@ -13,6 +16,8 @@ import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -57,6 +62,15 @@ public class DimensionController {
return true;
}
@PostMapping("/batchUpdateStatus")
public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
dimensionService.batchUpdateStatus(metaBatchReq, user);
return true;
}
@PostMapping("/mockDimensionAlias")
public List<String> mockMetricAlias(@RequestBody DimensionReq dimensionReq,
HttpServletRequest request,
@@ -76,7 +90,9 @@ public class DimensionController {
@GetMapping("/getDimensionList/{modelId}")
public List<DimensionResp> getDimension(@PathVariable("modelId") Long modelId) {
return dimensionService.getDimensions(modelId);
DimensionFilter dimensionFilter = new DimensionFilter();
dimensionFilter.setModelIds(Lists.newArrayList(modelId));
return dimensionService.getDimensions(dimensionFilter);
}
@@ -94,15 +110,20 @@ public class DimensionController {
@DeleteMapping("deleteDimension/{id}")
public Boolean deleteDimension(@PathVariable("id") Long id) throws Exception {
dimensionService.deleteDimension(id);
public Boolean deleteDimension(@PathVariable("id") Long id,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
dimensionService.deleteDimension(id, user);
return true;
}
@GetMapping("/getAllHighSensitiveDimension")
public List<DimensionResp> getAllHighSensitiveDimension() {
return dimensionService.getAllHighSensitiveDimension();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
return dimensionService.getDimensions(metaFilter);
}

30
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java
View File

@@ -2,9 +2,12 @@ package com.tencent.supersonic.semantic.model.rest;
import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.pojo.DrillDownDimension;
import com.tencent.supersonic.semantic.api.model.request.MetaBatchReq;
import com.tencent.supersonic.semantic.api.model.request.MetricReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
@@ -14,6 +17,8 @@ import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -41,7 +46,7 @@ public class MetricController {
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
metricService.creatExprMetric(metricReq, user);
metricService.createMetric(metricReq, user);
return true;
}
@@ -54,6 +59,15 @@ public class MetricController {
return true;
}
@PostMapping("/batchUpdateStatus")
public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
metricService.batchUpdateStatus(metaBatchReq, user);
return true;
}
@PostMapping("/mockMetricAlias")
public List<String> mockMetricAlias(@RequestBody MetricReq metricReq,
@@ -65,7 +79,8 @@ public class MetricController {
@GetMapping("/getMetricList/{modelId}")
public List<MetricResp> getMetricList(@PathVariable("modelId") Long modelId) {
return metricService.getMetrics(modelId);
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId));
return metricService.getMetrics(metaFilter);
}
@@ -84,14 +99,19 @@ public class MetricController {
@DeleteMapping("deleteMetric/{id}")
public Boolean deleteMetric(@PathVariable("id") Long id) throws Exception {
metricService.deleteMetric(id);
public Boolean deleteMetric(@PathVariable("id") Long id,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
metricService.deleteMetric(id, user);
return true;
}
@GetMapping("/getAllHighSensitiveMetric")
public List<MetricResp> getAllHighSensitiveMetric() {
return metricService.getAllHighSensitiveMetric();
MetricFilter metricFilter = new MetricFilter();
metricFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
return metricService.getMetrics(metricFilter);
}

7
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/ModelController.java
View File

@@ -50,8 +50,11 @@ public class ModelController {
}
@DeleteMapping("/deleteModel/{modelId}")
public Boolean deleteModel(@PathVariable("modelId") Long modelId) {
modelService.deleteModel(modelId);
public Boolean deleteModel(@PathVariable("modelId") Long modelId,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
modelService.deleteModel(modelId, user);
return true;
}

44
semantic/model/src/main/resources/mapper/DatasourceDOMapper.xml
View File

@@ -8,14 +8,15 @@
<result column="biz_name" jdbcType="VARCHAR" property="bizName" />
<result column="description" jdbcType="VARCHAR" property="description" />
<result column="database_id" jdbcType="BIGINT" property="databaseId" />
<result column="status" jdbcType="INTEGER" property="status" />
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt" />
<result column="created_by" jdbcType="VARCHAR" property="createdBy" />
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
<result column="depends" jdbcType="VARCHAR" property="depends" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO">
<result column="datasource_detail" jdbcType="LONGVARCHAR" property="datasourceDetail" />
<result column="depends" jdbcType="LONGVARCHAR" property="depends" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
@@ -47,11 +48,11 @@
</where>
</sql>
<sql id="Base_Column_List">
id, model_id, name, biz_name, description, database_id, created_at, created_by, updated_at,
updated_by,depends
id, model_id, name, biz_name, description, database_id, status, created_at, created_by,
updated_at, updated_by
</sql>
<sql id="Blob_Column_List">
datasource_detail
datasource_detail, depends
</sql>
<select id="selectByExampleWithBLOBs" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDOExample" resultMap="ResultMapWithBLOBs">
select
@@ -98,15 +99,17 @@
delete from s2_datasource
where id = #{id,jdbcType=BIGINT}
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO">
<insert id="insert" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO" useGeneratedKeys="true" keyProperty="id">
insert into s2_datasource (id, model_id, name,
biz_name, description, database_id,
created_at, created_by, updated_at,
updated_by, datasource_detail)
status, created_at, created_by,
updated_at, updated_by, datasource_detail,
depends)
values (#{id,jdbcType=BIGINT}, #{modelId,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR},
#{bizName,jdbcType=VARCHAR}, #{description,jdbcType=VARCHAR}, #{databaseId,jdbcType=BIGINT},
#{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP},
#{updatedBy,jdbcType=VARCHAR}, #{datasourceDetail,jdbcType=LONGVARCHAR})
#{status,jdbcType=INTEGER}, #{createdAt,jdbcType=TIMESTAMP}, #{createdBy,jdbcType=VARCHAR},
#{updatedAt,jdbcType=TIMESTAMP}, #{updatedBy,jdbcType=VARCHAR}, #{datasourceDetail,jdbcType=LONGVARCHAR},
#{depends,jdbcType=LONGVARCHAR})
</insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO">
insert into s2_datasource
@@ -129,6 +132,9 @@
<if test="databaseId != null">
database_id,
</if>
<if test="status != null">
status,
</if>
<if test="createdAt != null">
created_at,
</if>
@@ -144,6 +150,9 @@
<if test="datasourceDetail != null">
datasource_detail,
</if>
<if test="depends != null">
depends,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
@@ -164,6 +173,9 @@
<if test="databaseId != null">
#{databaseId,jdbcType=BIGINT},
</if>
<if test="status != null">
#{status,jdbcType=INTEGER},
</if>
<if test="createdAt != null">
#{createdAt,jdbcType=TIMESTAMP},
</if>
@@ -179,6 +191,9 @@
<if test="datasourceDetail != null">
#{datasourceDetail,jdbcType=LONGVARCHAR},
</if>
<if test="depends != null">
#{depends,jdbcType=LONGVARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDOExample" resultType="java.lang.Long">
@@ -205,6 +220,9 @@
<if test="databaseId != null">
database_id = #{databaseId,jdbcType=BIGINT},
</if>
<if test="status != null">
status = #{status,jdbcType=INTEGER},
</if>
<if test="createdAt != null">
created_at = #{createdAt,jdbcType=TIMESTAMP},
</if>
@@ -220,6 +238,9 @@
<if test="datasourceDetail != null">
datasource_detail = #{datasourceDetail,jdbcType=LONGVARCHAR},
</if>
<if test="depends != null">
depends = #{depends,jdbcType=LONGVARCHAR},
</if>
</set>
where id = #{id,jdbcType=BIGINT}
</update>
@@ -230,11 +251,13 @@
biz_name = #{bizName,jdbcType=VARCHAR},
description = #{description,jdbcType=VARCHAR},
database_id = #{databaseId,jdbcType=BIGINT},
status = #{status,jdbcType=INTEGER},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
datasource_detail = #{datasourceDetail,jdbcType=LONGVARCHAR}
datasource_detail = #{datasourceDetail,jdbcType=LONGVARCHAR},
depends = #{depends,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
<update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DatasourceDO">
@@ -244,6 +267,7 @@
biz_name = #{bizName,jdbcType=VARCHAR},
description = #{description,jdbcType=VARCHAR},
database_id = #{databaseId,jdbcType=BIGINT},
status = #{status,jdbcType=INTEGER},
created_at = #{createdAt,jdbcType=TIMESTAMP},
created_by = #{createdBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},

2
semantic/model/src/main/resources/mapper/DimensionDOMapper.xml
View File

@@ -107,7 +107,7 @@
delete from s2_dimension
where id = #{id,jdbcType=BIGINT}
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO">
<insert id="insert" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO" useGeneratedKeys="true" keyProperty="id">
insert into s2_dimension (id, model_id, datasource_id,
name, biz_name, description,
status, sensitive_level, type,

2
semantic/model/src/main/resources/mapper/MetricDOMapper.xml
View File

@@ -104,7 +104,7 @@
delete from s2_metric
where id = #{id,jdbcType=BIGINT}
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO">
<insert id="insert" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO" useGeneratedKeys="true" keyProperty="id">
insert into s2_metric (id, model_id, name,
biz_name, description, status,
sensitive_level, type, created_at,

64
semantic/model/src/main/resources/mapper/ModelDOMapper.xml
View File

@@ -7,6 +7,7 @@
<result column="biz_name" jdbcType="VARCHAR" property="bizName" />
<result column="domain_id" jdbcType="BIGINT" property="domainId" />
<result column="alias" jdbcType="VARCHAR" property="alias" />
<result column="description" jdbcType="VARCHAR" property="description" />
<result column="viewer" jdbcType="VARCHAR" property="viewer" />
<result column="view_org" jdbcType="VARCHAR" property="viewOrg" />
<result column="admin" jdbcType="VARCHAR" property="admin" />
@@ -17,6 +18,7 @@
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
<result column="drill_down_dimensions" jdbcType="VARCHAR" property="drillDownDimensions" />
<result column="status" jdbcType="INTEGER" property="status" />
</resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO">
<result column="entity" jdbcType="LONGVARCHAR" property="entity" />
@@ -80,8 +82,8 @@
</where>
</sql>
<sql id="Base_Column_List">
id, name, biz_name, domain_id, alias, viewer, view_org, admin, admin_org, is_open,
created_by, created_at, updated_by, updated_at, drill_down_dimensions
id, name, biz_name, domain_id, alias, description, viewer, view_org, admin, admin_org,
is_open, created_by, created_at, updated_by, updated_at, drill_down_dimensions, status
</sql>
<sql id="Blob_Column_List">
entity
@@ -133,17 +135,19 @@
</delete>
<insert id="insert" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO">
insert into s2_model (id, name, biz_name,
domain_id, alias, viewer,
view_org, admin, admin_org,
is_open, created_by, created_at,
updated_by, updated_at, drill_down_dimensions,
entity)
domain_id, alias, description,
viewer, view_org, admin,
admin_org, is_open, created_by,
created_at, updated_by, updated_at,
drill_down_dimensions, status, entity
)
values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{bizName,jdbcType=VARCHAR},
#{domainId,jdbcType=BIGINT}, #{alias,jdbcType=VARCHAR}, #{viewer,jdbcType=VARCHAR},
#{viewOrg,jdbcType=VARCHAR}, #{admin,jdbcType=VARCHAR}, #{adminOrg,jdbcType=VARCHAR},
#{isOpen,jdbcType=INTEGER}, #{createdBy,jdbcType=VARCHAR}, #{createdAt,jdbcType=TIMESTAMP},
#{updatedBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP}, #{drillDownDimensions,jdbcType=VARCHAR},
#{entity,jdbcType=LONGVARCHAR})
#{domainId,jdbcType=BIGINT}, #{alias,jdbcType=VARCHAR}, #{description,jdbcType=VARCHAR},
#{viewer,jdbcType=VARCHAR}, #{viewOrg,jdbcType=VARCHAR}, #{admin,jdbcType=VARCHAR},
#{adminOrg,jdbcType=VARCHAR}, #{isOpen,jdbcType=INTEGER}, #{createdBy,jdbcType=VARCHAR},
#{createdAt,jdbcType=TIMESTAMP}, #{updatedBy,jdbcType=VARCHAR}, #{updatedAt,jdbcType=TIMESTAMP},
#{drillDownDimensions,jdbcType=VARCHAR}, #{status,jdbcType=INTEGER}, #{entity,jdbcType=LONGVARCHAR}
)
</insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.semantic.model.domain.dataobject.ModelDO">
insert into s2_model
@@ -163,6 +167,9 @@
<if test="alias != null">
alias,
</if>
<if test="description != null">
description,
</if>
<if test="viewer != null">
viewer,
</if>
@@ -193,6 +200,9 @@
<if test="drillDownDimensions != null">
drill_down_dimensions,
</if>
<if test="status != null">
status,
</if>
<if test="entity != null">
entity,
</if>
@@ -213,6 +223,9 @@
<if test="alias != null">
#{alias,jdbcType=VARCHAR},
</if>
<if test="description != null">
#{description,jdbcType=VARCHAR},
</if>
<if test="viewer != null">
#{viewer,jdbcType=VARCHAR},
</if>
@@ -243,6 +256,9 @@
<if test="drillDownDimensions != null">
#{drillDownDimensions,jdbcType=VARCHAR},
</if>
<if test="status != null">
#{status,jdbcType=INTEGER},
</if>
<if test="entity != null">
#{entity,jdbcType=LONGVARCHAR},
</if>
@@ -272,6 +288,9 @@
<if test="record.alias != null">
alias = #{record.alias,jdbcType=VARCHAR},
</if>
<if test="record.description != null">
description = #{record.description,jdbcType=VARCHAR},
</if>
<if test="record.viewer != null">
viewer = #{record.viewer,jdbcType=VARCHAR},
</if>
@@ -302,6 +321,9 @@
<if test="record.drillDownDimensions != null">
drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR},
</if>
<if test="record.status != null">
status = #{record.status,jdbcType=INTEGER},
</if>
<if test="record.entity != null">
entity = #{record.entity,jdbcType=LONGVARCHAR},
</if>
@@ -317,6 +339,7 @@
biz_name = #{record.bizName,jdbcType=VARCHAR},
domain_id = #{record.domainId,jdbcType=BIGINT},
alias = #{record.alias,jdbcType=VARCHAR},
description = #{record.description,jdbcType=VARCHAR},
viewer = #{record.viewer,jdbcType=VARCHAR},
view_org = #{record.viewOrg,jdbcType=VARCHAR},
admin = #{record.admin,jdbcType=VARCHAR},
@@ -327,6 +350,7 @@
updated_by = #{record.updatedBy,jdbcType=VARCHAR},
updated_at = #{record.updatedAt,jdbcType=TIMESTAMP},
drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR},
status = #{record.status,jdbcType=INTEGER},
entity = #{record.entity,jdbcType=LONGVARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
@@ -339,6 +363,7 @@
biz_name = #{record.bizName,jdbcType=VARCHAR},
domain_id = #{record.domainId,jdbcType=BIGINT},
alias = #{record.alias,jdbcType=VARCHAR},
description = #{record.description,jdbcType=VARCHAR},
viewer = #{record.viewer,jdbcType=VARCHAR},
view_org = #{record.viewOrg,jdbcType=VARCHAR},
admin = #{record.admin,jdbcType=VARCHAR},
@@ -348,7 +373,8 @@
created_at = #{record.createdAt,jdbcType=TIMESTAMP},
updated_by = #{record.updatedBy,jdbcType=VARCHAR},
updated_at = #{record.updatedAt,jdbcType=TIMESTAMP},
drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR}
drill_down_dimensions = #{record.drillDownDimensions,jdbcType=VARCHAR},
status = #{record.status,jdbcType=INTEGER}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
@@ -368,6 +394,9 @@
<if test="alias != null">
alias = #{alias,jdbcType=VARCHAR},
</if>
<if test="description != null">
description = #{description,jdbcType=VARCHAR},
</if>
<if test="viewer != null">
viewer = #{viewer,jdbcType=VARCHAR},
</if>
@@ -398,6 +427,9 @@
<if test="drillDownDimensions != null">
drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR},
</if>
<if test="status != null">
status = #{status,jdbcType=INTEGER},
</if>
<if test="entity != null">
entity = #{entity,jdbcType=LONGVARCHAR},
</if>
@@ -410,6 +442,7 @@
biz_name = #{bizName,jdbcType=VARCHAR},
domain_id = #{domainId,jdbcType=BIGINT},
alias = #{alias,jdbcType=VARCHAR},
description = #{description,jdbcType=VARCHAR},
viewer = #{viewer,jdbcType=VARCHAR},
view_org = #{viewOrg,jdbcType=VARCHAR},
admin = #{admin,jdbcType=VARCHAR},
@@ -420,6 +453,7 @@
updated_by = #{updatedBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR},
status = #{status,jdbcType=INTEGER},
entity = #{entity,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=BIGINT}
</update>
@@ -429,6 +463,7 @@
biz_name = #{bizName,jdbcType=VARCHAR},
domain_id = #{domainId,jdbcType=BIGINT},
alias = #{alias,jdbcType=VARCHAR},
description = #{description,jdbcType=VARCHAR},
viewer = #{viewer,jdbcType=VARCHAR},
view_org = #{viewOrg,jdbcType=VARCHAR},
admin = #{admin,jdbcType=VARCHAR},
@@ -438,7 +473,8 @@
created_at = #{createdAt,jdbcType=TIMESTAMP},
updated_by = #{updatedBy,jdbcType=VARCHAR},
updated_at = #{updatedAt,jdbcType=TIMESTAMP},
drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR}
drill_down_dimensions = #{drillDownDimensions,jdbcType=VARCHAR},
status = #{status,jdbcType=INTEGER}
where id = #{id,jdbcType=BIGINT}
</update>
</mapper>

21
semantic/model/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml
View File

@@ -64,7 +64,7 @@
created_at, created_by, updated_by, updated_at, semantic_type
</sql>
<insert id="batchInsert" parameterType="java.util.List">
<insert id="batchInsert" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
insert into s2_dimension (name, biz_name,
description, status, model_id,
type, type_params, expr,
@@ -107,10 +107,21 @@
</foreach>
</update>
<update id="batchUpdateStatus" parameterType="java.util.List">
<foreach collection="list" item="dimension" separator=";">
update s2_dimension
set
status = #{dimension.status,jdbcType=INTEGER},
updated_by = #{dimension.updatedBy,jdbcType=VARCHAR},
updated_at = #{dimension.updatedAt,jdbcType=TIMESTAMP}
where id = #{dimension.id,jdbcType=BIGINT}
</foreach>
</update>
<select id="query" resultMap="ResultMapWithBLOBs">
select *
from s2_dimension
where 1=1
where status != 3
<if test="key != null and key != ''">
and ( id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
@@ -130,6 +141,9 @@
<if test="sensitiveLevel != null">
and sensitive_level = #{sensitiveLevel}
</if>
<if test="status != null">
and status = #{status}
</if>
<if test="modelIds != null and modelIds.size >0">
and model_id in
<foreach collection="modelIds" index="index" item="model" open="(" close=")"
@@ -140,6 +154,9 @@
<if test="createdBy != null">
and created_by = #{createdBy}
</if>
<if test="datasourceId != null">
and datasource_id = #{datasourceId}
</if>
</select>

24
semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml
View File

@@ -62,7 +62,7 @@
type_params
</sql>
<insert id="batchInsert" parameterType="java.util.List">
<insert id="batchInsert" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
insert into s2_metric (model_id, name,
biz_name, description, type,status,sensitive_level,
created_at, created_by, updated_at,
@@ -100,10 +100,20 @@
</foreach>
</update>
<update id="batchUpdateStatus" parameterType="java.util.List">
<foreach collection="list" item="metric" separator=";">
update s2_metric
set status = #{metric.status,jdbcType=INTEGER},
updated_at = #{metric.updatedAt,jdbcType=TIMESTAMP},
updated_by = #{metric.updatedBy,jdbcType=VARCHAR}
where id = #{metric.id,jdbcType=BIGINT}
</foreach>
</update>
<select id="query" resultMap="ResultMapWithBLOBs">
select *
from s2_metric
where 1=1
where status != 3
<if test="type != null and type != ''">
and type = #{type}
</if>
@@ -127,6 +137,9 @@
<if test="sensitiveLevel != null">
and sensitive_level = #{sensitiveLevel}
</if>
<if test="status != null">
and status = #{status}
</if>
<if test="modelIds != null and modelIds.size >0">
and model_id in
<foreach collection="modelIds" index="index" item="model" open="(" close=")"
@@ -134,6 +147,13 @@
#{model}
</foreach>
</if>
<if test="ids != null and ids.size >0">
and id in
<foreach collection="ids" index="index" item="id" open="(" close=")"
separator=",">
#{id}
</foreach>
</if>
<if test="createdBy != null">
and created_by = #{createdBy}
</if>

0
semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DatabaseApplicationServiceTest.java
View File

0
semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DatasourceApplicationServiceTest.java
View File

0
semantic/model/src/test/java/com/tencent/supersonic/semantic/model/application/DomainApplicationServiceTest.java
View File

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.query.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
@@ -13,6 +14,7 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
@@ -22,6 +24,7 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.util.Sets;
import org.springframework.beans.BeanUtils;
@@ -89,8 +92,11 @@ public class AuthCommonService {
public Set<String> getHighSensitiveColsByModelId(Long modelId) {
Set<String> highSensitiveCols = new HashSet<>();
List<DimensionResp> highSensitiveDimensions = dimensionService.getHighSensitiveDimension(modelId);
List<MetricResp> highSensitiveMetrics = metricService.getHighSensitiveMetric(modelId);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
List<DimensionResp> highSensitiveDimensions = dimensionService.getDimensions(metaFilter);
List<MetricResp> highSensitiveMetrics = metricService.getMetrics(metaFilter);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java
View File

@@ -118,8 +118,8 @@ public class SchemaServiceImpl implements SchemaService {
}
@Override
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricCmd, User user) {
return metricService.queryMetric(pageMetricCmd, user);
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user) {
return metricService.queryMetric(pageMetricReq, user);
}
@Override

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -5,6 +5,7 @@ import static com.tencent.supersonic.common.pojo.Constants.MINUS;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
@@ -15,6 +16,7 @@ import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
@@ -38,6 +40,7 @@ import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
@@ -271,8 +274,11 @@ public class DataPermissionAOP {
private Set<String> getHighSensitiveColsByModelId(Long modelId) {
Set<String> highSensitiveCols = new HashSet<>();
List<DimensionResp> highSensitiveDimensions = dimensionService.getHighSensitiveDimension(modelId);
List<MetricResp> highSensitiveMetrics = metricService.getHighSensitiveMetric(modelId);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
List<DimensionResp> highSensitiveDimensions = dimensionService.getDimensions(metaFilter);
List<MetricResp> highSensitiveMetrics = metricService.getMetrics(metaFilter);
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
highSensitiveDimensions.stream().forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
@@ -387,7 +393,7 @@ public class DataPermissionAOP {
modelNameCn = modelInfos.get(0).getName();
}
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryStructReq.getModelId());
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds));
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));

9
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.semantic.query.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
@@ -21,6 +22,7 @@ import java.util.Set;
import java.util.Objects;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
@@ -54,11 +56,12 @@ public class DimValueAspect {
}
Object[] args = joinPoint.getArgs();
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2QLReq.getModelId()));
String sql = queryS2QLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue.
List<FilterExpression> filterExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(queryS2QLReq.getModelId());
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Set<String> fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet());
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
filterExpressionList.stream().forEach(expression -> {
@@ -145,8 +148,8 @@ public class DimValueAspect {
Object[] args = joinPoint.getArgs();
QueryStructReq queryStructReq = (QueryStructReq) args[0];
Long modelId = queryStructReq.getModelId();
List<DimensionResp> dimensions = dimensionService.getDimensions(modelId);
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(modelId));
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions);
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions);

5
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java
View File

@@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
import com.tencent.supersonic.semantic.query.service.AuthCommonService;
import java.util.ArrayList;
import java.util.HashSet;
@@ -169,8 +170,8 @@ public class S2QLDataAspect {
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryS2QLReq.getModelId());
MetaFilter metaFilter = new MetaFilter(modelIds);
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(metaFilter);
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));