topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) transfer term nature modelId to viewId before providing it to chat and put the modelId of metadata into the dict word instead of viewId (#739)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-02-23 10:29:21 +08:00
committed by GitHub
parent 16643e8d75
commit e95a528219
52 changed files with 456 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java
View File

@@ -45,7 +45,8 @@ public class GroupByCorrector extends BaseSemanticCorrector {
ViewResp viewResp = viewService.getView(viewId);
List<Long> modelIds = viewResp.getViewDetail().getViewModelConfigs().stream().map(config -> config.getId())
.collect(Collectors.toList());
MetaFilter metaFilter = new MetaFilter(modelIds);
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(modelIds);
List<ModelResp> modelRespList = modelService.getModelList(metaFilter);
for (ModelResp modelResp : modelRespList) {
List<Dim> dimList = modelResp.getModelDetail().getDimensions();

6
chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java
View File

@@ -10,9 +10,10 @@ import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult;
import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.server.service.KnowledgeService;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
/***
* A mapper that recognizes schema elements with vector embedding.
@@ -24,7 +25,8 @@ public class EmbeddingMapper extends BaseMapper {
public void doMap(QueryContext queryContext) {
//1. query from embedding by queryText
String queryText = queryContext.getQueryText();
List<S2Term> terms = HanlpHelper.getTerms(queryText);
KnowledgeService knowledgeService = ContextUtils.getBean(KnowledgeService.class);
List<S2Term> terms = knowledgeService.getTerms(queryText);
EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class);
List<EmbeddingResult> matchResults = matchStrategy.getMatches(queryContext, terms);

29
chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java
View File

@@ -1,36 +1,27 @@
package com.tencent.supersonic.chat.core.parser.sql.llm;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.common.util.DatePeriodEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import java.util.Objects;
public class S2SqlDateHelper {
public static String getReferenceDate(QueryContext queryContext, Long modelId) {
public static String getReferenceDate(QueryContext queryContext, Long viewId) {
String defaultDate = DateUtils.getBeforeDate(0);
if (Objects.isNull(modelId)) {
if (Objects.isNull(viewId)) {
return defaultDate;
}
ChatConfigFilter filter = new ChatConfigFilter();
filter.setModelId(modelId);
ChatConfigRichResp chatConfigRichResp = queryContext.getModelIdToChatRichConfig().get(modelId);
if (Objects.isNull(chatConfigRichResp)) {
ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId);
if (viewSchema == null || viewSchema.getTagTypeTimeDefaultConfig() == null) {
return defaultDate;
}
if (Objects.isNull(chatConfigRichResp.getChatDetailRichConfig()) || Objects.isNull(
chatConfigRichResp.getChatDetailRichConfig().getChatDefaultConfig())) {
return defaultDate;
}
ChatDefaultRichConfigResp chatDefaultConfig = chatConfigRichResp.getChatDetailRichConfig()
.getChatDefaultConfig();
Integer unit = chatDefaultConfig.getUnit();
String period = chatDefaultConfig.getPeriod();
TimeDefaultConfig tagTypeTimeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig();
Integer unit = tagTypeTimeDefaultConfig.getUnit();
String period = tagTypeTimeDefaultConfig.getPeriod();
if (Objects.nonNull(unit)) {
// If the unit is set to less than 0, then do not add relative date.
if (unit < 0) {

5
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java
View File

@@ -44,6 +44,7 @@ public class ViewSchemaBuilder {
SchemaElement metricToAdd = SchemaElement.builder()
.view(resp.getId())
.model(metric.getModelId())
.id(metric.getId())
.name(metric.getName())
.bizName(metric.getBizName())
@@ -84,6 +85,7 @@ public class ViewSchemaBuilder {
}
SchemaElement dimToAdd = SchemaElement.builder()
.view(resp.getId())
.model(dim.getModelId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())
@@ -96,6 +98,7 @@ public class ViewSchemaBuilder {
SchemaElement dimValueToAdd = SchemaElement.builder()
.view(resp.getId())
.model(dim.getModelId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())
@@ -107,6 +110,7 @@ public class ViewSchemaBuilder {
if (dim.getIsTag() == 1) {
SchemaElement tagToAdd = SchemaElement.builder()
.view(resp.getId())
.model(dim.getModelId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())
@@ -126,6 +130,7 @@ public class ViewSchemaBuilder {
if (dim != null) {
SchemaElement entity = SchemaElement.builder()
.view(resp.getId())
.model(dim.getModelId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())

8
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java
View File

@@ -38,11 +38,11 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
queryResult.setRecommendedDimensions(dimensionRecommended);
}
private List<SchemaElement> getDimensions(Long metricId, Long modelId) {
private List<SchemaElement> getDimensions(Long metricId, Long viewId) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
ViewSchema modelSchema = semanticService.getModelSchema(modelId);
ViewSchema viewSchema = semanticService.getViewSchema(viewId);
List<Long> drillDownDimensions = Lists.newArrayList();
Set<SchemaElement> metricElements = modelSchema.getMetrics();
Set<SchemaElement> metricElements = viewSchema.getMetrics();
if (!CollectionUtils.isEmpty(metricElements)) {
Optional<SchemaElement> metric = metricElements.stream().filter(schemaElement ->
metricId.equals(schemaElement.getId())
@@ -54,7 +54,7 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
}
}
final List<Long> drillDownDimensionsFinal = drillDownDimensions;
return modelSchema.getDimensions().stream()
return viewSchema.getDimensions().stream()
.filter(dim -> filterDimension(drillDownDimensionsFinal, dim))
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(recommend_dimension_size)

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java
View File

@@ -50,7 +50,7 @@ public class SemanticService {
return schemaService.getSemanticSchema();
}
public ViewSchema getModelSchema(Long id) {
public ViewSchema getViewSchema(Long id) {
return schemaService.getViewSchema(id);
}

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java
View File

@@ -219,7 +219,7 @@ public class ConfigServiceImpl implements ConfigService {
}
BeanUtils.copyProperties(chatConfigResp, chatConfigRich);
ViewSchema viewSchema = semanticService.getModelSchema(modelId);
ViewSchema viewSchema = semanticService.getViewSchema(modelId);
if (viewSchema == null) {
return chatConfigRich;
}

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java
View File

@@ -48,7 +48,7 @@ public class RecommendServiceImpl implements RecommendService {
if (Objects.isNull(modelId)) {
return new RecommendResp();
}
ViewSchema modelSchema = semanticService.getModelSchema(modelId);
ViewSchema modelSchema = semanticService.getViewSchema(modelId);
if (Objects.isNull(modelSchema)) {
return new RecommendResp();
}

19
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java
View File

@@ -1,17 +1,19 @@
package com.tencent.supersonic.chat.server.service.impl;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@@ -28,7 +30,6 @@ public class WordService {
addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words);
addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words);
addWordsByType(DictWordType.VIEW, semanticSchema.getViews(), words);
addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words);
addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words);
@@ -36,6 +37,7 @@ public class WordService {
}
private void addWordsByType(DictWordType value, List<SchemaElement> metas, List<DictWord> natures) {
metas = distinct(metas);
List<DictWord> natureList = WordBuilderFactory.get(value).getDictWords(metas);
log.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
natures.addAll(natureList);
@@ -48,4 +50,13 @@ public class WordService {
public void setPreDictWords(List<DictWord> preDictWords) {
this.preDictWords = preDictWords;
}
private List<SchemaElement> distinct(List<SchemaElement> metas) {
return metas.stream()
.collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1))
.values()
.stream()
.collect(Collectors.toList());
}
}

8
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java
View File

@@ -19,13 +19,13 @@ public class ModelDetail {
private String tableQuery;
private List<Identify> identifiers;
private List<Identify> identifiers = Lists.newArrayList();
private List<Dim> dimensions;
private List<Dim> dimensions = Lists.newArrayList();
private List<Measure> measures;
private List<Measure> measures = Lists.newArrayList();
private List<Field> fields;
private List<Field> fields = Lists.newArrayList();
public String getSqlQuery() {
if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) {

13
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
@Data
public class Param {
@@ -10,7 +11,7 @@ public class Param {
@NotBlank(message = "Invald parameter name")
private String name;
@NotBlank(message = "Invalid parameter value")
@NotNull(message = "Invalid parameter value")
private String value;
public Param() {
@@ -21,14 +22,4 @@ public class Param {
this.value = value;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("{");
sb.append("\"name\":\"")
.append(name).append('\"');
sb.append(",\"value\":\"")
.append(value).append('\"');
sb.append('}');
return sb.toString();
}
}

5
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.api.pojo;
import com.google.common.base.Objects;
import com.google.common.collect.Lists;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@@ -19,6 +18,7 @@ import java.util.List;
public class SchemaElement implements Serializable {
private Long view;
private Long model;
private Long id;
private String name;
private String bizName;
@@ -52,7 +52,4 @@ public class SchemaElement implements Serializable {
return Objects.hashCode(view, id, name, bizName, type);
}
public List<String> getModelNames() {
return Lists.newArrayList(name);
}
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java
View File

@@ -30,6 +30,8 @@ public class MetricResp extends SchemaItem {
private Long domainId;
private String modelBizName;
private String modelName;
//ATOMIC DERIVED

29
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java
View File

@@ -6,8 +6,15 @@ import com.hankcs.hanlp.corpus.tag.Nature;
import com.hankcs.hanlp.dictionary.CoreDictionary;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -17,11 +24,6 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@Slf4j
public class SearchService {
@@ -39,14 +41,14 @@ public class SearchService {
* @param key
* @return
*/
public static List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> detectModelIds) {
return prefixSearch(key, limit, trie, detectModelIds);
public static List<HanlpMapResult> prefixSearch(String key, int limit, Map<Long, List<Long>> modelIdToViewIds) {
return prefixSearch(key, limit, trie, modelIdToViewIds);
}
public static List<HanlpMapResult> prefixSearch(String key, int limit, BinTrie<List<String>> binTrie,
Set<Long> detectModelIds) {
Set<Map.Entry<String, List<String>>> result = prefixSearchLimit(key, limit, binTrie, detectModelIds);
return result.stream().map(
Map<Long, List<Long>> modelIdToViewIds) {
Set<Map.Entry<String, List<String>>> result = prefixSearchLimit(key, limit, binTrie, modelIdToViewIds.keySet());
List<HanlpMapResult> hanlpMapResults = result.stream().map(
entry -> {
String name = entry.getKey().replace("#", " ");
return new HanlpMapResult(name, entry.getValue(), key);
@@ -54,6 +56,13 @@ public class SearchService {
).sorted((a, b) -> -(b.getName().length() - a.getName().length()))
.limit(SEARCH_SIZE)
.collect(Collectors.toList());
for (HanlpMapResult hanlpMapResult : hanlpMapResults) {
List<String> natures = hanlpMapResult.getNatures().stream()
.map(nature -> NatureHelper.changeModel2View(nature, modelIdToViewIds))
.flatMap(Collection::stream).collect(Collectors.toList());
hanlpMapResult.setNatures(natures);
}
return hanlpMapResults;
}
/***

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java
View File

@@ -38,11 +38,11 @@ public class DimensionWordBuilder extends BaseWordBuilder {
private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) {
DictWord dictWord = new DictWord();
dictWord.setWord(word);
Long viewId = schemaElement.getView();
String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId()
Long modelId = schemaElement.getModel();
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.DIMENSION.getType();
if (isSuffix) {
nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId()
nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.SUFFIX.getType() + DictWordType.DIMENSION.getType();
}
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java
View File

@@ -27,8 +27,8 @@ public class EntityWordBuilder extends BaseWordBuilder {
return result;
}
Long view = schemaElement.getView();
String nature = DictWordType.NATURE_SPILT + view + DictWordType.NATURE_SPILT + schemaElement.getId()
Long modelId = schemaElement.getModel();
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.ENTITY.getType();
if (!CollectionUtils.isEmpty(schemaElement.getAlias())) {

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java
View File

@@ -38,11 +38,11 @@ public class MetricWordBuilder extends BaseWordBuilder {
private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) {
DictWord dictWord = new DictWord();
dictWord.setWord(word);
Long viewId = schemaElement.getView();
String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId()
Long modelId = schemaElement.getModel();
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.METRIC.getType();
if (isSuffix) {
nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId()
nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId()
+ DictWordType.SUFFIX.getType() + DictWordType.METRIC.getType();
}
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java
View File

@@ -27,8 +27,8 @@ public class ValueWordBuilder extends BaseWordBuilder {
schemaElement.getAlias().stream().forEach(value -> {
DictWord dictWord = new DictWord();
Long viewId = schemaElement.getView();
String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId();
Long modelId = schemaElement.getModel();
String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId();
dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature));
dictWord.setWord(value);
result.add(dictWord);

41
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java
View File

@@ -1,34 +1,36 @@
package com.tencent.supersonic.headless.core.knowledge.helper;
import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath;
import com.google.common.collect.Lists;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.corpus.tag.Nature;
import com.hankcs.hanlp.dictionary.CoreDictionary;
import com.hankcs.hanlp.dictionary.DynamicCustomDictionary;
import com.hankcs.hanlp.seg.Segment;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.knowledge.HadoopFileIOAdapter;
import com.tencent.supersonic.headless.core.knowledge.MapResult;
import com.tencent.supersonic.headless.core.knowledge.MultiCustomDictionary;
import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ResourceUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.hankcs.hanlp.seg.common.Term;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ResourceUtils;
import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath;
/**
* HanLP helper
@@ -212,18 +214,25 @@ public class HanlpHelper {
}
}
public static List<com.tencent.supersonic.headless.api.pojo.response.S2Term> getTerms(String text) {
public static List<S2Term> getTerms(String text, Map<Long, List<Long>> modelIdToViewIds) {
return getSegment().seg(text.toLowerCase()).stream()
.filter(term -> term.getNature().startsWith(DictWordType.NATURE_SPILT))
.map(term -> transform2ApiTerm(term))
.map(term -> transform2ApiTerm(term, modelIdToViewIds))
.flatMap(Collection::stream)
.collect(Collectors.toList());
}
public static S2Term transform2ApiTerm(Term term) {
S2Term knowledgeTerm = new S2Term();
BeanUtils.copyProperties(term, knowledgeTerm);
knowledgeTerm.setFrequency(term.getFrequency());
return knowledgeTerm;
public static List<S2Term> transform2ApiTerm(Term term, Map<Long, List<Long>> modelIdToViewIds) {
List<S2Term> s2Terms = Lists.newArrayList();
List<String> natures = NatureHelper.changeModel2View(String.valueOf(term.getNature()), modelIdToViewIds);
for (String nature : natures) {
S2Term s2Term = new S2Term();
BeanUtils.copyProperties(term, s2Term);
s2Term.setNature(Nature.create(nature));
s2Term.setFrequency(term.getFrequency());
s2Terms.add(s2Term);
}
return s2Terms;
}
}

41
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java
View File

@@ -1,12 +1,14 @@
package com.tencent.supersonic.headless.core.knowledge.helper;
import com.google.common.collect.Lists;
import com.hankcs.hanlp.corpus.tag.Nature;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Comparator;
@@ -81,6 +83,43 @@ public class NatureHelper {
return null;
}
private static Long getModelId(String nature) {
try {
String[] split = nature.split(DictWordType.NATURE_SPILT);
if (split.length <= 1) {
return null;
}
return Long.valueOf(split[1]);
} catch (NumberFormatException e) {
log.error("", e);
}
return null;
}
private static Nature changeModel2View(String nature, Long viewId) {
try {
String[] split = nature.split(DictWordType.NATURE_SPILT);
if (split.length <= 1) {
return null;
}
split[1] = String.valueOf(viewId);
return Nature.create(StringUtils.join(split, DictWordType.NATURE_SPILT));
} catch (NumberFormatException e) {
log.error("", e);
}
return null;
}
public static List<String> changeModel2View(String nature, Map<Long, List<Long>> modelIdToViewIds) {
Long modelId = getModelId(nature);
List<Long> viewIds = modelIdToViewIds.get(modelId);
if (CollectionUtils.isEmpty(viewIds)) {
return Lists.newArrayList();
}
return viewIds.stream().map(viewId -> String.valueOf(changeModel2View(nature, viewId)))
.collect(Collectors.toList());
}
public static boolean isDimensionValueViewId(String nature) {
if (StringUtils.isEmpty(nature)) {
return false;

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
View File

@@ -11,11 +11,13 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.MetricService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Collection;
@@ -28,6 +30,9 @@ import java.util.stream.Collectors;
@Slf4j
public class MetricDrillDownChecker {
@Autowired
private MetricService metricService;
@Around("execution(* com.tencent.supersonic.headless.core.parser.QueryParser.parse(..))")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] objects = joinPoint.getArgs();
@@ -52,7 +57,7 @@ public class MetricDrillDownChecker {
List<DimensionResp> necessaryDimensions = getNecessaryDimensions(metric, semanticSchemaResp);
List<DimensionResp> dimensionsMissing = getNecessaryDimensionMissing(necessaryDimensions, dimensionFields);
if (!CollectionUtils.isEmpty(dimensionsMissing)) {
String errMsg = String.format("指标:%s 缺失必要维度:%s", metric.getName(),
String errMsg = String.format("指标:%s 缺失必要下钻维度:%s", metric.getName(),
dimensionsMissing.stream().map(DimensionResp::getName).collect(Collectors.toList()));
throw new InvalidArgumentException(errMsg);
}
@@ -92,8 +97,9 @@ public class MetricDrillDownChecker {
return true;
}
List<String> relateDimensions = metricResps.stream()
.filter(metric -> !CollectionUtils.isEmpty(metric.getDrillDownDimensions()))
.map(metric -> metric.getDrillDownDimensions().stream()
.map(this::getDrillDownDimensions)
.filter(drillDownDimensions -> !CollectionUtils.isEmpty(drillDownDimensions))
.map(drillDownDimensions -> drillDownDimensions.stream()
.map(DrillDownDimension::getDimensionId).collect(Collectors.toList()))
.flatMap(Collection::stream)
.map(id -> convertDimensionIdToBizName(id, semanticSchemaResp))
@@ -111,7 +117,7 @@ public class MetricDrillDownChecker {
if (metric == null) {
return Lists.newArrayList();
}
List<DrillDownDimension> drillDownDimensions = metric.getDrillDownDimensions();
List<DrillDownDimension> drillDownDimensions = getDrillDownDimensions(metric);
if (CollectionUtils.isEmpty(drillDownDimensions)) {
return Lists.newArrayList();
}
@@ -147,4 +153,8 @@ public class MetricDrillDownChecker {
return dimension.getBizName();
}
private List<DrillDownDimension> getDrillDownDimensions(MetricResp metricResp) {
return metricService.getDrillDownDimension(metricResp.getId());
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
View File

@@ -30,7 +30,7 @@ import java.util.stream.Collectors;
@Slf4j
public class ModelYamlManager {
public static DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) {
public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) {
ModelDetail modelDetail = modelResp.getModelDetail();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -109,8 +109,9 @@ public class DownloadServiceImpl implements DownloadService {
metaFilter.setIds(metricIds);
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
Map<String, List<MetricResp>> metricMap = getMetricMap(metricResps);
List<Long> dimensionIds = metricResps.stream().map(MetricResp::getRelateDimension)
.map(RelateDimension::getDrillDownDimensions).flatMap(Collection::stream)
List<Long> dimensionIds = metricResps.stream()
.map(metricResp -> metricService.getDrillDownDimension(metricResp.getId()))
.flatMap(Collection::stream)
.map(DrillDownDimension::getDimensionId).collect(Collectors.toList());
metaFilter.setIds(dimensionIds);
Map<Long, DimensionResp> dimensionRespMap = dimensionService.getDimensions(metaFilter)

15
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java
View File

@@ -8,13 +8,14 @@ import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.server.service.KnowledgeService;
import com.tencent.supersonic.headless.server.service.ViewService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@@ -68,17 +69,19 @@ public class KnowledgeServiceImpl implements KnowledgeService {
@Override
public List<S2Term> getTerms(String text) {
return HanlpHelper.getTerms(text);
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>());
return HanlpHelper.getTerms(text, modelIdToViewIds);
}
@Override
public List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> viewIds) {
Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds));
return prefixSearchByModel(key, limit, modelIdToViewIds.keySet());
return prefixSearchByModel(key, limit, modelIdToViewIds);
}
public List<HanlpMapResult> prefixSearchByModel(String key, int limit, Set<Long> models) {
return SearchService.prefixSearch(key, limit, models);
public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
Map<Long, List<Long>> modelIdToViewIds) {
return SearchService.prefixSearch(key, limit, modelIdToViewIds);
}
@Override

65
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -5,6 +5,7 @@ import com.alibaba.fastjson.TypeReference;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DataEvent;
@@ -16,6 +17,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
@@ -40,6 +42,12 @@ import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -48,11 +56,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
@Slf4j
@@ -228,28 +231,44 @@ public class MetricServiceImpl implements MetricService {
}
private List<MetricResp> filterByField(List<MetricResp> metricResps, List<String> fields) {
List<MetricResp> metricRespFiltered = Lists.newArrayList();
Set<MetricResp> metricRespFiltered = Sets.newHashSet();
for (MetricResp metricResp : metricResps) {
for (String field : fields) {
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
.stream().map(MetricParam::getId).collect(Collectors.toList());
List<MetricResp> metricById = metricResps.stream()
.filter(metric -> ids.contains(metric.getId()))
.collect(Collectors.toList());
for (MetricResp metric : metricById) {
if (metric.getExpr().contains(field)) {
metricRespFiltered.add(metricResp);
}
}
} else {
if (metricResp.getExpr().contains(field)) {
metricRespFiltered.add(metricResp);
}
filterByField(metricResps, metricResp, fields, metricRespFiltered);
}
return new ArrayList<>(metricRespFiltered);
}
private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
List<String> fields, Set<MetricResp> metricRespFiltered) {
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
.stream().map(MetricParam::getId).collect(Collectors.toList());
List<MetricResp> metricById = metricResps.stream()
.filter(metric -> ids.contains(metric.getId()))
.collect(Collectors.toList());
for (MetricResp metric : metricById) {
if (filterByField(metricResps, metric, fields, metricRespFiltered)) {
metricRespFiltered.add(metricResp);
return true;
}
}
} else if (MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())) {
if (fields.stream().anyMatch(field -> metricResp.getExpr().contains(field))) {
metricRespFiltered.add(metricResp);
return true;
}
} else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
List<MeasureParam> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
List<String> fieldNameDepended = measures.stream().map(MeasureParam::getBizName)
//measure bizName = model bizName_fieldName
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
.collect(Collectors.toList());
if (fields.stream().anyMatch(fieldNameDepended::contains)) {
metricRespFiltered.add(metricResp);
return true;
}
}
return metricRespFiltered;
return false;
}
@Override

43
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
View File

@@ -64,6 +64,13 @@ public class SchemaServiceImpl implements SchemaService {
protected final Cache<String, List<ItemUseResp>> itemUseCache =
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
protected final Cache<ViewFilterReq, List<ViewSchemaResp>> viewSchemaCache =
CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build();
protected final Cache<SchemaFilterReq, SemanticSchemaResp> semanticSchemaCache =
CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build();
private final StatUtils statUtils;
private final ModelService modelService;
private final DimensionService dimensionService;
@@ -91,6 +98,22 @@ public class SchemaServiceImpl implements SchemaService {
@SneakyThrows
@Override
public List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter) {
List<ViewSchemaResp> viewList = viewSchemaCache.getIfPresent(filter);
if (CollectionUtils.isEmpty(viewList)) {
viewList = buildViewSchema(filter);
viewSchemaCache.put(filter, viewList);
}
return viewList;
}
public ViewSchemaResp fetchViewSchema(Long viewId) {
if (viewId == null) {
return null;
}
return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null);
}
public List<ViewSchemaResp> buildViewSchema(ViewFilterReq filter) {
List<ViewSchemaResp> viewSchemaResps = new ArrayList<>();
List<Long> viewIds = filter.getViewIds();
MetaFilter metaFilter = new MetaFilter();
@@ -127,13 +150,6 @@ public class SchemaServiceImpl implements SchemaService {
return viewSchemaResps;
}
public ViewSchemaResp fetchViewSchema(Long viewId) {
if (viewId == null) {
return null;
}
return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null);
}
public List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds) {
List<ModelSchemaResp> modelSchemaResps = Lists.newArrayList();
if (CollectionUtils.isEmpty(modelIds)) {
@@ -258,8 +274,7 @@ public class SchemaServiceImpl implements SchemaService {
return viewService.getViewList(metaFilter);
}
@Override
public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) {
public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) {
SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp();
semanticSchemaResp.setViewId(schemaFilterReq.getViewId());
semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds());
@@ -294,6 +309,16 @@ public class SchemaServiceImpl implements SchemaService {
return semanticSchemaResp;
}
@Override
public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) {
SemanticSchemaResp semanticSchemaResp = semanticSchemaCache.getIfPresent(schemaFilterReq);
if (semanticSchemaResp == null) {
semanticSchemaResp = buildSemanticSchema(schemaFilterReq);
semanticSchemaCache.put(schemaFilterReq, semanticSchemaResp);
}
return semanticSchemaResp;
}
@SneakyThrows
@Override
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {

76
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java
View File

@@ -10,17 +10,29 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.ViewDetail;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO;
import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ViewService;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
@@ -29,12 +41,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
public class ViewServiceImpl
@@ -46,21 +54,33 @@ public class ViewServiceImpl
@Autowired
private DomainService domainService;
@Lazy
@Autowired
private DimensionService dimensionService;
@Lazy
@Autowired
private MetricService metricService;
@Override
public ViewResp save(ViewReq viewReq, User user) {
viewReq.createdBy(user.getName());
ViewDO viewDO = convert(viewReq);
viewDO.setStatus(StatusEnum.ONLINE.getCode());
ViewResp viewResp = convert(viewDO);
conflictCheck(viewResp);
save(viewDO);
return convert(viewDO);
return viewResp;
}
@Override
public ViewResp update(ViewReq viewReq, User user) {
viewReq.updatedBy(user.getName());
ViewDO viewDO = convert(viewReq);
ViewResp viewResp = convert(viewDO);
conflictCheck(viewResp);
updateById(viewDO);
return convert(viewDO);
return viewResp;
}
@Override
@@ -78,6 +98,9 @@ public class ViewServiceImpl
if (!CollectionUtils.isEmpty(metaFilter.getIds())) {
wrapper.lambda().in(ViewDO::getId, metaFilter.getIds());
}
if (metaFilter.getStatus() != null) {
wrapper.lambda().eq(ViewDO::getStatus, metaFilter.getStatus());
}
wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode());
return list(wrapper).stream().map(this::convert).collect(Collectors.toList());
}
@@ -175,4 +198,43 @@ public class ViewServiceImpl
viewResp -> viewResp.getAllModels().stream().map(modelId -> Pair.of(modelId, viewResp.getId())))
.collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList())));
}
private void conflictCheck(ViewResp viewResp) {
List<Long> allDimensionIds = viewResp.getAllDimensions();
List<Long> allMetricIds = viewResp.getAllMetrics();
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(allDimensionIds);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
metaFilter.setIds(allMetricIds);
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<String> duplicateDimensionNames = findDuplicates(dimensionResps, DimensionResp::getName);
List<String> duplicateDimensionBizNames = findDuplicates(dimensionResps, DimensionResp::getBizName);
List<String> duplicateMetricNames = findDuplicates(metricResps, MetricResp::getName);
List<String> duplicateMetricBizNames = findDuplicates(metricResps, MetricResp::getBizName);
if (!duplicateDimensionNames.isEmpty()) {
throw new InvalidArgumentException("存在相同的维度名: " + duplicateDimensionNames);
}
if (!duplicateDimensionBizNames.isEmpty()) {
throw new InvalidArgumentException("存在相同的维度英文名: " + duplicateDimensionBizNames);
}
if (!duplicateMetricNames.isEmpty()) {
throw new InvalidArgumentException("存在相同的指标名: " + duplicateMetricNames);
}
if (!duplicateMetricBizNames.isEmpty()) {
throw new InvalidArgumentException("存在相同的指标英文名: " + duplicateMetricBizNames);
}
}
private <T, R> List<String> findDuplicates(List<T> list, Function<T, R> keyExtractor) {
return list.stream()
.collect(Collectors.groupingBy(keyExtractor, Collectors.counting()))
.entrySet().stream()
.filter(entry -> entry.getValue() > 1)
.map(Map.Entry::getKey)
.map(Object::toString)
.collect(Collectors.toList());
}
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
View File

@@ -72,6 +72,7 @@ public class MetricConverter {
ModelResp modelResp = modelMap.get(metricDO.getModelId());
if (modelResp != null) {
metricResp.setModelName(modelResp.getName());
metricResp.setModelBizName(modelResp.getBizName());
metricResp.setDomainId(modelResp.getDomainId());
}
metricResp.setIsCollect(collect != null && collect.contains(metricDO.getId()));

2
launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java
View File

@@ -224,7 +224,7 @@ public class BenchMarkDemoDataLoader {
new ViewModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList()),
new ViewModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList()),
new ViewModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList()),
new ViewModelConfig(8L, Lists.newArrayList(13L, 14L, 15L), Lists.newArrayList(8L, 9L))
new ViewModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L))
);
ViewDetail viewDetail = new ViewDetail();
viewDetail.setViewModelConfigs(viewModelConfigs);

22
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt
View File

@@ -5,14 +5,14 @@ dean _1_2 36
john _1_2 50
jack _1_2 38
admin _1_2 70
周杰伦 _2_7 100
陈奕迅 _2_7 100
林俊杰 _2_7 100
张碧晨 _2_7 100
程响 _2_7 100
Taylor#Swift _2_7 100
内地 _2_4 100
欧美 _2_4 100
港台 _2_4 100
流行 _2_6 100
国风 _2_6 100
周杰伦 _4_7 100
陈奕迅 _4_7 100
林俊杰 _4_7 100
张碧晨 _4_7 100
程响 _4_7 100
Taylor#Swift _4_7 100
内地 _4_4 100
欧美 _4_4 100
港台 _4_4 100
流行 _4_6 100
国风 _4_6 100

12
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt
View File

@@ -1,6 +1,6 @@
p1 _2_3 52
p2 _2_3 47
p3 _2_3 31
p4 _2_3 36
p5 _2_3 50
p6 _2_3 38
p1 _3_3 52
p2 _3_3 47
p3 _3_3 31
p4 _3_3 36
p5 _3_3 50
p6 _3_3 38

18
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt
View File

@@ -1,9 +1,9 @@
周杰伦 _2_7 9000
周深 _2_7 8000
周传雄 _2_7 7000
周华建 _2_7 6000
陈奕迅 _2_7 8000
林俊杰 _2_7 7000
张碧晨 _2_7 7000
程响 _2_7 7000
Taylor#Swift _2_7 7000
周杰伦 _4_7 9000
周深 _4_7 8000
周传雄 _4_7 7000
周华建 _4_7 6000
陈奕迅 _4_7 8000
林俊杰 _4_7 7000
张碧晨 _4_7 7000
程响 _4_7 7000
Taylor#Swift _4_7 7000

8
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt
View File

@@ -1,4 +1,4 @@
美国 _3_8 1
加拿大 _3_8 1
锡尔赫特、吉大港、库斯蒂亚 _3_8 1
孟加拉国 _3_8 3
美国 _5_8 1
加拿大 _5_8 1
锡尔赫特、吉大港、库斯蒂亚 _5_8 1
孟加拉国 _5_8 3

12
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt
View File

@@ -1,6 +1,6 @@
现代 _3_9 1
tagore _3_9 1
蓝调 _3_9 1
流行 _3_9 1
民间 _3_9 1
nazrul _3_9 1
现代 _5_9 1
tagore _5_9 1
蓝调 _5_9 1
流行 _5_9 1
民间 _5_9 1
nazrul _5_9 1

8
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt
View File

@@ -1,4 +1,4 @@
美国 _3_11 1
印度 _3_11 2
英国 _3_11 1
孟加拉国 _3_11 2
美国 _6_11 1
印度 _6_11 2
英国 _6_11 1
孟加拉国 _6_11 2

4
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt
View File

@@ -1,2 +1,2 @@
男性 _3_12 3
女性 _3_12 3
男性 _6_12 3
女性 _6_12 3

4
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt
View File

@@ -1,2 +1,2 @@
mp4 _3_14 4
mp3 _3_14 2
mp4 _7_14 4
mp3 _7_14 2

8
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt
View File

@@ -1,4 +1,4 @@
美国 _3_17 1
印度 _3_17 2
英国 _3_17 1
孟加拉国 _3_17 2
美国 _8_17 1
印度 _8_17 2
英国 _8_17 1
孟加拉国 _8_17 2

4
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt
View File

@@ -1,2 +1,2 @@
英文 _3_18 2
孟加拉语 _3_18 4
英文 _8_18 2
孟加拉语 _8_18 4

12
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt
View File

@@ -1,6 +1,6 @@
阿米·奥帕尔·霍伊 _3_16 1
我的爱 _3_16 1
打败它 _3_16 1
阿杰伊阿卡什 _3_16 1
Tumi#长袍#尼罗布 _3_16 1
舒克诺#帕塔尔#努普尔#帕埃 _3_16 1
阿米·奥帕尔·霍伊 _8_16 1
我的爱 _8_16 1
打败它 _8_16 1
阿杰伊阿卡什 _8_16 1
Tumi#长袍#尼罗布 _8_16 1
舒克诺#帕塔尔#努普尔#帕埃 _8_16 1

33
launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java Normal file
View File

@@ -0,0 +1,33 @@
package com.tencent.supersonic.headless;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp;
import com.tencent.supersonic.headless.server.service.ModelService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
public class ModelSchemaTest extends BaseTest {
@Autowired
private ModelService modelService;
@Test
void testGetUnAvailableItem() {
FieldRemovedReq fieldRemovedReq = new FieldRemovedReq();
fieldRemovedReq.setModelId(2L);
fieldRemovedReq.setFields(Lists.newArrayList("pv"));
UnAvailableItemResp unAvailableItemResp = modelService.getUnAvailableItem(fieldRemovedReq);
List<Long> expectedUnAvailableMetricId = Lists.newArrayList(1L, 3L);
List<Long> actualUnAvailableMetricId = unAvailableItemResp.getMetricResps()
.stream().map(MetricResp::getId).sorted(Comparator.naturalOrder()).collect(Collectors.toList());
Assertions.assertEquals(expectedUnAvailableMetricId, actualUnAvailableMetricId);
}
}

22
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt
View File

@@ -5,14 +5,14 @@ dean _1_2 36
john _1_2 50
jack _1_2 38
admin _1_2 70
周杰伦 _2_7 100
陈奕迅 _2_7 100
林俊杰 _2_7 100
张碧晨 _2_7 100
程响 _2_7 100
Taylor#Swift _2_7 100
内地 _2_4 100
欧美 _2_4 100
港台 _2_4 100
流行 _2_6 100
国风 _2_6 100
周杰伦 _4_7 100
陈奕迅 _4_7 100
林俊杰 _4_7 100
张碧晨 _4_7 100
程响 _4_7 100
Taylor#Swift _4_7 100
内地 _4_4 100
欧美 _4_4 100
港台 _4_4 100
流行 _4_6 100
国风 _4_6 100

12
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt
View File

@@ -1,6 +1,6 @@
p1 _2_3 52
p2 _2_3 47
p3 _2_3 31
p4 _2_3 36
p5 _2_3 50
p6 _2_3 38
p1 _3_3 52
p2 _3_3 47
p3 _3_3 31
p4 _3_3 36
p5 _3_3 50
p6 _3_3 38

18
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt
View File

@@ -1,9 +1,9 @@
周杰伦 _2_7 9000
周深 _2_7 8000
周传雄 _2_7 7000
周华建 _2_7 6000
陈奕迅 _2_7 8000
林俊杰 _2_7 7000
张碧晨 _2_7 7000
程响 _2_7 7000
Taylor#Swift _2_7 7000
周杰伦 _4_7 9000
周深 _4_7 8000
周传雄 _4_7 7000
周华建 _4_7 6000
陈奕迅 _4_7 8000
林俊杰 _4_7 7000
张碧晨 _4_7 7000
程响 _4_7 7000
Taylor#Swift _4_7 7000

2
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt
View File

@@ -1,4 +1,4 @@
美国 _5_8 1
加拿大 _5_8 1
锡尔赫特、吉大港、库斯蒂亚 _5_8 1
孟加拉国 _5_8 3
孟加拉国 _5_8 3

2
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt
View File

@@ -3,4 +3,4 @@ tagore _5_9 1
蓝调 _5_9 1
流行 _5_9 1
民间 _5_9 1
nazrul _5_9 1
nazrul _5_9 1

8
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt
View File

@@ -1,4 +1,4 @@
美国 _6_10 1
印度 _6_10 2
英国 _6_10 1
孟加拉国 _6_10 2
美国 _6_11 1
印度 _6_11 2
英国 _6_11 1
孟加拉国 _6_11 2

4
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt
View File

@@ -1,2 +1,2 @@
男性 _6_11 3
女性 _6_11 3
男性 _6_12 3
女性 _6_12 3

2
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt
View File

@@ -1,2 +1,2 @@
mp4 _7_14 4
mp3 _7_14 2
mp3 _7_14 2

8
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt
View File

@@ -1,4 +1,4 @@
美国 _8_16 1
印度 _8_16 2
英国 _8_16 1
孟加拉国 _8_16 2
美国 _8_17 1
印度 _8_17 2
英国 _8_17 1
孟加拉国 _8_17 2

4
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt
View File

@@ -1,2 +1,2 @@
英文 _8_17 2
孟加拉语 _8_17 4
英文 _8_18 2
孟加拉语 _8_18 4

12
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt
View File

@@ -1,6 +1,6 @@
阿米·奥帕尔·霍伊 _8_19 1
我的爱 _8_19 1
打败它 _8_19 1
阿杰伊阿卡什 _8_19 1
Tumi#长袍#尼罗布 _8_19 1
舒克诺#帕塔尔#努普尔#帕埃 _8_19 1
阿米·奥帕尔·霍伊 _8_16 1
我的爱 _8_16 1
打败它 _8_16 1
阿杰伊阿卡什 _8_16 1
Tumi#长袍#尼罗布 _8_16 1
舒克诺#帕塔尔#努普尔#帕埃 _8_16 1