From e95a5282193bfce8b23d466788b1b672de751805 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Fri, 23 Feb 2024 10:29:21 +0800 Subject: [PATCH] (improvement)(headless) transfer term nature modelId to viewId before providing it to chat and put the modelId of metadata into the dict word instead of viewId (#739) Co-authored-by: jolunoluo --- .../chat/core/corrector/GroupByCorrector.java | 3 +- .../chat/core/mapper/EmbeddingMapper.java | 6 +- .../core/parser/sql/llm/S2SqlDateHelper.java | 29 +++---- .../query/semantic/ViewSchemaBuilder.java | 5 ++ .../execute/DimensionRecommendProcessor.java | 8 +- .../chat/server/service/SemanticService.java | 2 +- .../service/impl/ConfigServiceImpl.java | 2 +- .../service/impl/RecommendServiceImpl.java | 2 +- .../chat/server/service/impl/WordService.java | 19 ++++- .../headless/api/pojo/ModelDetail.java | 8 +- .../supersonic/headless/api/pojo/Param.java | 13 +--- .../headless/api/pojo/SchemaElement.java | 5 +- .../api/pojo/response/MetricResp.java | 2 + .../core/knowledge/SearchService.java | 29 ++++--- .../builder/DimensionWordBuilder.java | 6 +- .../knowledge/builder/EntityWordBuilder.java | 4 +- .../knowledge/builder/MetricWordBuilder.java | 6 +- .../knowledge/builder/ValueWordBuilder.java | 4 +- .../core/knowledge/helper/HanlpHelper.java | 41 ++++++---- .../core/knowledge/helper/NatureHelper.java | 41 +++++++++- .../server/aspect/MetricDrillDownChecker.java | 18 ++++- .../server/manager/ModelYamlManager.java | 2 +- .../service/impl/DownloadServiceImpl.java | 5 +- .../service/impl/KnowledgeServiceImpl.java | 15 ++-- .../service/impl/MetricServiceImpl.java | 65 ++++++++++------ .../service/impl/SchemaServiceImpl.java | 43 ++++++++--- .../server/service/impl/ViewServiceImpl.java | 76 +++++++++++++++++-- .../server/utils/MetricConverter.java | 1 + .../supersonic/BenchMarkDemoDataLoader.java | 2 +- .../data/dictionary/custom/DimValue_1_2.txt | 22 +++--- .../data/dictionary/custom/DimValue_1_3.txt | 12 +-- .../data/dictionary/custom/DimValue_4_9.txt | 18 ++--- .../data/dictionary/custom/DimValue_5_10.txt | 8 +- .../data/dictionary/custom/DimValue_5_11.txt | 12 +-- .../data/dictionary/custom/DimValue_6_12.txt | 8 +- .../data/dictionary/custom/DimValue_6_13.txt | 4 +- .../data/dictionary/custom/DimValue_7_16.txt | 4 +- .../data/dictionary/custom/DimValue_8_18.txt | 8 +- .../data/dictionary/custom/DimValue_8_19.txt | 4 +- .../data/dictionary/custom/DimValue_8_21.txt | 12 +-- .../supersonic/headless/ModelSchemaTest.java | 33 ++++++++ .../data/dictionary/custom/DimValue_1_2.txt | 22 +++--- .../data/dictionary/custom/DimValue_1_3.txt | 12 +-- .../data/dictionary/custom/DimValue_4_9.txt | 18 ++--- .../data/dictionary/custom/DimValue_5_10.txt | 2 +- .../data/dictionary/custom/DimValue_5_11.txt | 2 +- .../data/dictionary/custom/DimValue_6_12.txt | 8 +- .../data/dictionary/custom/DimValue_6_13.txt | 4 +- .../data/dictionary/custom/DimValue_7_16.txt | 2 +- .../data/dictionary/custom/DimValue_8_18.txt | 8 +- .../data/dictionary/custom/DimValue_8_19.txt | 4 +- .../data/dictionary/custom/DimValue_8_21.txt | 12 +-- 52 files changed, 456 insertions(+), 245 deletions(-) create mode 100644 launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java index fb8172ba7..a88c9c945 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/GroupByCorrector.java @@ -45,7 +45,8 @@ public class GroupByCorrector extends BaseSemanticCorrector { ViewResp viewResp = viewService.getView(viewId); List modelIds = viewResp.getViewDetail().getViewModelConfigs().stream().map(config -> config.getId()) .collect(Collectors.toList()); - MetaFilter metaFilter = new MetaFilter(modelIds); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setIds(modelIds); List modelRespList = modelService.getModelList(metaFilter); for (ModelResp modelResp : modelRespList) { List dimList = modelResp.getModelDetail().getDimensions(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java index 296d951f8..00d4a7b5b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java @@ -10,9 +10,10 @@ import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult; import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; +import com.tencent.supersonic.headless.server.service.KnowledgeService; +import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.Objects; -import lombok.extern.slf4j.Slf4j; /*** * A mapper that recognizes schema elements with vector embedding. @@ -24,7 +25,8 @@ public class EmbeddingMapper extends BaseMapper { public void doMap(QueryContext queryContext) { //1. query from embedding by queryText String queryText = queryContext.getQueryText(); - List terms = HanlpHelper.getTerms(queryText); + KnowledgeService knowledgeService = ContextUtils.getBean(KnowledgeService.class); + List terms = knowledgeService.getTerms(queryText); EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class); List matchResults = matchStrategy.getMatches(queryContext, terms); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java index 61a51f49d..17589b566 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/S2SqlDateHelper.java @@ -1,36 +1,27 @@ package com.tencent.supersonic.chat.core.parser.sql.llm; -import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; -import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; import com.tencent.supersonic.chat.core.pojo.QueryContext; import com.tencent.supersonic.common.util.DatePeriodEnum; import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; + import java.util.Objects; public class S2SqlDateHelper { - public static String getReferenceDate(QueryContext queryContext, Long modelId) { + public static String getReferenceDate(QueryContext queryContext, Long viewId) { String defaultDate = DateUtils.getBeforeDate(0); - if (Objects.isNull(modelId)) { + if (Objects.isNull(viewId)) { return defaultDate; } - ChatConfigFilter filter = new ChatConfigFilter(); - filter.setModelId(modelId); - ChatConfigRichResp chatConfigRichResp = queryContext.getModelIdToChatRichConfig().get(modelId); - - if (Objects.isNull(chatConfigRichResp)) { + ViewSchema viewSchema = queryContext.getSemanticSchema().getViewSchemaMap().get(viewId); + if (viewSchema == null || viewSchema.getTagTypeTimeDefaultConfig() == null) { return defaultDate; } - if (Objects.isNull(chatConfigRichResp.getChatDetailRichConfig()) || Objects.isNull( - chatConfigRichResp.getChatDetailRichConfig().getChatDefaultConfig())) { - return defaultDate; - } - - ChatDefaultRichConfigResp chatDefaultConfig = chatConfigRichResp.getChatDetailRichConfig() - .getChatDefaultConfig(); - Integer unit = chatDefaultConfig.getUnit(); - String period = chatDefaultConfig.getPeriod(); + TimeDefaultConfig tagTypeTimeDefaultConfig = viewSchema.getTagTypeTimeDefaultConfig(); + Integer unit = tagTypeTimeDefaultConfig.getUnit(); + String period = tagTypeTimeDefaultConfig.getPeriod(); if (Objects.nonNull(unit)) { // If the unit is set to less than 0, then do not add relative date. if (unit < 0) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java index 44d9522a8..1bbc151f6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java @@ -44,6 +44,7 @@ public class ViewSchemaBuilder { SchemaElement metricToAdd = SchemaElement.builder() .view(resp.getId()) + .model(metric.getModelId()) .id(metric.getId()) .name(metric.getName()) .bizName(metric.getBizName()) @@ -84,6 +85,7 @@ public class ViewSchemaBuilder { } SchemaElement dimToAdd = SchemaElement.builder() .view(resp.getId()) + .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -96,6 +98,7 @@ public class ViewSchemaBuilder { SchemaElement dimValueToAdd = SchemaElement.builder() .view(resp.getId()) + .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -107,6 +110,7 @@ public class ViewSchemaBuilder { if (dim.getIsTag() == 1) { SchemaElement tagToAdd = SchemaElement.builder() .view(resp.getId()) + .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) @@ -126,6 +130,7 @@ public class ViewSchemaBuilder { if (dim != null) { SchemaElement entity = SchemaElement.builder() .view(resp.getId()) + .model(dim.getModelId()) .id(dim.getId()) .name(dim.getName()) .bizName(dim.getBizName()) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java index 84035877f..5c599c81e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java @@ -38,11 +38,11 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { queryResult.setRecommendedDimensions(dimensionRecommended); } - private List getDimensions(Long metricId, Long modelId) { + private List getDimensions(Long metricId, Long viewId) { SemanticService semanticService = ContextUtils.getBean(SemanticService.class); - ViewSchema modelSchema = semanticService.getModelSchema(modelId); + ViewSchema viewSchema = semanticService.getViewSchema(viewId); List drillDownDimensions = Lists.newArrayList(); - Set metricElements = modelSchema.getMetrics(); + Set metricElements = viewSchema.getMetrics(); if (!CollectionUtils.isEmpty(metricElements)) { Optional metric = metricElements.stream().filter(schemaElement -> metricId.equals(schemaElement.getId()) @@ -54,7 +54,7 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor { } } final List drillDownDimensionsFinal = drillDownDimensions; - return modelSchema.getDimensions().stream() + return viewSchema.getDimensions().stream() .filter(dim -> filterDimension(drillDownDimensionsFinal, dim)) .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(recommend_dimension_size) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java index 3b5e5f60f..ec9423da2 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java @@ -50,7 +50,7 @@ public class SemanticService { return schemaService.getSemanticSchema(); } - public ViewSchema getModelSchema(Long id) { + public ViewSchema getViewSchema(Long id) { return schemaService.getViewSchema(id); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index 5fb1c43ae..d3b6b4cc2 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -219,7 +219,7 @@ public class ConfigServiceImpl implements ConfigService { } BeanUtils.copyProperties(chatConfigResp, chatConfigRich); - ViewSchema viewSchema = semanticService.getModelSchema(modelId); + ViewSchema viewSchema = semanticService.getViewSchema(modelId); if (viewSchema == null) { return chatConfigRich; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java index fd9cf56e3..9d76b7bca 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java @@ -48,7 +48,7 @@ public class RecommendServiceImpl implements RecommendService { if (Objects.isNull(modelId)) { return new RecommendResp(); } - ViewSchema modelSchema = semanticService.getModelSchema(modelId); + ViewSchema modelSchema = semanticService.getViewSchema(modelId); if (Objects.isNull(modelSchema)) { return new RecommendResp(); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java index fce843828..14e2c5b90 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java @@ -1,17 +1,19 @@ package com.tencent.supersonic.chat.server.service.impl; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.headless.core.knowledge.DictWord; -import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory; import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter; import com.tencent.supersonic.chat.core.utils.ComponentFactory; import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.core.knowledge.DictWord; +import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; @Service @@ -28,7 +30,6 @@ public class WordService { addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words); addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words); - addWordsByType(DictWordType.VIEW, semanticSchema.getViews(), words); addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words); addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words); @@ -36,6 +37,7 @@ public class WordService { } private void addWordsByType(DictWordType value, List metas, List natures) { + metas = distinct(metas); List natureList = WordBuilderFactory.get(value).getDictWords(metas); log.debug("nature type:{} , nature size:{}", value.name(), natureList.size()); natures.addAll(natureList); @@ -48,4 +50,13 @@ public class WordService { public void setPreDictWords(List preDictWords) { this.preDictWords = preDictWords; } + + private List distinct(List metas) { + return metas.stream() + .collect(Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1)) + .values() + .stream() + .collect(Collectors.toList()); + } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index ba2960953..958272840 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -19,13 +19,13 @@ public class ModelDetail { private String tableQuery; - private List identifiers; + private List identifiers = Lists.newArrayList(); - private List dimensions; + private List dimensions = Lists.newArrayList(); - private List measures; + private List measures = Lists.newArrayList(); - private List fields; + private List fields = Lists.newArrayList(); public String getSqlQuery() { if (StringUtils.isNotBlank(sqlQuery) && sqlQuery.endsWith(";")) { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java index 6dec090f5..dbf36dce5 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Param.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo; import lombok.Data; import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; @Data public class Param { @@ -10,7 +11,7 @@ public class Param { @NotBlank(message = "Invald parameter name") private String name; - @NotBlank(message = "Invalid parameter value") + @NotNull(message = "Invalid parameter value") private String value; public Param() { @@ -21,14 +22,4 @@ public class Param { this.value = value; } - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("{"); - sb.append("\"name\":\"") - .append(name).append('\"'); - sb.append(",\"value\":\"") - .append(value).append('\"'); - sb.append('}'); - return sb.toString(); - } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java index 22e50df9e..f80f5cda5 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo; import com.google.common.base.Objects; -import com.google.common.collect.Lists; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -19,6 +18,7 @@ import java.util.List; public class SchemaElement implements Serializable { private Long view; + private Long model; private Long id; private String name; private String bizName; @@ -52,7 +52,4 @@ public class SchemaElement implements Serializable { return Objects.hashCode(view, id, name, bizName, type); } - public List getModelNames() { - return Lists.newArrayList(name); - } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java index 8f0161a3d..e92752ec1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java @@ -30,6 +30,8 @@ public class MetricResp extends SchemaItem { private Long domainId; + private String modelBizName; + private String modelName; //ATOMIC DERIVED diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java index 61809f67e..7e75f1ffb 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java @@ -6,8 +6,15 @@ import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.common.pojo.enums.DictWordType; +import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; +import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -17,11 +24,6 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; -import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; - @Slf4j public class SearchService { @@ -39,14 +41,14 @@ public class SearchService { * @param key * @return */ - public static List prefixSearch(String key, int limit, Set detectModelIds) { - return prefixSearch(key, limit, trie, detectModelIds); + public static List prefixSearch(String key, int limit, Map> modelIdToViewIds) { + return prefixSearch(key, limit, trie, modelIdToViewIds); } public static List prefixSearch(String key, int limit, BinTrie> binTrie, - Set detectModelIds) { - Set>> result = prefixSearchLimit(key, limit, binTrie, detectModelIds); - return result.stream().map( + Map> modelIdToViewIds) { + Set>> result = prefixSearchLimit(key, limit, binTrie, modelIdToViewIds.keySet()); + List hanlpMapResults = result.stream().map( entry -> { String name = entry.getKey().replace("#", " "); return new HanlpMapResult(name, entry.getValue(), key); @@ -54,6 +56,13 @@ public class SearchService { ).sorted((a, b) -> -(b.getName().length() - a.getName().length())) .limit(SEARCH_SIZE) .collect(Collectors.toList()); + for (HanlpMapResult hanlpMapResult : hanlpMapResults) { + List natures = hanlpMapResult.getNatures().stream() + .map(nature -> NatureHelper.changeModel2View(nature, modelIdToViewIds)) + .flatMap(Collection::stream).collect(Collectors.toList()); + hanlpMapResult.setNatures(natures); + } + return hanlpMapResults; } /*** diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java index 96247a048..9d662fab1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java @@ -38,11 +38,11 @@ public class DimensionWordBuilder extends BaseWordBuilder { private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) { DictWord dictWord = new DictWord(); dictWord.setWord(word); - Long viewId = schemaElement.getView(); - String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + Long modelId = schemaElement.getModel(); + String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.DIMENSION.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.SUFFIX.getType() + DictWordType.DIMENSION.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java index 8ce190e2c..145eb29c1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java @@ -27,8 +27,8 @@ public class EntityWordBuilder extends BaseWordBuilder { return result; } - Long view = schemaElement.getView(); - String nature = DictWordType.NATURE_SPILT + view + DictWordType.NATURE_SPILT + schemaElement.getId() + Long modelId = schemaElement.getModel(); + String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.ENTITY.getType(); if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java index c54fca038..fe6af5ef6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java @@ -38,11 +38,11 @@ public class MetricWordBuilder extends BaseWordBuilder { private DictWord getOnwWordNature(String word, SchemaElement schemaElement, boolean isSuffix) { DictWord dictWord = new DictWord(); dictWord.setWord(word); - Long viewId = schemaElement.getView(); - String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + Long modelId = schemaElement.getModel(); + String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.METRIC.getType(); if (isSuffix) { - nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId() + nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.SUFFIX.getType() + DictWordType.METRIC.getType(); } dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java index 73f1bbbc0..95bbb39b0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java @@ -27,8 +27,8 @@ public class ValueWordBuilder extends BaseWordBuilder { schemaElement.getAlias().stream().forEach(value -> { DictWord dictWord = new DictWord(); - Long viewId = schemaElement.getView(); - String nature = DictWordType.NATURE_SPILT + viewId + DictWordType.NATURE_SPILT + schemaElement.getId(); + Long modelId = schemaElement.getModel(); + String nature = DictWordType.NATURE_SPILT + modelId + DictWordType.NATURE_SPILT + schemaElement.getId(); dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY, nature)); dictWord.setWord(value); result.add(dictWord); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java index 2883babce..c4a5b510d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java @@ -1,34 +1,36 @@ package com.tencent.supersonic.headless.core.knowledge.helper; -import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; - +import com.google.common.collect.Lists; import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.corpus.tag.Nature; import com.hankcs.hanlp.dictionary.CoreDictionary; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; import com.hankcs.hanlp.seg.Segment; +import com.hankcs.hanlp.seg.common.Term; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.core.knowledge.DictWord; import com.tencent.supersonic.headless.core.knowledge.HadoopFileIOAdapter; import com.tencent.supersonic.headless.core.knowledge.MapResult; import com.tencent.supersonic.headless.core.knowledge.MultiCustomDictionary; import com.tencent.supersonic.headless.core.knowledge.SearchService; -import com.tencent.supersonic.common.pojo.enums.DictWordType; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ResourceUtils; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; -import com.hankcs.hanlp.seg.common.Term; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.util.CollectionUtils; -import org.springframework.util.ResourceUtils; +import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; /** * HanLP helper @@ -212,18 +214,25 @@ public class HanlpHelper { } } - public static List getTerms(String text) { + public static List getTerms(String text, Map> modelIdToViewIds) { return getSegment().seg(text.toLowerCase()).stream() .filter(term -> term.getNature().startsWith(DictWordType.NATURE_SPILT)) - .map(term -> transform2ApiTerm(term)) + .map(term -> transform2ApiTerm(term, modelIdToViewIds)) + .flatMap(Collection::stream) .collect(Collectors.toList()); } - public static S2Term transform2ApiTerm(Term term) { - S2Term knowledgeTerm = new S2Term(); - BeanUtils.copyProperties(term, knowledgeTerm); - knowledgeTerm.setFrequency(term.getFrequency()); - return knowledgeTerm; + public static List transform2ApiTerm(Term term, Map> modelIdToViewIds) { + List s2Terms = Lists.newArrayList(); + List natures = NatureHelper.changeModel2View(String.valueOf(term.getNature()), modelIdToViewIds); + for (String nature : natures) { + S2Term s2Term = new S2Term(); + BeanUtils.copyProperties(term, s2Term); + s2Term.setNature(Nature.create(nature)); + s2Term.setFrequency(term.getFrequency()); + s2Terms.add(s2Term); + } + return s2Terms; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java index a37d35b92..7f88d626b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java @@ -1,12 +1,14 @@ package com.tencent.supersonic.headless.core.knowledge.helper; +import com.google.common.collect.Lists; import com.hankcs.hanlp.corpus.tag.Nature; +import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.response.S2Term; import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat; -import com.tencent.supersonic.common.pojo.enums.DictWordType; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.Comparator; @@ -81,6 +83,43 @@ public class NatureHelper { return null; } + private static Long getModelId(String nature) { + try { + String[] split = nature.split(DictWordType.NATURE_SPILT); + if (split.length <= 1) { + return null; + } + return Long.valueOf(split[1]); + } catch (NumberFormatException e) { + log.error("", e); + } + return null; + } + + private static Nature changeModel2View(String nature, Long viewId) { + try { + String[] split = nature.split(DictWordType.NATURE_SPILT); + if (split.length <= 1) { + return null; + } + split[1] = String.valueOf(viewId); + return Nature.create(StringUtils.join(split, DictWordType.NATURE_SPILT)); + } catch (NumberFormatException e) { + log.error("", e); + } + return null; + } + + public static List changeModel2View(String nature, Map> modelIdToViewIds) { + Long modelId = getModelId(nature); + List viewIds = modelIdToViewIds.get(modelId); + if (CollectionUtils.isEmpty(viewIds)) { + return Lists.newArrayList(); + } + return viewIds.stream().map(viewId -> String.valueOf(changeModel2View(nature, viewId))) + .collect(Collectors.toList()); + } + public static boolean isDimensionValueViewId(String nature) { if (StringUtils.isEmpty(nature)) { return false; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java index 7eeeb62ea..2f0b4f57b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java @@ -11,11 +11,13 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.server.service.MetricService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.util.Collection; @@ -28,6 +30,9 @@ import java.util.stream.Collectors; @Slf4j public class MetricDrillDownChecker { + @Autowired + private MetricService metricService; + @Around("execution(* com.tencent.supersonic.headless.core.parser.QueryParser.parse(..))") public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { Object[] objects = joinPoint.getArgs(); @@ -52,7 +57,7 @@ public class MetricDrillDownChecker { List necessaryDimensions = getNecessaryDimensions(metric, semanticSchemaResp); List dimensionsMissing = getNecessaryDimensionMissing(necessaryDimensions, dimensionFields); if (!CollectionUtils.isEmpty(dimensionsMissing)) { - String errMsg = String.format("指标:%s 缺失必要维度:%s", metric.getName(), + String errMsg = String.format("指标:%s 缺失必要下钻维度:%s", metric.getName(), dimensionsMissing.stream().map(DimensionResp::getName).collect(Collectors.toList())); throw new InvalidArgumentException(errMsg); } @@ -92,8 +97,9 @@ public class MetricDrillDownChecker { return true; } List relateDimensions = metricResps.stream() - .filter(metric -> !CollectionUtils.isEmpty(metric.getDrillDownDimensions())) - .map(metric -> metric.getDrillDownDimensions().stream() + .map(this::getDrillDownDimensions) + .filter(drillDownDimensions -> !CollectionUtils.isEmpty(drillDownDimensions)) + .map(drillDownDimensions -> drillDownDimensions.stream() .map(DrillDownDimension::getDimensionId).collect(Collectors.toList())) .flatMap(Collection::stream) .map(id -> convertDimensionIdToBizName(id, semanticSchemaResp)) @@ -111,7 +117,7 @@ public class MetricDrillDownChecker { if (metric == null) { return Lists.newArrayList(); } - List drillDownDimensions = metric.getDrillDownDimensions(); + List drillDownDimensions = getDrillDownDimensions(metric); if (CollectionUtils.isEmpty(drillDownDimensions)) { return Lists.newArrayList(); } @@ -147,4 +153,8 @@ public class MetricDrillDownChecker { return dimension.getBizName(); } + private List getDrillDownDimensions(MetricResp metricResp) { + return metricService.getDrillDownDimension(metricResp.getId()); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index ee0bc723e..b73a758ac 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; @Slf4j public class ModelYamlManager { - public static DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) { + public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) { ModelDetail modelDetail = modelResp.getModelDetail(); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java index 974670442..617de4893 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java @@ -109,8 +109,9 @@ public class DownloadServiceImpl implements DownloadService { metaFilter.setIds(metricIds); List metricResps = metricService.getMetrics(metaFilter); Map> metricMap = getMetricMap(metricResps); - List dimensionIds = metricResps.stream().map(MetricResp::getRelateDimension) - .map(RelateDimension::getDrillDownDimensions).flatMap(Collection::stream) + List dimensionIds = metricResps.stream() + .map(metricResp -> metricService.getDrillDownDimension(metricResp.getId())) + .flatMap(Collection::stream) .map(DrillDownDimension::getDimensionId).collect(Collectors.toList()); metaFilter.setIds(dimensionIds); Map dimensionRespMap = dimensionService.getDimensions(metaFilter) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java index 5c517feb9..67b85cca6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java @@ -8,13 +8,14 @@ import com.tencent.supersonic.headless.core.knowledge.SearchService; import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.server.service.KnowledgeService; import com.tencent.supersonic.headless.server.service.ViewService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; @Service @Slf4j @@ -68,17 +69,19 @@ public class KnowledgeServiceImpl implements KnowledgeService { @Override public List getTerms(String text) { - return HanlpHelper.getTerms(text); + Map> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>()); + return HanlpHelper.getTerms(text, modelIdToViewIds); } @Override public List prefixSearch(String key, int limit, Set viewIds) { Map> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds)); - return prefixSearchByModel(key, limit, modelIdToViewIds.keySet()); + return prefixSearchByModel(key, limit, modelIdToViewIds); } - public List prefixSearchByModel(String key, int limit, Set models) { - return SearchService.prefixSearch(key, limit, models); + public List prefixSearchByModel(String key, int limit, + Map> modelIdToViewIds) { + return SearchService.prefixSearch(key, limit, modelIdToViewIds); } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index ac77d5c90..171d71ee3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -5,6 +5,7 @@ import com.alibaba.fastjson.TypeReference; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DataEvent; @@ -16,6 +17,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.ChatGptHelper; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; +import com.tencent.supersonic.headless.api.pojo.MeasureParam; import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; @@ -40,6 +42,12 @@ import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.MetricCheckUtils; import com.tencent.supersonic.headless.server.utils.MetricConverter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -48,11 +56,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Service @Slf4j @@ -228,28 +231,44 @@ public class MetricServiceImpl implements MetricService { } private List filterByField(List metricResps, List fields) { - List metricRespFiltered = Lists.newArrayList(); + Set metricRespFiltered = Sets.newHashSet(); for (MetricResp metricResp : metricResps) { - for (String field : fields) { - if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { - List ids = metricResp.getMetricDefineByMetricParams().getMetrics() - .stream().map(MetricParam::getId).collect(Collectors.toList()); - List metricById = metricResps.stream() - .filter(metric -> ids.contains(metric.getId())) - .collect(Collectors.toList()); - for (MetricResp metric : metricById) { - if (metric.getExpr().contains(field)) { - metricRespFiltered.add(metricResp); - } - } - } else { - if (metricResp.getExpr().contains(field)) { - metricRespFiltered.add(metricResp); - } + filterByField(metricResps, metricResp, fields, metricRespFiltered); + } + return new ArrayList<>(metricRespFiltered); + } + + private boolean filterByField(List metricResps, MetricResp metricResp, + List fields, Set metricRespFiltered) { + if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { + List ids = metricResp.getMetricDefineByMetricParams().getMetrics() + .stream().map(MetricParam::getId).collect(Collectors.toList()); + List metricById = metricResps.stream() + .filter(metric -> ids.contains(metric.getId())) + .collect(Collectors.toList()); + for (MetricResp metric : metricById) { + if (filterByField(metricResps, metric, fields, metricRespFiltered)) { + metricRespFiltered.add(metricResp); + return true; } } + } else if (MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())) { + if (fields.stream().anyMatch(field -> metricResp.getExpr().contains(field))) { + metricRespFiltered.add(metricResp); + return true; + } + } else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) { + List measures = metricResp.getMetricDefineByMeasureParams().getMeasures(); + List fieldNameDepended = measures.stream().map(MeasureParam::getBizName) + //measure bizName = model bizName_fieldName + .map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", "")) + .collect(Collectors.toList()); + if (fields.stream().anyMatch(fieldNameDepended::contains)) { + metricRespFiltered.add(metricResp); + return true; + } } - return metricRespFiltered; + return false; } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index c92a09447..a1cf5160f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -64,6 +64,13 @@ public class SchemaServiceImpl implements SchemaService { protected final Cache> itemUseCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build(); + + protected final Cache> viewSchemaCache = + CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build(); + + protected final Cache semanticSchemaCache = + CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build(); + private final StatUtils statUtils; private final ModelService modelService; private final DimensionService dimensionService; @@ -91,6 +98,22 @@ public class SchemaServiceImpl implements SchemaService { @SneakyThrows @Override public List fetchViewSchema(ViewFilterReq filter) { + List viewList = viewSchemaCache.getIfPresent(filter); + if (CollectionUtils.isEmpty(viewList)) { + viewList = buildViewSchema(filter); + viewSchemaCache.put(filter, viewList); + } + return viewList; + } + + public ViewSchemaResp fetchViewSchema(Long viewId) { + if (viewId == null) { + return null; + } + return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null); + } + + public List buildViewSchema(ViewFilterReq filter) { List viewSchemaResps = new ArrayList<>(); List viewIds = filter.getViewIds(); MetaFilter metaFilter = new MetaFilter(); @@ -127,13 +150,6 @@ public class SchemaServiceImpl implements SchemaService { return viewSchemaResps; } - public ViewSchemaResp fetchViewSchema(Long viewId) { - if (viewId == null) { - return null; - } - return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null); - } - public List fetchModelSchemaResps(List modelIds) { List modelSchemaResps = Lists.newArrayList(); if (CollectionUtils.isEmpty(modelIds)) { @@ -258,8 +274,7 @@ public class SchemaServiceImpl implements SchemaService { return viewService.getViewList(metaFilter); } - @Override - public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) { + public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) { SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp(); semanticSchemaResp.setViewId(schemaFilterReq.getViewId()); semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds()); @@ -294,6 +309,16 @@ public class SchemaServiceImpl implements SchemaService { return semanticSchemaResp; } + @Override + public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) { + SemanticSchemaResp semanticSchemaResp = semanticSchemaCache.getIfPresent(schemaFilterReq); + if (semanticSchemaResp == null) { + semanticSchemaResp = buildSemanticSchema(schemaFilterReq); + semanticSchemaCache.put(schemaFilterReq, semanticSchemaResp); + } + return semanticSchemaResp; + } + @SneakyThrows @Override public List getStatInfo(ItemUseReq itemUseReq) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java index 9b60a20a9..07f55f6a4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java @@ -10,17 +10,29 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.ViewDetail; import com.tencent.supersonic.headless.api.pojo.request.ViewReq; +import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO; import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper; import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; +import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ViewService; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + import java.util.Arrays; import java.util.Comparator; import java.util.Date; @@ -29,12 +41,8 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; @Service public class ViewServiceImpl @@ -46,21 +54,33 @@ public class ViewServiceImpl @Autowired private DomainService domainService; + @Lazy + @Autowired + private DimensionService dimensionService; + + @Lazy + @Autowired + private MetricService metricService; + @Override public ViewResp save(ViewReq viewReq, User user) { viewReq.createdBy(user.getName()); ViewDO viewDO = convert(viewReq); viewDO.setStatus(StatusEnum.ONLINE.getCode()); + ViewResp viewResp = convert(viewDO); + conflictCheck(viewResp); save(viewDO); - return convert(viewDO); + return viewResp; } @Override public ViewResp update(ViewReq viewReq, User user) { viewReq.updatedBy(user.getName()); ViewDO viewDO = convert(viewReq); + ViewResp viewResp = convert(viewDO); + conflictCheck(viewResp); updateById(viewDO); - return convert(viewDO); + return viewResp; } @Override @@ -78,6 +98,9 @@ public class ViewServiceImpl if (!CollectionUtils.isEmpty(metaFilter.getIds())) { wrapper.lambda().in(ViewDO::getId, metaFilter.getIds()); } + if (metaFilter.getStatus() != null) { + wrapper.lambda().eq(ViewDO::getStatus, metaFilter.getStatus()); + } wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode()); return list(wrapper).stream().map(this::convert).collect(Collectors.toList()); } @@ -175,4 +198,43 @@ public class ViewServiceImpl viewResp -> viewResp.getAllModels().stream().map(modelId -> Pair.of(modelId, viewResp.getId()))) .collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList()))); } + + private void conflictCheck(ViewResp viewResp) { + List allDimensionIds = viewResp.getAllDimensions(); + List allMetricIds = viewResp.getAllMetrics(); + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setIds(allDimensionIds); + List dimensionResps = dimensionService.getDimensions(metaFilter); + metaFilter.setIds(allMetricIds); + List metricResps = metricService.getMetrics(metaFilter); + + List duplicateDimensionNames = findDuplicates(dimensionResps, DimensionResp::getName); + List duplicateDimensionBizNames = findDuplicates(dimensionResps, DimensionResp::getBizName); + + List duplicateMetricNames = findDuplicates(metricResps, MetricResp::getName); + List duplicateMetricBizNames = findDuplicates(metricResps, MetricResp::getBizName); + if (!duplicateDimensionNames.isEmpty()) { + throw new InvalidArgumentException("存在相同的维度名: " + duplicateDimensionNames); + } + if (!duplicateDimensionBizNames.isEmpty()) { + throw new InvalidArgumentException("存在相同的维度英文名: " + duplicateDimensionBizNames); + } + if (!duplicateMetricNames.isEmpty()) { + throw new InvalidArgumentException("存在相同的指标名: " + duplicateMetricNames); + } + if (!duplicateMetricBizNames.isEmpty()) { + throw new InvalidArgumentException("存在相同的指标英文名: " + duplicateMetricBizNames); + } + } + + private List findDuplicates(List list, Function keyExtractor) { + return list.stream() + .collect(Collectors.groupingBy(keyExtractor, Collectors.counting())) + .entrySet().stream() + .filter(entry -> entry.getValue() > 1) + .map(Map.Entry::getKey) + .map(Object::toString) + .collect(Collectors.toList()); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java index 282b6ff39..98cbbea9e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java @@ -72,6 +72,7 @@ public class MetricConverter { ModelResp modelResp = modelMap.get(metricDO.getModelId()); if (modelResp != null) { metricResp.setModelName(modelResp.getName()); + metricResp.setModelBizName(modelResp.getBizName()); metricResp.setDomainId(modelResp.getDomainId()); } metricResp.setIsCollect(collect != null && collect.contains(metricDO.getId())); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java index c54d033a4..76f38b21d 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java @@ -224,7 +224,7 @@ public class BenchMarkDemoDataLoader { new ViewModelConfig(5L, Lists.newArrayList(8L), Lists.newArrayList()), new ViewModelConfig(6L, Lists.newArrayList(9L, 10L), Lists.newArrayList()), new ViewModelConfig(7L, Lists.newArrayList(11L, 12L), Lists.newArrayList()), - new ViewModelConfig(8L, Lists.newArrayList(13L, 14L, 15L), Lists.newArrayList(8L, 9L)) + new ViewModelConfig(8L, Lists.newArrayList(13L, 14L), Lists.newArrayList(8L, 9L)) ); ViewDetail viewDetail = new ViewDetail(); viewDetail.setViewModelConfigs(viewModelConfigs); diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt index 9871036d4..b01cf97ae 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt @@ -5,14 +5,14 @@ dean _1_2 36 john _1_2 50 jack _1_2 38 admin _1_2 70 -周杰伦 _2_7 100 -陈奕迅 _2_7 100 -林俊杰 _2_7 100 -张碧晨 _2_7 100 -程响 _2_7 100 -Taylor#Swift _2_7 100 -内地 _2_4 100 -欧美 _2_4 100 -港台 _2_4 100 -流行 _2_6 100 -国风 _2_6 100 \ No newline at end of file +周杰伦 _4_7 100 +陈奕迅 _4_7 100 +林俊杰 _4_7 100 +张碧晨 _4_7 100 +程响 _4_7 100 +Taylor#Swift _4_7 100 +内地 _4_4 100 +欧美 _4_4 100 +港台 _4_4 100 +流行 _4_6 100 +国风 _4_6 100 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt index d0d411751..e442a219c 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt @@ -1,6 +1,6 @@ -p1 _2_3 52 -p2 _2_3 47 -p3 _2_3 31 -p4 _2_3 36 -p5 _2_3 50 -p6 _2_3 38 \ No newline at end of file +p1 _3_3 52 +p2 _3_3 47 +p3 _3_3 31 +p4 _3_3 36 +p5 _3_3 50 +p6 _3_3 38 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt index 6699a0fc8..3bc23d613 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt @@ -1,9 +1,9 @@ -周杰伦 _2_7 9000 -周深 _2_7 8000 -周传雄 _2_7 7000 -周华建 _2_7 6000 -陈奕迅 _2_7 8000 -林俊杰 _2_7 7000 -张碧晨 _2_7 7000 -程响 _2_7 7000 -Taylor#Swift _2_7 7000 \ No newline at end of file +周杰伦 _4_7 9000 +周深 _4_7 8000 +周传雄 _4_7 7000 +周华建 _4_7 6000 +陈奕迅 _4_7 8000 +林俊杰 _4_7 7000 +张碧晨 _4_7 7000 +程响 _4_7 7000 +Taylor#Swift _4_7 7000 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt index cce180c73..605735e47 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt @@ -1,4 +1,4 @@ -美国 _3_8 1 -加拿大 _3_8 1 -锡尔赫特、吉大港、库斯蒂亚 _3_8 1 -孟加拉国 _3_8 3 +美国 _5_8 1 +加拿大 _5_8 1 +锡尔赫特、吉大港、库斯蒂亚 _5_8 1 +孟加拉国 _5_8 3 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt index bd4498b5d..ad0bf23b0 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt @@ -1,6 +1,6 @@ -现代 _3_9 1 -tagore _3_9 1 -蓝调 _3_9 1 -流行 _3_9 1 -民间 _3_9 1 -nazrul _3_9 1 +现代 _5_9 1 +tagore _5_9 1 +蓝调 _5_9 1 +流行 _5_9 1 +民间 _5_9 1 +nazrul _5_9 1 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt index 9e65454db..fbdf86ead 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt @@ -1,4 +1,4 @@ -美国 _3_11 1 -印度 _3_11 2 -英国 _3_11 1 -孟加拉国 _3_11 2 +美国 _6_11 1 +印度 _6_11 2 +英国 _6_11 1 +孟加拉国 _6_11 2 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt index 1f6344ad7..51a3b1edc 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt @@ -1,2 +1,2 @@ -男性 _3_12 3 -女性 _3_12 3 +男性 _6_12 3 +女性 _6_12 3 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt index 0ddbbfdc3..71561720f 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt @@ -1,2 +1,2 @@ -mp4 _3_14 4 -mp3 _3_14 2 +mp4 _7_14 4 +mp3 _7_14 2 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt index 65cb9fbf7..f7dd4e308 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt @@ -1,4 +1,4 @@ -美国 _3_17 1 -印度 _3_17 2 -英国 _3_17 1 -孟加拉国 _3_17 2 +美国 _8_17 1 +印度 _8_17 2 +英国 _8_17 1 +孟加拉国 _8_17 2 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt index 5705011ee..360d9b05d 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt @@ -1,2 +1,2 @@ -英文 _3_18 2 -孟加拉语 _3_18 4 +英文 _8_18 2 +孟加拉语 _8_18 4 diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt index 1235c3ad6..d96cbf33f 100644 --- a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt @@ -1,6 +1,6 @@ -阿米·奥帕尔·霍伊 _3_16 1 -我的爱 _3_16 1 -打败它 _3_16 1 -阿杰伊阿卡什 _3_16 1 -Tumi#长袍#尼罗布 _3_16 1 -舒克诺#帕塔尔#努普尔#帕埃 _3_16 1 +阿米·奥帕尔·霍伊 _8_16 1 +我的爱 _8_16 1 +打败它 _8_16 1 +阿杰伊阿卡什 _8_16 1 +Tumi#长袍#尼罗布 _8_16 1 +舒克诺#帕塔尔#努普尔#帕埃 _8_16 1 diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java new file mode 100644 index 000000000..02e6d90a8 --- /dev/null +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/ModelSchemaTest.java @@ -0,0 +1,33 @@ +package com.tencent.supersonic.headless; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.request.FieldRemovedReq; +import com.tencent.supersonic.headless.api.pojo.response.MetricResp; +import com.tencent.supersonic.headless.api.pojo.response.UnAvailableItemResp; +import com.tencent.supersonic.headless.server.service.ModelService; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +public class ModelSchemaTest extends BaseTest { + + @Autowired + private ModelService modelService; + + @Test + void testGetUnAvailableItem() { + FieldRemovedReq fieldRemovedReq = new FieldRemovedReq(); + fieldRemovedReq.setModelId(2L); + fieldRemovedReq.setFields(Lists.newArrayList("pv")); + UnAvailableItemResp unAvailableItemResp = modelService.getUnAvailableItem(fieldRemovedReq); + List expectedUnAvailableMetricId = Lists.newArrayList(1L, 3L); + List actualUnAvailableMetricId = unAvailableItemResp.getMetricResps() + .stream().map(MetricResp::getId).sorted(Comparator.naturalOrder()).collect(Collectors.toList()); + Assertions.assertEquals(expectedUnAvailableMetricId, actualUnAvailableMetricId); + } + +} diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt index 9871036d4..b01cf97ae 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt @@ -5,14 +5,14 @@ dean _1_2 36 john _1_2 50 jack _1_2 38 admin _1_2 70 -周杰伦 _2_7 100 -陈奕迅 _2_7 100 -林俊杰 _2_7 100 -张碧晨 _2_7 100 -程响 _2_7 100 -Taylor#Swift _2_7 100 -内地 _2_4 100 -欧美 _2_4 100 -港台 _2_4 100 -流行 _2_6 100 -国风 _2_6 100 \ No newline at end of file +周杰伦 _4_7 100 +陈奕迅 _4_7 100 +林俊杰 _4_7 100 +张碧晨 _4_7 100 +程响 _4_7 100 +Taylor#Swift _4_7 100 +内地 _4_4 100 +欧美 _4_4 100 +港台 _4_4 100 +流行 _4_6 100 +国风 _4_6 100 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt index d0d411751..e442a219c 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt @@ -1,6 +1,6 @@ -p1 _2_3 52 -p2 _2_3 47 -p3 _2_3 31 -p4 _2_3 36 -p5 _2_3 50 -p6 _2_3 38 \ No newline at end of file +p1 _3_3 52 +p2 _3_3 47 +p3 _3_3 31 +p4 _3_3 36 +p5 _3_3 50 +p6 _3_3 38 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt index 6699a0fc8..3bc23d613 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt @@ -1,9 +1,9 @@ -周杰伦 _2_7 9000 -周深 _2_7 8000 -周传雄 _2_7 7000 -周华建 _2_7 6000 -陈奕迅 _2_7 8000 -林俊杰 _2_7 7000 -张碧晨 _2_7 7000 -程响 _2_7 7000 -Taylor#Swift _2_7 7000 \ No newline at end of file +周杰伦 _4_7 9000 +周深 _4_7 8000 +周传雄 _4_7 7000 +周华建 _4_7 6000 +陈奕迅 _4_7 8000 +林俊杰 _4_7 7000 +张碧晨 _4_7 7000 +程响 _4_7 7000 +Taylor#Swift _4_7 7000 \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt index 6402c14f5..605735e47 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_10.txt @@ -1,4 +1,4 @@ 美国 _5_8 1 加拿大 _5_8 1 锡尔赫特、吉大港、库斯蒂亚 _5_8 1 -孟加拉国 _5_8 3 \ No newline at end of file +孟加拉国 _5_8 3 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt index 6cef5f046..ad0bf23b0 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_5_11.txt @@ -3,4 +3,4 @@ tagore _5_9 1 蓝调 _5_9 1 流行 _5_9 1 民间 _5_9 1 -nazrul _5_9 1 \ No newline at end of file +nazrul _5_9 1 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt index b5458d1ab..fbdf86ead 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_12.txt @@ -1,4 +1,4 @@ -美国 _6_10 1 -印度 _6_10 2 -英国 _6_10 1 -孟加拉国 _6_10 2 \ No newline at end of file +美国 _6_11 1 +印度 _6_11 2 +英国 _6_11 1 +孟加拉国 _6_11 2 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt index 33944bd5d..51a3b1edc 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_6_13.txt @@ -1,2 +1,2 @@ -男性 _6_11 3 -女性 _6_11 3 \ No newline at end of file +男性 _6_12 3 +女性 _6_12 3 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt index 46f76c558..71561720f 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_7_16.txt @@ -1,2 +1,2 @@ mp4 _7_14 4 -mp3 _7_14 2 \ No newline at end of file +mp3 _7_14 2 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt index 78ea079a1..f7dd4e308 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_18.txt @@ -1,4 +1,4 @@ -美国 _8_16 1 -印度 _8_16 2 -英国 _8_16 1 -孟加拉国 _8_16 2 \ No newline at end of file +美国 _8_17 1 +印度 _8_17 2 +英国 _8_17 1 +孟加拉国 _8_17 2 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt index 86a5882f3..360d9b05d 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_19.txt @@ -1,2 +1,2 @@ -英文 _8_17 2 -孟加拉语 _8_17 4 \ No newline at end of file +英文 _8_18 2 +孟加拉语 _8_18 4 diff --git a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt index 0c00c7fa7..d96cbf33f 100644 --- a/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt +++ b/launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_8_21.txt @@ -1,6 +1,6 @@ -阿米·奥帕尔·霍伊 _8_19 1 -我的爱 _8_19 1 -打败它 _8_19 1 -阿杰伊阿卡什 _8_19 1 -Tumi#长袍#尼罗布 _8_19 1 -舒克诺#帕塔尔#努普尔#帕埃 _8_19 1 \ No newline at end of file +阿米·奥帕尔·霍伊 _8_16 1 +我的爱 _8_16 1 +打败它 _8_16 1 +阿杰伊阿卡什 _8_16 1 +Tumi#长袍#尼罗布 _8_16 1 +舒克诺#帕塔尔#努普尔#帕埃 _8_16 1