From 1fcd880042630b9572ecb65240db9707ac701775 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Sun, 26 May 2024 23:11:48 +0800 Subject: [PATCH] (improvement)(headless)take only M dimensionValue or N metric/dimension per rond. (#1032) --- .../chat/mapper/HanlpDictMatchStrategy.java | 29 +++++++++---------- .../core/config/OptimizationConfig.java | 4 ++- .../service/impl/DimensionServiceImpl.java | 12 ++++---- .../service/impl/MetricServiceImpl.java | 8 ++--- .../server/service/MetricServiceImplTest.java | 18 ++++++------ 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java index e81512906..f76a237d4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java @@ -6,12 +6,6 @@ import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult; import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService; import com.tencent.supersonic.headless.core.config.OptimizationConfig; import com.tencent.supersonic.headless.core.pojo.QueryContext; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -19,6 +13,11 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; /** * HanlpDictMatchStrategy uses HanLP to @@ -40,7 +39,7 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { @Override public Map> match(QueryContext queryContext, List terms, - Set detectDataSetIds) { + Set detectDataSetIds) { String text = queryContext.getQueryText(); if (Objects.isNull(terms) || StringUtils.isEmpty(text)) { return null; @@ -62,7 +61,7 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { } public void detectByStep(QueryContext queryContext, Set existResults, Set detectDataSetIds, - String detectSegment, int offset) { + String detectSegment, int offset) { // step1. pre search Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize(); LinkedHashSet hanlpMapResults = knowledgeBaseService.prefixSearch(detectSegment, @@ -97,19 +96,19 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy { return parseResult; }).collect(Collectors.toCollection(LinkedHashSet::new)); - // step5. take only one dimension or 10 metric/dimension value per rond. - List dimensionMetrics = hanlpMapResults.stream() + // step5. take only M dimensionValue or N metric/dimension per rond. + List dimensionValues = hanlpMapResults.stream() .filter(entry -> mapperHelper.existDimensionValues(entry.getNatures())) - .collect(Collectors.toList()) - .stream() - .limit(1) + .limit(optimizationConfig.getOneDetectionDimensionValueSize()) .collect(Collectors.toList()); Integer oneDetectionSize = optimizationConfig.getOneDetectionSize(); List oneRoundResults = hanlpMapResults.stream().limit(oneDetectionSize) .collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(dimensionMetrics)) { - oneRoundResults = dimensionMetrics; + + // add the dimensionValue/term if it exists dimensionValue + if (CollectionUtils.isNotEmpty(dimensionValues)) { + oneRoundResults = dimensionValues; List termOneRoundResults = hanlpMapResults.stream() .filter(hanlpMapResult -> mapperHelper.existTerms(hanlpMapResult.getNatures())) .collect(Collectors.toList()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java index 8b7502131..52c179064 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/OptimizationConfig.java @@ -16,10 +16,12 @@ public class OptimizationConfig { @Value("${s2.one.detection.size:8}") private Integer oneDetectionSize; - @Value("${s2.one.detection.max.size:20}") private Integer oneDetectionMaxSize; + @Value("${s2.one.detection.dimensionValue.size:1}") + private Integer oneDetectionDimensionValueSize; + @Value("${s2.metric.dimension.min.threshold:0.3}") private Double metricDimensionMinThresholdConfig; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 29c366dc4..151e6ced9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -69,7 +69,7 @@ public class DimensionServiceImpl implements DimensionService { private ModelService modelService; - private AliasGenerateHelper chatGptHelper; + private AliasGenerateHelper aliasGenerateHelper; private DatabaseService databaseService; @@ -85,14 +85,14 @@ public class DimensionServiceImpl implements DimensionService { public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService, - AliasGenerateHelper chatGptHelper, + AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService, ModelRelaService modelRelaService, DataSetService dataSetService, TagMetaService tagMetaService) { this.modelService = modelService; this.dimensionRepository = dimensionRepository; - this.chatGptHelper = chatGptHelper; + this.aliasGenerateHelper = aliasGenerateHelper; this.databaseService = databaseService; this.modelRelaService = modelRelaService; this.dataSetService = dataSetService; @@ -341,8 +341,8 @@ public class DimensionServiceImpl implements DimensionService { @Override public List mockAlias(DimensionReq dimensionReq, String mockType, User user) { - String mockAlias = chatGptHelper.generateAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(), - "", dimensionReq.getDescription(), false); + String mockAlias = aliasGenerateHelper.generateAlias(mockType, dimensionReq.getName(), + dimensionReq.getBizName(), "", dimensionReq.getDescription(), false); return JSONObject.parseObject(mockAlias, new TypeReference>() { }); } @@ -363,7 +363,7 @@ public class DimensionServiceImpl implements DimensionService { String value = (String) stringObjectMap.get(dimensionReq.getBizName()); valueList.add(value); } - String json = chatGptHelper.generateDimensionValueAlias(JSON.toJSONString(valueList)); + String json = aliasGenerateHelper.generateDimensionValueAlias(JSON.toJSONString(valueList)); log.info("return llm res is :{}", json); JSONObject jsonObject = JSON.parseObject(json); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index e1b8ca132..6721a43b8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -97,7 +97,7 @@ public class MetricServiceImpl implements MetricService { private DimensionService dimensionService; - private AliasGenerateHelper chatGptHelper; + private AliasGenerateHelper aliasGenerateHelper; private CollectService collectService; @@ -111,7 +111,7 @@ public class MetricServiceImpl implements MetricService { public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, - AliasGenerateHelper chatGptHelper, + AliasGenerateHelper aliasGenerateHelper, CollectService collectService, DataSetService dataSetService, ApplicationEventPublisher eventPublisher, @@ -120,7 +120,7 @@ public class MetricServiceImpl implements MetricService { @Lazy MetaDiscoveryService metaDiscoveryService) { this.metricRepository = metricRepository; this.modelService = modelService; - this.chatGptHelper = chatGptHelper; + this.aliasGenerateHelper = aliasGenerateHelper; this.eventPublisher = eventPublisher; this.collectService = collectService; this.dataSetService = dataSetService; @@ -535,7 +535,7 @@ public class MetricServiceImpl implements MetricService { @Override public List mockAlias(MetricBaseReq metricReq, String mockType, User user) { - String mockAlias = chatGptHelper.generateAlias(mockType, metricReq.getName(), metricReq.getBizName(), "", + String mockAlias = aliasGenerateHelper.generateAlias(mockType, metricReq.getName(), metricReq.getBizName(), "", metricReq.getDescription(), !"".equals(metricReq.getDataFormatType())); return JSONObject.parseObject(mockAlias, new TypeReference>() { }); diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java index e13e1aef4..5d5fca348 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.DataFormat; @@ -7,7 +10,6 @@ import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; -import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.MeasureParam; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; @@ -21,17 +23,14 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.service.impl.DataSetServiceImpl; import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl; +import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; import com.tencent.supersonic.headless.server.utils.MetricConverter; +import java.util.HashMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.springframework.context.ApplicationEventPublisher; -import java.util.HashMap; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - public class MetricServiceImplTest { @Test @@ -64,15 +63,16 @@ public class MetricServiceImplTest { private MetricService mockMetricService(MetricRepository metricRepository, ModelService modelService) { - AliasGenerateHelper chatGptHelper = Mockito.mock(AliasGenerateHelper.class); + AliasGenerateHelper aliasGenerateHelper = Mockito.mock(AliasGenerateHelper.class); CollectService collectService = Mockito.mock(CollectService.class); ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class); DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class); DimensionService dimensionService = Mockito.mock(DimensionService.class); TagMetaService tagMetaService = Mockito.mock(TagMetaService.class); MetaDiscoveryService metaDiscoveryService = Mockito.mock(MetaDiscoveryService.class); - return new MetricServiceImpl(metricRepository, modelService, chatGptHelper, collectService, dataSetService, - eventPublisher, dimensionService, tagMetaService, metaDiscoveryService); + return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper, + collectService, dataSetService, eventPublisher, dimensionService, + tagMetaService, metaDiscoveryService); } private MetricReq buildMetricReq() {