(improvement)(chat) Update ConfigureDemo and dict txt for test (#438)

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-28 00:31:54 +08:00
committed by GitHub
parent 4222d7e2b5
commit 02b9dc6947
39 changed files with 467 additions and 645 deletions

View File

@@ -10,7 +10,7 @@ import lombok.NoArgsConstructor;
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RelateSchemaElement {
public class RelatedSchemaElement {
private Long dimensionId;

View File

@@ -1,14 +1,15 @@
package com.tencent.supersonic.chat.api.pojo;
import com.google.common.base.Objects;
import java.io.Serializable;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.util.List;
@Data
@Getter
@Builder
@@ -24,7 +25,7 @@ public class SchemaElement implements Serializable {
private SchemaElementType type;
private List<String> alias;
private List<SchemaValueMap> schemaValueMaps;
private List<RelateSchemaElement> relateSchemaElements;
private List<RelatedSchemaElement> relatedSchemaElements;
private String defaultAgg;

View File

@@ -1,51 +0,0 @@
package com.tencent.supersonic.chat.parser.llm.time;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LLMTimeEnhancementParse implements SemanticParser {
@Override
public void parse(QueryContext queryContext, ChatContext chatContext) {
log.info("before queryContext:{},chatContext:{}", queryContext, chatContext);
ChatGptHelper chatGptHelper = ContextUtils.getBean(ChatGptHelper.class);
try {
String inferredTime = chatGptHelper.inferredTime(queryContext.getRequest().getQueryText());
if (!queryContext.getCandidateQueries().isEmpty()) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
DateConf dateInfo = query.getParseInfo().getDateInfo();
JSONObject jsonObject = JSON.parseObject(inferredTime);
if (jsonObject.containsKey("date")) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(jsonObject.getString("date"));
dateInfo.setEndDate(jsonObject.getString("date"));
query.getParseInfo().setDateInfo(dateInfo);
} else if (jsonObject.containsKey("start")) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(jsonObject.getString("start"));
dateInfo.setEndDate(jsonObject.getString("end"));
query.getParseInfo().setDateInfo(dateInfo);
}
}
}
} catch (Exception exception) {
log.error("{} parse error,this reason is:{}", LLMTimeEnhancementParse.class.getSimpleName(),
(Object) exception.getStackTrace());
}
log.info("{} after queryContext:{},chatContext:{}",
LLMTimeEnhancementParse.class.getSimpleName(), queryContext, chatContext);
}
}

View File

@@ -4,7 +4,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
@@ -133,9 +133,9 @@ public class MetricCheckPostProcessor implements PostProcessor {
return false;
}
List<String> relateDimensions = metricElements.stream()
.filter(schemaElement -> !CollectionUtils.isEmpty(schemaElement.getRelateSchemaElements()))
.map(schemaElement -> schemaElement.getRelateSchemaElements().stream()
.map(RelateSchemaElement::getDimensionId).collect(Collectors.toList()))
.filter(schemaElement -> !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements()))
.map(schemaElement -> schemaElement.getRelatedSchemaElements().stream()
.map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList()))
.flatMap(Collection::stream)
.map(id -> convertDimensionIdToName(id, semanticSchema))
.filter(Objects::nonNull)
@@ -158,12 +158,12 @@ public class MetricCheckPostProcessor implements PostProcessor {
if (metric == null) {
return Lists.newArrayList();
}
List<RelateSchemaElement> relateSchemaElements = metric.getRelateSchemaElements();
List<RelatedSchemaElement> relateSchemaElements = metric.getRelatedSchemaElements();
if (CollectionUtils.isEmpty(relateSchemaElements)) {
return Lists.newArrayList();
}
return relateSchemaElements.stream()
.filter(RelateSchemaElement::isNecessary).map(RelateSchemaElement::getDimensionId)
.filter(RelatedSchemaElement::isNecessary).map(RelatedSchemaElement::getDimensionId)
.collect(Collectors.toList());
}

View File

@@ -4,10 +4,8 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
@@ -18,10 +16,8 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -42,47 +38,9 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
@Override
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryCtx) {
candidateElementMatches = filterElementMatches(candidateElementMatches);
return super.match(candidateElementMatches, queryCtx);
}
private List<SchemaElementMatch> filterElementMatches(List<SchemaElementMatch> candidateElementMatches) {
List<SchemaElementMatch> filteredMatches = new ArrayList<>();
if (CollectionUtils.isEmpty(candidateElementMatches)
|| Objects.isNull(candidateElementMatches.get(0).getElement().getModel())) {
return candidateElementMatches;
}
Long modelId = candidateElementMatches.get(0).getElement().getModel();
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigResp chatConfig = configService.fetchConfigByModelId(modelId);
List<Long> blackDimIdList = new ArrayList<>();
List<Long> blackMetricIdList = new ArrayList<>();
if (Objects.nonNull(chatConfig.getChatAggConfig())
&& Objects.nonNull(chatConfig.getChatAggConfig().getVisibility())) {
blackDimIdList = chatConfig.getChatAggConfig().getVisibility().getBlackDimIdList();
blackMetricIdList = chatConfig.getChatAggConfig().getVisibility().getBlackMetricIdList();
}
for (SchemaElementMatch schemaElementMatch : candidateElementMatches) {
SchemaElementType type = schemaElementMatch.getElement().getType();
if (SchemaElementType.DIMENSION.equals(type) || SchemaElementType.VALUE.equals(type)) {
if (!blackDimIdList.contains(schemaElementMatch.getElement().getId())) {
filteredMatches.add(schemaElementMatch);
}
} else if (SchemaElementType.METRIC.equals(type)) {
if (!blackMetricIdList.contains(schemaElementMatch.getElement().getId())) {
filteredMatches.add(schemaElementMatch);
}
} else {
filteredMatches.add(schemaElementMatch);
}
}
return filteredMatches;
}
@Override
public void fillParseInfo(ChatContext chatContext) {
super.fillParseInfo(chatContext);

View File

@@ -3,8 +3,6 @@ package com.tencent.supersonic.chat.query.rule.tag;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
@@ -13,10 +11,8 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -37,47 +33,9 @@ public abstract class TagSemanticQuery extends RuleSemanticQuery {
@Override
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContext queryCtx) {
candidateElementMatches = filterElementMatches(candidateElementMatches);
return super.match(candidateElementMatches, queryCtx);
}
private List<SchemaElementMatch> filterElementMatches(List<SchemaElementMatch> candidateElementMatches) {
List<SchemaElementMatch> filteredMatches = new ArrayList<>();
if (CollectionUtils.isEmpty(candidateElementMatches)
|| Objects.isNull(candidateElementMatches.get(0).getElement().getModel())) {
return candidateElementMatches;
}
Long modelId = candidateElementMatches.get(0).getElement().getModel();
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigResp chatConfig = configService.fetchConfigByModelId(modelId);
List<Long> blackDimIdList = new ArrayList<>();
List<Long> blackMetricIdList = new ArrayList<>();
if (Objects.nonNull(chatConfig.getChatDetailConfig())
&& Objects.nonNull(chatConfig.getChatDetailConfig().getVisibility())) {
blackDimIdList = chatConfig.getChatDetailConfig().getVisibility().getBlackDimIdList();
blackMetricIdList = chatConfig.getChatDetailConfig().getVisibility().getBlackMetricIdList();
}
for (SchemaElementMatch schemaElementMatch : candidateElementMatches) {
SchemaElementType type = schemaElementMatch.getElement().getType();
if (SchemaElementType.DIMENSION.equals(type) || SchemaElementType.VALUE.equals(type)) {
if (!blackDimIdList.contains(schemaElementMatch.getElement().getId())) {
filteredMatches.add(schemaElementMatch);
}
} else if (SchemaElementType.METRIC.equals(type)) {
if (!blackMetricIdList.contains(schemaElementMatch.getElement().getId())) {
filteredMatches.add(schemaElementMatch);
}
} else {
filteredMatches.add(schemaElementMatch);
}
}
return filteredMatches;
}
@Override
public void fillParseInfo(ChatContext chatContext) {
super.fillParseInfo(chatContext);

View File

@@ -2,23 +2,13 @@ package com.tencent.supersonic.chat.service.impl;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.RecommendReq;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.RecommendReq;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp;
import com.tencent.supersonic.chat.api.pojo.response.RecommendResp;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.RecommendService;
import com.tencent.supersonic.chat.service.SemanticService;
@@ -28,6 +18,14 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/***
* Recommend Service impl
*/
@@ -59,11 +57,11 @@ public class RecommendServiceImpl implements RecommendService {
if (recommendReq.getMetricId() != null && !CollectionUtils.isEmpty(metricElements)) {
Optional<SchemaElement> metric = metricElements.stream().filter(schemaElement ->
recommendReq.getMetricId().equals(schemaElement.getId())
&& !CollectionUtils.isEmpty(schemaElement.getRelateSchemaElements()))
&& !CollectionUtils.isEmpty(schemaElement.getRelatedSchemaElements()))
.findFirst();
if (metric.isPresent()) {
drillDownDimensions = metric.get().getRelateSchemaElements().stream()
.map(RelateSchemaElement::getDimensionId).collect(Collectors.toList());
drillDownDimensions = metric.get().getRelatedSchemaElements().stream()
.map(RelatedSchemaElement::getDimensionId).collect(Collectors.toList());
}
}
final List<Long> drillDownDimensionsFinal = drillDownDimensions;
@@ -112,25 +110,7 @@ public class RecommendServiceImpl implements RecommendService {
@Override
public RecommendResp recommendMetricMode(RecommendReq recommendReq, Long limit) {
RecommendResp recommendResponse = recommend(recommendReq, limit);
// filter black Item
if (Objects.isNull(recommendResponse)) {
return recommendResponse;
}
ChatConfigRichResp chatConfigRich = configService.getConfigRichInfo(recommendReq.getModelId());
if (Objects.nonNull(chatConfigRich) && Objects.nonNull(chatConfigRich.getChatAggRichConfig())
&& Objects.nonNull(chatConfigRich.getChatAggRichConfig().getVisibility())) {
List<Long> blackMetricIdList = chatConfigRich.getChatAggRichConfig().getVisibility().getBlackMetricIdList();
List<SchemaElement> metrics = filterBlackItem(recommendResponse.getMetrics(), blackMetricIdList);
recommendResponse.setMetrics(metrics);
List<Long> blackDimIdList = chatConfigRich.getChatAggRichConfig().getVisibility().getBlackDimIdList();
List<SchemaElement> dimensions = filterBlackItem(recommendResponse.getDimensions(), blackDimIdList);
recommendResponse.setDimensions(dimensions);
}
return recommendResponse;
return recommend(recommendReq, limit);
}
@Override

View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.postprocessor;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
@@ -118,10 +118,10 @@ class MetricCheckPostProcessorTest {
ModelSchema modelSchema = new ModelSchema();
Set<SchemaElement> metrics = Sets.newHashSet(
mockElement(1L, "访问次数", SchemaElementType.METRIC,
Lists.newArrayList(RelateSchemaElement.builder().dimensionId(2L).isNecessary(false).build(),
RelateSchemaElement.builder().dimensionId(1L).isNecessary(false).build())),
Lists.newArrayList(RelatedSchemaElement.builder().dimensionId(2L).isNecessary(false).build(),
RelatedSchemaElement.builder().dimensionId(1L).isNecessary(false).build())),
mockElement(2L, "访问用户数", SchemaElementType.METRIC,
Lists.newArrayList(RelateSchemaElement.builder().dimensionId(2L).isNecessary(true).build()))
Lists.newArrayList(RelatedSchemaElement.builder().dimensionId(2L).isNecessary(true).build()))
);
modelSchema.setMetrics(metrics);
modelSchema.setDimensions(mockDimensions());
@@ -148,9 +148,9 @@ class MetricCheckPostProcessorTest {
}
private SchemaElement mockElement(Long id, String name, SchemaElementType type,
List<RelateSchemaElement> relateSchemaElements) {
List<RelatedSchemaElement> relateSchemaElements) {
return SchemaElement.builder().id(id).name(name).type(type)
.relateSchemaElements(relateSchemaElements).build();
.relatedSchemaElements(relateSchemaElements).build();
}
private SemanticParseInfo mockParseInfo(String correctSql) {

View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.knowledge.semantic;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.RelateSchemaElement;
import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
@@ -50,7 +50,7 @@ public class ModelSchemaBuilder {
.type(SchemaElementType.METRIC)
.useCnt(metric.getUseCnt())
.alias(alias)
.relateSchemaElements(getRelateSchemaElement(metric))
.relatedSchemaElements(getRelateSchemaElement(metric))
.defaultAgg(metric.getDefaultAgg())
.build();
metrics.add(metricToAdd);
@@ -138,13 +138,13 @@ public class ModelSchemaBuilder {
return modelSchema;
}
private static List<RelateSchemaElement> getRelateSchemaElement(MetricSchemaResp metricSchemaResp) {
private static List<RelatedSchemaElement> getRelateSchemaElement(MetricSchemaResp metricSchemaResp) {
RelateDimension relateDimension = metricSchemaResp.getRelateDimension();
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
return Lists.newArrayList();
}
return relateDimension.getDrillDownDimensions().stream().map(dimension -> {
RelateSchemaElement relateSchemaElement = new RelateSchemaElement();
RelatedSchemaElement relateSchemaElement = new RelatedSchemaElement();
BeanUtils.copyProperties(dimension, relateSchemaElement);
return relateSchemaElement;
}).collect(Collectors.toList());