(improvement)(build) Add spotless during the build process. (#1639)

This commit is contained in:
lexluo09
2024-09-07 00:36:17 +08:00
committed by GitHub
parent ee15a88b06
commit 5f59e89eea
986 changed files with 15609 additions and 12706 deletions

View File

@@ -1,11 +1,9 @@
package com.tencent.supersonic.chat.api.pojo.enums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import org.apache.commons.lang3.StringUtils;
public enum MemoryReviewResult {
POSITIVE,
NEGATIVE;

View File

@@ -1,10 +1,7 @@
package com.tencent.supersonic.chat.api.pojo.enums;
public enum MemoryStatus {
PENDING,
ENABLED,
DISABLED;
}

View File

@@ -7,18 +7,13 @@ import java.util.List;
@Data
public class ChatAggConfigReq {
/**
* invisible dimensions/metrics
*/
/** invisible dimensions/metrics */
private ItemVisibility visibility;
/**
* information about dictionary about the model
*/
/** information about dictionary about the model */
private List<KnowledgeInfoReq> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultConfigReq chatDefaultConfig;
}
}

View File

@@ -1,35 +1,24 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import lombok.ToString;
import java.util.List;
/**
* extended information command about model
*/
/** extended information command about model */
@Data
@ToString
public class ChatConfigBaseReq {
private Long modelId;
/**
* the recommended questions about the model
*/
/** the recommended questions about the model */
private List<RecommendedQuestionReq> recommendedQuestions;
/**
* the llm examples about the model
*/
/** the llm examples about the model */
private String llmExamples;
/**
* available status
*/
/** available status */
private StatusEnum status;
}

View File

@@ -2,9 +2,8 @@ package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
@Data
public class ChatConfigEditReqReq extends ChatConfigBaseReq {
private Long id;
}
}

View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
@NoArgsConstructor
@Data
public class ChatConfigFilter {
@@ -12,4 +11,4 @@ public class ChatConfigFilter {
private Long id;
private Long modelId;
private StatusEnum status = StatusEnum.ONLINE;
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.ArrayList;
@@ -11,6 +10,4 @@ public class ChatDefaultConfigReq {
private List<Long> dimensionIds = new ArrayList<>();
private List<Long> metricIds = new ArrayList<>();
}
}

View File

@@ -7,18 +7,13 @@ import java.util.List;
@Data
public class ChatDetailConfigReq {
/**
* invisible dimensions/metrics
*/
/** invisible dimensions/metrics */
private ItemVisibility visibility;
/**
* information about dictionary about the model
*/
/** information about dictionary about the model */
private List<KnowledgeInfoReq> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultConfigReq chatDefaultConfig;
}
}

View File

@@ -18,5 +18,4 @@ public class ChatExecuteReq {
private int parseId;
private String queryText;
private boolean saveAnswer;
}

View File

@@ -26,5 +26,4 @@ public class ChatMemoryFilter {
private MemoryReviewResult llmReviewRet;
private MemoryReviewResult humanReviewRet;
}

View File

@@ -1,13 +1,12 @@
package com.tencent.supersonic.chat.api.pojo.request;
import javax.validation.constraints.NotNull;
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.Data;
import javax.validation.constraints.NotNull;
@Data
public class ChatMemoryUpdateReq extends RecordInfo {
@@ -23,5 +22,4 @@ public class ChatMemoryUpdateReq extends RecordInfo {
private MemoryReviewResult humanReviewRet;
private String humanReviewCmt;
}

View File

@@ -15,5 +15,4 @@ public class ChatParseReq {
private QueryFilters queryFilters;
private boolean saveAnswer = true;
private SchemaMapInfo mapInfo = new SchemaMapInfo();
}

View File

@@ -1,29 +1,22 @@
package com.tencent.supersonic.chat.api.pojo.request;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* the entity info about the model
*/
import java.util.List;
/** the entity info about the model */
@Data
@AllArgsConstructor
@ToString
@NoArgsConstructor
public class Entity {
/**
* uniquely identifies an entity
*/
/** uniquely identifies an entity */
private Long entityId;
/**
* entity name list
*/
/** entity name list */
private List<String> names;
}
}

View File

@@ -6,18 +6,13 @@ import lombok.ToString;
import java.util.ArrayList;
import java.util.List;
@Data
@ToString
public class ItemNameVisibilityInfo {
/**
* invisible dimensions
*/
/** invisible dimensions */
private List<String> blackDimNameList = new ArrayList<>();
/**
* invisible metrics
*/
/** invisible metrics */
private List<String> blackMetricNameList = new ArrayList<>();
}

View File

@@ -1,22 +1,18 @@
package com.tencent.supersonic.chat.api.pojo.request;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.ToString;
import java.util.ArrayList;
import java.util.List;
@Data
@ToString
public class ItemVisibility {
/**
* invisible dimensions
*/
/** invisible dimensions */
private List<Long> blackDimIdList = new ArrayList<>();
/**
* invisible metrics
*/
/** invisible metrics */
private List<Long> blackMetricIdList = new ArrayList<>();
}
}

View File

@@ -5,13 +5,11 @@ import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* advanced knowledge config
*/
/** advanced knowledge config */
@Data
public class KnowledgeAdvancedConfig {
private List<String> blackList = new ArrayList<>();
private List<String> whiteList = new ArrayList<>();
private List<String> ruleList = new ArrayList<>();
}
}

View File

@@ -1,35 +1,23 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import javax.validation.constraints.NotNull;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import lombok.Data;
/**
* information about dictionary about the model
*/
/** information about dictionary about the model */
@Data
public class KnowledgeInfoReq {
/**
* metricId、DimensionId、modelId
*/
/** metricId、DimensionId、modelId */
private Long itemId;
private String bizName;
/**
* type: IntentionTypeEnum
* temporarily only supports dimension-related information
*/
@NotNull
private TypeEnums type = TypeEnums.DIMENSION;
/** type: IntentionTypeEnum temporarily only supports dimension-related information */
@NotNull private TypeEnums type = TypeEnums.DIMENSION;
private Boolean searchEnable = false;
/**
* advanced knowledge config for single item
*/
/** advanced knowledge config for single item */
private KnowledgeAdvancedConfig knowledgeAdvancedConfig;
}

View File

@@ -3,10 +3,8 @@ package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.PageBaseReq;
import lombok.Data;
@Data
public class PageMemoryReq extends PageBaseReq {
private ChatMemoryFilter chatMemoryFilter = new ChatMemoryFilter();
}

View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.List;
@Data

View File

@@ -1,12 +1,10 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
@Data
public class PluginQueryReq {
private String name;
private String parseMode;

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
@Data
@@ -9,5 +8,4 @@ public class RecommendReq {
private Long modelId;
private Long metricId;
}
}

View File

@@ -12,5 +12,4 @@ import lombok.ToString;
public class RecommendedQuestionReq {
private String question;
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@@ -21,5 +20,4 @@ public class SimilarQueryReq {
private Long dataSetId;
private Integer agentId;
}
}

View File

@@ -9,18 +9,13 @@ import java.util.List;
@Data
public class ChatAggRichConfigResp {
/**
* invisible dimensions/metrics
*/
/** invisible dimensions/metrics */
private ItemVisibilityInfo visibility;
/**
* information about dictionary about the model
*/
/** information about dictionary about the model */
private List<KnowledgeInfoReq> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultRichConfigResp chatDefaultConfig;
}
}

View File

@@ -4,12 +4,11 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data
public class ChatConfigResp {
@@ -25,13 +24,11 @@ public class ChatConfigResp {
private String llmExamples;
/**
* available status
*/
/** available status */
private StatusEnum statusEnum;
private String createdBy;
private String updatedBy;
private Date createdAt;
private Date updatedAt;
}
}

View File

@@ -2,11 +2,11 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data
public class ChatConfigRichResp {
@@ -23,9 +23,7 @@ public class ChatConfigRichResp {
private List<RecommendedQuestionReq> recommendedQuestions;
/**
* available status
*/
/** available status */
private StatusEnum statusEnum;
private String createdBy;

View File

@@ -1,9 +1,8 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import lombok.Data;
import java.util.List;
@@ -14,18 +13,11 @@ public class ChatDefaultRichConfigResp {
private List<SchemaElement> dimensions;
private List<SchemaElement> metrics;
/**
* default time span unit
*/
/** default time span unit */
private Integer unit = 1;
/**
* default time type:
* DAY, WEEK, MONTH, YEAR
*/
/** default time type: DAY, WEEK, MONTH, YEAR */
private String period = Constants.DAY;
private TimeMode timeMode;
}
}

View File

@@ -9,19 +9,13 @@ import java.util.List;
@Data
public class ChatDetailRichConfigResp {
/**
* invisible dimensions/metrics
*/
/** invisible dimensions/metrics */
private ItemVisibilityInfo visibility;
/**
* information about dictionary about the model
*/
/** information about dictionary about the model */
private List<KnowledgeInfoReq> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultRichConfigResp chatDefaultConfig;
}
}

View File

@@ -27,4 +27,4 @@ public class DictLatestTaskResp {
private Date createdAt;
private Long elapsedMs;
}
}

View File

@@ -1,15 +1,13 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import lombok.Data;
import java.util.List;
import lombok.Data;
@Data
public class EntityRichInfoResp {
/**
* entity alias
*/
/** entity alias */
private List<String> names;
private SchemaElement dimItem;

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.api.pojo.response;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class ItemVisibilityInfo {
@@ -11,4 +11,4 @@ public class ItemVisibilityInfo {
private List<Long> blackMetricIdList;
private List<Long> whiteDimIdList;
private List<Long> whiteMetricIdList;
}
}

View File

@@ -3,10 +3,10 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseTimeCostResp;
import lombok.Data;
import java.util.Date;
import java.util.List;
@Data
public class QueryResp {
@@ -20,5 +20,4 @@ public class QueryResp {
private List<SemanticParseInfo> parseInfos;
private List<SimilarQueryRecallResp> similarQueries;
private ParseTimeCostResp parseTimeCost = new ParseTimeCostResp();
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.api.pojo.response;
import lombok.Data;
import java.util.List;
@@ -14,5 +13,4 @@ public class ShowCaseResp {
private int pageSize;
private int current;
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.api.pojo.response;
import lombok.Builder;
import lombok.Data;
@@ -11,5 +10,4 @@ public class SimilarQueryRecallResp {
private Long queryId;
private String queryText;
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.agent;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -28,10 +27,9 @@ public class Agent extends RecordInfo {
private String name;
private String description;
/**
* 0 offline, 1 online
*/
/** 0 offline, 1 online */
private Integer status;
private List<String> examples;
private String agentConfig;
private ChatModelConfig modelConfig;
@@ -46,13 +44,13 @@ public class Agent extends RecordInfo {
}
List<Map> toolList = (List) map.get("tools");
return toolList.stream()
.filter(tool -> {
.filter(
tool -> {
if (Objects.isNull(type)) {
return true;
}
return type.name().equals(tool.get("type"));
}
)
})
.map(JSONObject::toJSONString)
.collect(Collectors.toList());
}
@@ -74,7 +72,8 @@ public class Agent extends RecordInfo {
if (CollectionUtils.isEmpty(tools)) {
return Lists.newArrayList();
}
return tools.stream().map(tool -> JSONObject.parseObject(tool, NL2SQLTool.class))
return tools.stream()
.map(tool -> JSONObject.parseObject(tool, NL2SQLTool.class))
.collect(Collectors.toList());
}
@@ -121,7 +120,8 @@ public class Agent extends RecordInfo {
if (CollectionUtils.isEmpty(commonAgentTools)) {
return new HashSet<>();
}
return commonAgentTools.stream().map(NL2SQLTool::getDataSetIds)
return commonAgentTools.stream()
.map(NL2SQLTool::getDataSetIds)
.filter(modelIds -> !CollectionUtils.isEmpty(modelIds))
.flatMap(Collection::stream)
.collect(Collectors.toSet());

View File

@@ -4,8 +4,8 @@ import com.google.common.collect.Lists;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.List;
@Data
@NoArgsConstructor
@@ -13,5 +13,4 @@ import java.util.List;
public class AgentConfig {
List<AgentTool> tools = Lists.newArrayList();
}

View File

@@ -21,5 +21,4 @@ public enum AgentToolType {
map.put(PLUGIN, PLUGIN.title);
return map;
}
}

View File

@@ -8,5 +8,4 @@ import java.util.List;
public class LLMParserTool extends NL2SQLTool {
private List<String> exampleQuestions;
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.agent;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -11,5 +10,4 @@ import lombok.NoArgsConstructor;
public class MultiTurnConfig {
private boolean enableMultiTurn;
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.agent;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -13,5 +12,4 @@ import java.util.List;
public class NL2SQLTool extends AgentTool {
protected List<Long> dataSetIds;
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.agent;
import lombok.Data;
import java.util.List;
@@ -9,5 +8,4 @@ import java.util.List;
public class PluginTool extends AgentTool {
private List<Long> plugins;
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.agent;
import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
@@ -9,7 +8,6 @@ import java.util.List;
@Data
public class RuleParserTool extends NL2SQLTool {
private List<String> queryModes;
private List<String> queryTypes;
@@ -17,5 +15,4 @@ public class RuleParserTool extends NL2SQLTool {
public boolean isContainsAllModel() {
return CollectionUtils.isNotEmpty(dataSetIds) && dataSetIds.contains(-1L);
}
}

View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.chat.server.config;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import lombok.ToString;
@@ -14,33 +14,22 @@ import java.util.List;
@ToString
public class ChatConfig {
/**
* database auto-increment primary key
*/
/** database auto-increment primary key */
private Long id;
private Long modelId;
/**
* the chatDetailConfig about the model
*/
/** the chatDetailConfig about the model */
private ChatDetailConfigReq chatDetailConfig;
/**
* the chatAggConfig about the model
*/
/** the chatAggConfig about the model */
private ChatAggConfigReq chatAggConfig;
private List<RecommendedQuestionReq> recommendedQuestions;
/**
* available status
*/
/** available status */
private StatusEnum status;
/**
* about createdBy, createdAt, updatedBy, updatedAt
*/
/** about createdBy, createdAt, updatedBy, updatedAt */
private RecordInfo recordInfo;
}

View File

@@ -8,4 +8,4 @@ public class ChatConfigFilterInternal {
private Long id;
private Long modelId;
private Integer status;
}
}

View File

@@ -1,10 +1,9 @@
package com.tencent.supersonic.chat.server.executor;
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
public interface ChatQueryExecutor {
QueryResult execute(ExecuteContext executeContext);
}

View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.server.executor;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.server.agent.Agent;
import com.tencent.supersonic.chat.server.agent.MultiTurnConfig;
import com.tencent.supersonic.chat.server.parser.ParserConfig;
@@ -8,7 +9,6 @@ import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.chat.server.service.ChatManageService;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import dev.langchain4j.data.message.AiMessage;
import dev.langchain4j.model.chat.ChatLanguageModel;
@@ -26,13 +26,14 @@ import static com.tencent.supersonic.chat.server.parser.ParserConfig.PARSER_MULT
public class PlainTextExecutor implements ChatQueryExecutor {
private static final String INSTRUCTION = ""
+ "#Role: You are a nice person to talk to.\n"
+ "#Task: Respond quickly and nicely to the user."
+ "#Rules: 1.ALWAYS use the same language as the input.\n"
+ "#History Inputs: %s\n"
+ "#Current Input: %s\n"
+ "#Your response: ";
private static final String INSTRUCTION =
""
+ "#Role: You are a nice person to talk to.\n"
+ "#Task: Respond quickly and nicely to the user."
+ "#Rules: 1.ALWAYS use the same language as the input.\n"
+ "#History Inputs: %s\n"
+ "#Current Input: %s\n"
+ "#Your response: ";
@Override
public QueryResult execute(ExecuteContext executeContext) {
@@ -40,14 +41,18 @@ public class PlainTextExecutor implements ChatQueryExecutor {
return null;
}
String promptStr = String.format(INSTRUCTION, getHistoryInputs(executeContext),
executeContext.getQueryText());
String promptStr =
String.format(
INSTRUCTION,
getHistoryInputs(executeContext),
executeContext.getQueryText());
Prompt prompt = PromptTemplate.from(promptStr).apply(Collections.EMPTY_MAP);
AgentService agentService = ContextUtils.getBean(AgentService.class);
Agent chatAgent = agentService.getAgent(executeContext.getAgent().getId());
ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatAgent.getModelConfig());
ChatLanguageModel chatLanguageModel =
ModelProvider.getChatModel(chatAgent.getModelConfig());
Response<AiMessage> response = chatLanguageModel.generate(prompt.toUserMessage());
QueryResult result = new QueryResult();
@@ -66,16 +71,21 @@ public class PlainTextExecutor implements ChatQueryExecutor {
ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class);
MultiTurnConfig agentMultiTurnConfig = chatAgent.getMultiTurnConfig();
Boolean globalMultiTurnConfig = Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE));
Boolean multiTurnConfig = agentMultiTurnConfig != null
? agentMultiTurnConfig.isEnableMultiTurn() : globalMultiTurnConfig;
Boolean globalMultiTurnConfig =
Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE));
Boolean multiTurnConfig =
agentMultiTurnConfig != null
? agentMultiTurnConfig.isEnableMultiTurn()
: globalMultiTurnConfig;
if (Boolean.TRUE.equals(multiTurnConfig)) {
List<QueryResp> queryResps = getHistoryQueries(executeContext.getChatId(), 5);
queryResps.stream().forEach(p -> {
historyInput.append(p.getQueryText());
historyInput.append(";");
});
queryResps.stream()
.forEach(
p -> {
historyInput.append(p.getQueryText());
historyInput.append(";");
});
}
return historyInput.toString();
@@ -83,17 +93,20 @@ public class PlainTextExecutor implements ChatQueryExecutor {
private List<QueryResp> getHistoryQueries(int chatId, int multiNum) {
ChatManageService chatManageService = ContextUtils.getBean(ChatManageService.class);
List<QueryResp> contextualParseInfoList = chatManageService.getChatQueries(chatId)
.stream()
.filter(q -> Objects.nonNull(q.getQueryResult())
&& q.getQueryResult().getQueryState() == QueryState.SUCCESS)
.collect(Collectors.toList());
List<QueryResp> contextualParseInfoList =
chatManageService.getChatQueries(chatId).stream()
.filter(
q ->
Objects.nonNull(q.getQueryResult())
&& q.getQueryResult().getQueryState()
== QueryState.SUCCESS)
.collect(Collectors.toList());
List<QueryResp> contextualList = contextualParseInfoList.subList(0,
Math.min(multiNum, contextualParseInfoList.size()));
List<QueryResp> contextualList =
contextualParseInfoList.subList(
0, Math.min(multiNum, contextualParseInfoList.size()));
Collections.reverse(contextualList);
return contextualList;
}
}

View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.chat.server.executor;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
import com.tencent.supersonic.chat.server.plugin.build.PluginSemanticQuery;
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
public class PluginExecutor implements ChatQueryExecutor {
@@ -18,5 +18,4 @@ public class PluginExecutor implements ChatQueryExecutor {
query.setParseInfo(parseInfo);
return query.build();
}
}

View File

@@ -1,8 +1,11 @@
package com.tencent.supersonic.chat.server.executor;
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
import com.tencent.supersonic.chat.server.pojo.ChatContext;
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
import com.tencent.supersonic.chat.server.service.ChatContextService;
import com.tencent.supersonic.chat.server.service.MemoryService;
import com.tencent.supersonic.chat.server.util.ResultFormatter;
import com.tencent.supersonic.common.pojo.Text2SQLExemplar;
@@ -10,13 +13,10 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.chat.server.pojo.ChatContext;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.chat.server.service.ChatContextService;
import lombok.SneakyThrows;
import org.apache.commons.lang3.StringUtils;
@@ -31,28 +31,35 @@ public class SqlExecutor implements ChatQueryExecutor {
QueryResult queryResult = doExecute(executeContext);
if (queryResult != null) {
String textResult = ResultFormatter.transform2TextNew(queryResult.getQueryColumns(),
queryResult.getQueryResults());
String textResult =
ResultFormatter.transform2TextNew(
queryResult.getQueryColumns(), queryResult.getQueryResults());
queryResult.setTextResult(textResult);
if (queryResult.getQueryState().equals(QueryState.SUCCESS)
&& queryResult.getQueryMode().equals(LLMSqlQuery.QUERY_MODE)) {
Text2SQLExemplar exemplar = JsonUtil.toObject(JsonUtil.toString(
executeContext.getParseInfo().getProperties()
.get(Text2SQLExemplar.PROPERTY_KEY)), Text2SQLExemplar.class);
Text2SQLExemplar exemplar =
JsonUtil.toObject(
JsonUtil.toString(
executeContext
.getParseInfo()
.getProperties()
.get(Text2SQLExemplar.PROPERTY_KEY)),
Text2SQLExemplar.class);
MemoryService memoryService = ContextUtils.getBean(MemoryService.class);
memoryService.createMemory(ChatMemoryDO.builder()
.agentId(executeContext.getAgent().getId())
.status(MemoryStatus.PENDING)
.question(exemplar.getQuestion())
.sideInfo(exemplar.getSideInfo())
.dbSchema(exemplar.getDbSchema())
.s2sql(exemplar.getSql())
.createdBy(executeContext.getUser().getName())
.updatedBy(executeContext.getUser().getName())
.createdAt(new Date())
.build());
memoryService.createMemory(
ChatMemoryDO.builder()
.agentId(executeContext.getAgent().getId())
.status(MemoryStatus.PENDING)
.question(exemplar.getQuestion())
.sideInfo(exemplar.getSideInfo())
.dbSchema(exemplar.getDbSchema())
.s2sql(exemplar.getSql())
.createdBy(executeContext.getUser().getName())
.updatedBy(executeContext.getUser().getName())
.createdAt(new Date())
.build());
}
}
@@ -71,9 +78,8 @@ public class SqlExecutor implements ChatQueryExecutor {
return null;
}
QuerySqlReq sqlReq = QuerySqlReq.builder()
.sql(parseInfo.getSqlInfo().getCorrectedS2SQL())
.build();
QuerySqlReq sqlReq =
QuerySqlReq.builder().sql(parseInfo.getSqlInfo().getCorrectedS2SQL()).build();
sqlReq.setSqlInfo(parseInfo.getSqlInfo());
sqlReq.setDataSetId(parseInfo.getDataSetId());
@@ -97,5 +103,4 @@ public class SqlExecutor implements ChatQueryExecutor {
}
return queryResult;
}
}

View File

@@ -27,27 +27,26 @@ public class MemoryReviewTask {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
private static final String INSTRUCTION = ""
+ "#Role: You are a senior data engineer experienced in writing SQL.\n"
+ "#Task: Your will be provided with a user question and the SQL written by junior engineer,"
+ "please take a review and give your opinion.\n"
+ "#Rules: "
+ "1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),comment=(your comment)`."
+ "2.ALWAYS recognize `数据日期` as the date field."
+ "3.IGNORE `数据日期` if not expressed in the `Question`."
+ "#Question: %s\n"
+ "#Schema: %s\n"
+ "#SideInfo: %s\n"
+ "#SQL: %s\n"
+ "#Response: ";
private static final String INSTRUCTION =
""
+ "#Role: You are a senior data engineer experienced in writing SQL.\n"
+ "#Task: Your will be provided with a user question and the SQL written by junior engineer,"
+ "please take a review and give your opinion.\n"
+ "#Rules: "
+ "1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),comment=(your comment)`."
+ "2.ALWAYS recognize `数据日期` as the date field."
+ "3.IGNORE `数据日期` if not expressed in the `Question`."
+ "#Question: %s\n"
+ "#Schema: %s\n"
+ "#SideInfo: %s\n"
+ "#SQL: %s\n"
+ "#Response: ";
private static final Pattern OUTPUT_PATTERN = Pattern.compile("opinion=(.*),.*comment=(.*)");
@Autowired
private MemoryService memoryService;
@Autowired private MemoryService memoryService;
@Autowired
private AgentService agentService;
@Autowired private AgentService agentService;
@Scheduled(fixedDelay = 60 * 1000)
public void review() {
@@ -68,7 +67,8 @@ public class MemoryReviewTask {
Prompt prompt = PromptTemplate.from(promptStr).apply(Collections.EMPTY_MAP);
keyPipelineLog.info("MemoryReviewTask reqPrompt:\n{}", promptStr);
ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatAgent.getModelConfig());
ChatLanguageModel chatLanguageModel =
ModelProvider.getChatModel(chatAgent.getModelConfig());
if (Objects.nonNull(chatLanguageModel)) {
String response = chatLanguageModel.generate(prompt.toUserMessage()).content().text();
keyPipelineLog.info("MemoryReviewTask modelResp:\n{}", response);
@@ -79,7 +79,8 @@ public class MemoryReviewTask {
}
private String createPromptString(ChatMemoryDO m) {
return String.format(INSTRUCTION, m.getQuestion(), m.getDbSchema(), m.getSideInfo(), m.getS2sql());
return String.format(
INSTRUCTION, m.getQuestion(), m.getDbSchema(), m.getSideInfo(), m.getS2sql());
}
private void processResponse(String response, ChatMemoryDO m) {

View File

@@ -6,5 +6,4 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
public interface ChatQueryParser {
void parse(ParseContext parseContext, ParseResp parseResp);
}

View File

@@ -6,12 +6,14 @@ import com.tencent.supersonic.chat.server.util.ComponentFactory;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@Slf4j
public class NL2PluginParser implements ChatQueryParser {
private final List<PluginRecognizer> pluginRecognizers = ComponentFactory.getPluginRecognizers();
private final List<PluginRecognizer> pluginRecognizers =
ComponentFactory.getPluginRecognizers();
@Override
public void parse(ParseContext parseContext, ParseResp parseResp) {
@@ -19,11 +21,13 @@ public class NL2PluginParser implements ChatQueryParser {
return;
}
pluginRecognizers.forEach(pluginRecognizer -> {
pluginRecognizer.recognize(parseContext, parseResp);
log.info("{} recallResult:{}", pluginRecognizer.getClass().getSimpleName(),
JsonUtil.toString(parseResp));
});
pluginRecognizers.forEach(
pluginRecognizer -> {
pluginRecognizer.recognize(parseContext, parseResp);
log.info(
"{} recallResult:{}",
pluginRecognizer.getClass().getSimpleName(),
JsonUtil.toString(parseResp));
});
}
}

View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.server.parser;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.server.agent.MultiTurnConfig;
import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
import com.tencent.supersonic.chat.server.pojo.ChatContext;
import com.tencent.supersonic.chat.server.pojo.ParseContext;
import com.tencent.supersonic.chat.server.service.ChatContextService;
import com.tencent.supersonic.chat.server.service.ChatManageService;
@@ -19,7 +20,6 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.server.pojo.ChatContext;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
import dev.langchain4j.data.message.AiMessage;
@@ -50,31 +50,33 @@ public class NL2SQLParser implements ChatQueryParser {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
private static final String REWRITE_USER_QUESTION_INSTRUCTION = ""
+ "#Role: You are a data product manager experienced in data requirements."
+ "#Task: Your will be provided with current and history questions asked by a user,"
+ "along with their mapped schema elements(metric, dimension and value),"
+ "please try understanding the semantics and rewrite a question."
+ "#Rules: "
+ "1.ALWAYS keep relevant entities, metrics, dimensions, values and date ranges."
+ "2.ONLY respond with the rewritten question."
+ "#Current Question: {{current_question}}"
+ "#Current Mapped Schema: {{current_schema}}"
+ "#History Question: {{history_question}}"
+ "#History Mapped Schema: {{history_schema}}"
+ "#History SQL: {{history_sql}}"
+ "#Rewritten Question: ";
private static final String REWRITE_USER_QUESTION_INSTRUCTION =
""
+ "#Role: You are a data product manager experienced in data requirements."
+ "#Task: Your will be provided with current and history questions asked by a user,"
+ "along with their mapped schema elements(metric, dimension and value),"
+ "please try understanding the semantics and rewrite a question."
+ "#Rules: "
+ "1.ALWAYS keep relevant entities, metrics, dimensions, values and date ranges."
+ "2.ONLY respond with the rewritten question."
+ "#Current Question: {{current_question}}"
+ "#Current Mapped Schema: {{current_schema}}"
+ "#History Question: {{history_question}}"
+ "#History Mapped Schema: {{history_schema}}"
+ "#History SQL: {{history_sql}}"
+ "#Rewritten Question: ";
private static final String REWRITE_ERROR_MESSAGE_INSTRUCTION = ""
+ "#Role: You are a data business partner who closely interacts with business people.\n"
+ "#Task: Your will be provided with user input, system output and some examples, "
+ "please respond shortly to teach user how to ask the right question, "
+ "by using `Examples` as references."
+ "#Rules: ALWAYS respond with the same language as the `Input`.\n"
+ "#Input: {{user_question}}\n"
+ "#Output: {{system_message}}\n"
+ "#Examples: {{examples}}\n"
+ "#Response: ";
private static final String REWRITE_ERROR_MESSAGE_INSTRUCTION =
""
+ "#Role: You are a data business partner who closely interacts with business people.\n"
+ "#Task: Your will be provided with user input, system output and some examples, "
+ "please respond shortly to teach user how to ask the right question, "
+ "by using `Examples` as references."
+ "#Rules: ALWAYS respond with the same language as the `Input`.\n"
+ "#Input: {{user_question}}\n"
+ "#Output: {{system_message}}\n"
+ "#Examples: {{examples}}\n"
+ "#Response: ";
@Override
public void parse(ParseContext parseContext, ParseResp parseResp) {
@@ -84,8 +86,8 @@ public class NL2SQLParser implements ChatQueryParser {
ChatContextService chatContextService = ContextUtils.getBean(ChatContextService.class);
ChatContext chatCtx = chatContextService.getOrCreateContext(parseContext.getChatId());
ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(
parseContext.getAgent().getModelConfig());
ChatLanguageModel chatLanguageModel =
ModelProvider.getChatModel(parseContext.getAgent().getModelConfig());
processMultiTurn(chatLanguageModel, parseContext);
QueryNLReq queryNLReq = QueryReqConverter.buildText2SqlQueryReq(parseContext, chatCtx);
@@ -96,11 +98,13 @@ public class NL2SQLParser implements ChatQueryParser {
if (ParseResp.ParseState.COMPLETED.equals(text2SqlParseResp.getState())) {
parseResp.getSelectedParses().addAll(text2SqlParseResp.getSelectedParses());
} else {
parseResp.setErrorMsg(rewriteErrorMessage(chatLanguageModel,
parseContext.getQueryText(),
text2SqlParseResp.getErrorMsg(),
queryNLReq.getDynamicExemplars(),
parseContext.getAgent().getExamples()));
parseResp.setErrorMsg(
rewriteErrorMessage(
chatLanguageModel,
parseContext.getQueryText(),
text2SqlParseResp.getErrorMsg(),
queryNLReq.getDynamicExemplars(),
parseContext.getAgent().getExamples()));
}
parseResp.setState(text2SqlParseResp.getState());
parseResp.getParseTimeCost().setSqlTime(text2SqlParseResp.getParseTimeCost().getSqlTime());
@@ -134,24 +138,35 @@ public class NL2SQLParser implements ChatQueryParser {
StringBuilder textBuilder = new StringBuilder();
textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
Optional<SchemaElement> metric = parseInfo.getMetrics().stream().findFirst();
metric.ifPresent(schemaElement ->
textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" "));
List<String> dimensionNames = parseInfo.getDimensions().stream()
.map(SchemaElement::getName).filter(Objects::nonNull).collect(Collectors.toList());
metric.ifPresent(
schemaElement ->
textBuilder.append("**指标:** ").append(schemaElement.getName()).append(" "));
List<String> dimensionNames =
parseInfo.getDimensions().stream()
.map(SchemaElement::getName)
.filter(Objects::nonNull)
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(dimensionNames)) {
textBuilder.append("**维度:** ").append(String.join(",", dimensionNames));
}
textBuilder.append("\n\n**筛选条件:** \n");
if (parseInfo.getDateInfo() != null) {
textBuilder.append("**数据时间:** ").append(parseInfo.getDateInfo().getStartDate()).append("~")
.append(parseInfo.getDateInfo().getEndDate()).append(" ");
textBuilder
.append("**数据时间:** ")
.append(parseInfo.getDateInfo().getStartDate())
.append("~")
.append(parseInfo.getDateInfo().getEndDate())
.append(" ");
}
if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())
|| CollectionUtils.isEmpty(parseInfo.getMetricFilters())) {
Set<QueryFilter> queryFilters = parseInfo.getDimensionFilters();
queryFilters.addAll(parseInfo.getMetricFilters());
for (QueryFilter queryFilter : queryFilters) {
textBuilder.append("**").append(queryFilter.getName()).append("**")
textBuilder
.append("**")
.append(queryFilter.getName())
.append("**")
.append(" ")
.append(queryFilter.getOperator().getValue())
.append(" ")
@@ -165,10 +180,13 @@ public class NL2SQLParser implements ChatQueryParser {
private void processMultiTurn(ChatLanguageModel chatLanguageModel, ParseContext parseContext) {
ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class);
MultiTurnConfig agentMultiTurnConfig = parseContext.getAgent().getMultiTurnConfig();
Boolean globalMultiTurnConfig = Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE));
Boolean globalMultiTurnConfig =
Boolean.valueOf(parserConfig.getParameterValue(PARSER_MULTI_TURN_ENABLE));
Boolean multiTurnConfig = agentMultiTurnConfig != null
? agentMultiTurnConfig.isEnableMultiTurn() : globalMultiTurnConfig;
Boolean multiTurnConfig =
agentMultiTurnConfig != null
? agentMultiTurnConfig.isEnableMultiTurn()
: globalMultiTurnConfig;
if (!Boolean.TRUE.equals(multiTurnConfig)) {
return;
}
@@ -186,7 +204,8 @@ public class NL2SQLParser implements ChatQueryParser {
SemanticParseInfo lastParseInfo = lastQuery.getParseInfos().get(0);
Long dataId = lastParseInfo.getDataSetId();
String curtMapStr = generateSchemaPrompt(currentMapResult.getMapInfo().getMatchedElements(dataId));
String curtMapStr =
generateSchemaPrompt(currentMapResult.getMapInfo().getMatchedElements(dataId));
String histMapStr = generateSchemaPrompt(lastParseInfo.getElementMatches());
String histSQL = lastParseInfo.getSqlInfo().getCorrectedS2SQL();
@@ -207,22 +226,31 @@ public class NL2SQLParser implements ChatQueryParser {
QueryNLReq rewrittenQueryNLReq = QueryReqConverter.buildText2SqlQueryReq(parseContext);
MapResp rewrittenQueryMapResult = chatLayerService.performMapping(rewrittenQueryNLReq);
parseContext.setMapInfo(rewrittenQueryMapResult.getMapInfo());
log.info("Last Query: {} Current Query: {}, Rewritten Query: {}",
lastQuery.getQueryText(), currentMapResult.getQueryText(), rewrittenQuery);
log.info(
"Last Query: {} Current Query: {}, Rewritten Query: {}",
lastQuery.getQueryText(),
currentMapResult.getQueryText(),
rewrittenQuery);
}
private String rewriteErrorMessage(ChatLanguageModel chatLanguageModel, String userQuestion,
String errMsg, List<Text2SQLExemplar> similarExemplars,
List<String> agentExamples) {
private String rewriteErrorMessage(
ChatLanguageModel chatLanguageModel,
String userQuestion,
String errMsg,
List<Text2SQLExemplar> similarExemplars,
List<String> agentExamples) {
Map<String, Object> variables = new HashMap<>();
variables.put("user_question", userQuestion);
variables.put("system_message", errMsg);
StringBuilder exampleStr = new StringBuilder();
similarExemplars.forEach(e ->
exampleStr.append(String.format("<Question:{%s},Schema:{%s}> ", e.getQuestion(), e.getDbSchema())));
agentExamples.forEach(e ->
exampleStr.append(String.format("<Question:{%s}> ", e)));
similarExemplars.forEach(
e ->
exampleStr.append(
String.format(
"<Question:{%s},Schema:{%s}> ",
e.getQuestion(), e.getDbSchema())));
agentExamples.forEach(e -> exampleStr.append(String.format("<Question:{%s}> ", e)));
variables.put("examples", exampleStr);
Prompt prompt = PromptTemplate.from(REWRITE_ERROR_MESSAGE_INSTRUCTION).apply(variables);
@@ -262,14 +290,18 @@ public class NL2SQLParser implements ChatQueryParser {
private List<QueryResp> getHistoryQueries(int chatId, int multiNum) {
ChatManageService chatManageService = ContextUtils.getBean(ChatManageService.class);
List<QueryResp> contextualParseInfoList = chatManageService.getChatQueries(chatId)
.stream()
.filter(q -> Objects.nonNull(q.getQueryResult())
&& q.getQueryResult().getQueryState() == QueryState.SUCCESS)
.collect(Collectors.toList());
List<QueryResp> contextualParseInfoList =
chatManageService.getChatQueries(chatId).stream()
.filter(
q ->
Objects.nonNull(q.getQueryResult())
&& q.getQueryResult().getQueryState()
== QueryState.SUCCESS)
.collect(Collectors.toList());
List<QueryResp> contextualList = contextualParseInfoList.subList(0,
Math.min(multiNum, contextualParseInfoList.size()));
List<QueryResp> contextualList =
contextualParseInfoList.subList(
0, Math.min(multiNum, contextualParseInfoList.size()));
Collections.reverse(contextualList);
return contextualList;
}
@@ -278,9 +310,8 @@ public class NL2SQLParser implements ChatQueryParser {
ExemplarServiceImpl exemplarManager = ContextUtils.getBean(ExemplarServiceImpl.class);
EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class);
String memoryCollectionName = embeddingConfig.getMemoryCollectionName(agentId);
List<Text2SQLExemplar> exemplars = exemplarManager.recallExemplars(memoryCollectionName,
queryNLReq.getQueryText(), 5);
List<Text2SQLExemplar> exemplars =
exemplarManager.recallExemplars(memoryCollectionName, queryNLReq.getQueryText(), 5);
queryNLReq.getDynamicExemplars().addAll(exemplars);
}
}

View File

@@ -10,8 +10,11 @@ import org.springframework.stereotype.Service;
public class ParserConfig extends ParameterConfig {
public static final Parameter PARSER_MULTI_TURN_ENABLE =
new Parameter("s2.parser.multi-turn.enable", "false",
"是否开启多轮对话", "开启多轮对话将消耗更多token",
"bool", "Parser相关配置");
new Parameter(
"s2.parser.multi-turn.enable",
"false",
"是否开启多轮对话",
"开启多轮对话将消耗更多token",
"bool",
"Parser相关配置");
}

View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.chat.server.pojo.ParseContext;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
public class PlainTextParser implements ChatQueryParser {
@Override
@@ -18,5 +17,4 @@ public class PlainTextParser implements ChatQueryParser {
parseResp.getSelectedParses().add(parseInfo);
parseResp.setState(ParseResp.ParseState.COMPLETED);
}
}

View File

@@ -10,61 +10,40 @@ import java.util.Date;
@Data
@TableName("s2_agent")
public class AgentDO {
/**
*
*/
/** */
@TableId(type = IdType.AUTO)
private Integer id;
/**
*
*/
/** */
private String name;
/**
*
*/
/** */
private String description;
/**
* 0 offline, 1 online
*/
/** 0 offline, 1 online */
private Integer status;
/**
*
*/
/** */
private String examples;
/**
*
*/
/** */
private String config;
/**
*
*/
/** */
private String createdBy;
/**
*
*/
/** */
private Date createdAt;
/**
*
*/
/** */
private String updatedBy;
/**
*
*/
/** */
private Date updatedAt;
/**
*
*/
/** */
private Integer enableSearch;
private Integer enableMemoryReview;
private String modelConfig;
private String multiTurnConfig;
@@ -72,5 +51,4 @@ public class AgentDO {
private String visualConfig;
private String promptConfig;
}

View File

@@ -1,18 +1,15 @@
package com.tencent.supersonic.chat.server.persistence.dataobject;
import java.util.Date;
import lombok.Data;
import lombok.ToString;
import java.util.Date;
@Data
@ToString
public class ChatConfigDO {
/**
* database auto-increment primary key
*/
/** database auto-increment primary key */
private Long id;
private Long modelId;
@@ -27,12 +24,10 @@ public class ChatConfigDO {
private String llmExamples;
/**
* record info
*/
/** record info */
private String createdBy;
private String updatedBy;
private Date createdAt;
private Date updatedAt;
}
}

View File

@@ -61,5 +61,4 @@ public class ChatMemoryDO {
@TableField("updated_at")
private Date updatedAt;
}

View File

@@ -4,54 +4,44 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_chat_query")
public class ChatQueryDO {
/**
*/
/** */
@TableId(type = IdType.AUTO)
private Long questionId;
/**
*/
/** */
private Integer agentId;
/**
*/
/** */
private Date createTime;
/**
*/
/** */
private String userName;
/**
*/
/** */
private Integer queryState;
/**
*/
/** */
private Long chatId;
/**
*/
/** */
private Integer score;
/**
*/
/** */
private String feedback;
/**
*/
/** */
private String queryText;
/**
*/
/** */
private String queryResult;
private String similarQueries;
private String parseTimeCost;
}

View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.server.persistence.dataobject;
import java.util.Date;
import lombok.Data;
import java.util.Date;
@Data
public class DictConfDO {
@@ -17,5 +17,4 @@ public class DictConfDO {
private String updatedBy;
private Date createdAt;
private Date updatedAt;
}
}

View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.chat.server.persistence.dataobject;
import java.util.Date;
import lombok.Data;
import lombok.ToString;
import org.apache.commons.codec.digest.DigestUtils;
import java.util.Date;
@Data
@ToString
public class DictTaskDO {
@@ -35,4 +35,4 @@ public class DictTaskDO {
public String getCommandMd5() {
return DigestUtils.md5Hex(command);
}
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.persistence.dataobject;
import com.tencent.supersonic.headless.core.config.DefaultMetric;
import com.tencent.supersonic.headless.core.config.Dim4Dict;
import lombok.Data;
@@ -9,7 +8,6 @@ import lombok.ToString;
import java.util.ArrayList;
import java.util.List;
@Data
@ToString
public class DimValueDO {
@@ -34,4 +32,4 @@ public class DimValueDO {
this.dimensions = dimensions;
return this;
}
}
}

View File

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@@ -36,5 +37,4 @@ public class PluginDO {
private String config;
private String comment;
}

View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.chat.server.persistence.dataobject;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.util.Date;
@@ -14,40 +14,25 @@ import java.util.Date;
@Getter
@AllArgsConstructor
public class StatisticsDO {
/**
* questionId
*/
/** questionId */
private Long questionId;
/**
* chatId
*/
/** chatId */
private Long chatId;
/**
* createTime
*/
/** createTime */
private Date createTime;
/**
* queryText
*/
/** queryText */
private String queryText;
/**
* userName
*/
/** userName */
private String userName;
/**
* interface
*/
/** interface */
private String interfaceName;
/**
* cost
*/
/** cost */
private Integer cost;
private Integer type;

View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface AgentDOMapper extends BaseMapper<AgentDO> {
}
public interface AgentDOMapper extends BaseMapper<AgentDO> {}

View File

@@ -2,9 +2,10 @@ package com.tencent.supersonic.chat.server.persistence.mapper;
import com.tencent.supersonic.chat.server.config.ChatConfigFilterInternal;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatConfigDO;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface ChatConfigMapper {

View File

@@ -2,9 +2,10 @@ package com.tencent.supersonic.chat.server.persistence.mapper;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface ChatMapper {
@@ -21,5 +22,4 @@ public interface ChatMapper {
boolean updateFeedback(QueryDO queryDO);
Boolean deleteChat(Long chatId, String userName);
}

View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ChatMemoryMapper extends BaseMapper<ChatMemoryDO> {
}
public interface ChatMemoryMapper extends BaseMapper<ChatMemoryDO> {}

View File

@@ -6,7 +6,6 @@ import org.apache.ibatis.annotations.Param;
import java.util.List;
@Mapper
public interface ChatParseMapper {
@@ -19,5 +18,4 @@ public interface ChatParseMapper {
List<ChatParseDO> getParseInfoList(List<Long> questionIds);
List<ChatParseDO> getContextualParseInfo(Integer chatId);
}

View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ChatQueryDOMapper extends BaseMapper<ChatQueryDO> {
}
public interface ChatQueryDOMapper extends BaseMapper<ChatQueryDO> {}

View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.PluginDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface PluginDOMapper extends BaseMapper<PluginDO> {
}
public interface PluginDOMapper extends BaseMapper<PluginDO> {}

View File

@@ -2,11 +2,11 @@ package com.tencent.supersonic.chat.server.persistence.mapper.custom;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface ShowCaseCustomMapper {
List<ChatQueryDO> queryShowCase(int start, int limit, int agentId, String userName);
}

View File

@@ -1,9 +1,8 @@
package com.tencent.supersonic.chat.server.persistence.repository;
import com.tencent.supersonic.chat.server.config.ChatConfig;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.server.config.ChatConfig;
import java.util.List;

View File

@@ -7,5 +7,4 @@ public interface ChatContextRepository {
ChatContext getOrCreateContext(Integer chatId);
void updateContext(ChatContext chatCtx);
}

View File

@@ -30,11 +30,12 @@ public interface ChatQueryRepository {
Long createChatQuery(ChatParseReq chatParseReq);
List<ChatParseDO> batchSaveParseInfo(ChatParseReq chatParseReq, ParseResp parseResult,
List<SemanticParseInfo> candidateParses);
List<ChatParseDO> batchSaveParseInfo(
ChatParseReq chatParseReq,
ParseResp parseResult,
List<SemanticParseInfo> candidateParses);
ChatParseDO getParseInfo(Long questionId, int parseId);
List<ChatParseDO> getParseInfoList(List<Long> questionIds);
}

View File

@@ -1,7 +1,8 @@
package com.tencent.supersonic.chat.server.persistence.repository;
import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO;
import java.util.List;
public interface ChatRepository {

View File

@@ -1,21 +1,21 @@
package com.tencent.supersonic.chat.server.persistence.repository.impl;
import com.tencent.supersonic.chat.server.config.ChatConfig;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.server.config.ChatConfigFilterInternal;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.server.config.ChatConfig;
import com.tencent.supersonic.chat.server.config.ChatConfigFilterInternal;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatConfigDO;
import com.tencent.supersonic.chat.server.persistence.mapper.ChatConfigMapper;
import com.tencent.supersonic.chat.server.util.ChatConfigHelper;
import com.tencent.supersonic.chat.server.persistence.repository.ChatConfigRepository;
import java.util.ArrayList;
import java.util.List;
import com.tencent.supersonic.chat.server.util.ChatConfigHelper;
import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Repository;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
@Repository
@Primary
public class ChatConfigRepositoryImpl implements ChatConfigRepository {
@@ -23,8 +23,8 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository {
private final ChatConfigHelper chatConfigHelper;
private final ChatConfigMapper chatConfigMapper;
public ChatConfigRepositoryImpl(ChatConfigHelper chatConfigHelper,
ChatConfigMapper chatConfigMapper) {
public ChatConfigRepositoryImpl(
ChatConfigHelper chatConfigHelper, ChatConfigMapper chatConfigMapper) {
this.chatConfigHelper = chatConfigHelper;
this.chatConfigMapper = chatConfigMapper;
}
@@ -41,7 +41,6 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository {
ChatConfigDO chaConfigDO = chatConfigHelper.chatConfig2DO(chaConfig);
return chatConfigMapper.editConfig(chaConfigDO);
}
@Override
@@ -52,9 +51,12 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository {
filterInternal.setStatus(filter.getStatus().getCode());
List<ChatConfigDO> chaConfigDOList = chatConfigMapper.search(filterInternal);
if (!CollectionUtils.isEmpty(chaConfigDOList)) {
chaConfigDOList.stream().forEach(chaConfigDO ->
chaConfigDescriptorList.add(chatConfigHelper
.chatConfigDO2Descriptor(chaConfigDO.getModelId(), chaConfigDO)));
chaConfigDOList.stream()
.forEach(
chaConfigDO ->
chaConfigDescriptorList.add(
chatConfigHelper.chatConfigDO2Descriptor(
chaConfigDO.getModelId(), chaConfigDO)));
}
return chaConfigDescriptorList;
}
@@ -64,5 +66,4 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository {
ChatConfigDO chaConfigPO = chatConfigMapper.fetchConfigByModelId(modelId);
return chatConfigHelper.chatConfigDO2Descriptor(modelId, chaConfigPO);
}
}

View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.chat.server.persistence.repository.impl;
import com.google.gson.Gson;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO;
import com.tencent.supersonic.chat.server.persistence.mapper.ChatContextMapper;
import com.tencent.supersonic.chat.server.persistence.repository.ChatContextRepository;
import com.tencent.supersonic.chat.server.pojo.ChatContext;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO;
import com.tencent.supersonic.chat.server.persistence.mapper.ChatContextMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Repository;
@@ -16,7 +16,6 @@ import org.springframework.stereotype.Repository;
@Slf4j
public class ChatContextRepositoryImpl implements ChatContextRepository {
private final ChatContextMapper chatContextMapper;
public ChatContextRepositoryImpl(ChatContextMapper chatContextMapper) {
@@ -50,8 +49,8 @@ public class ChatContextRepositoryImpl implements ChatContextRepository {
chatContext.setUser(contextDO.getUser());
chatContext.setQueryText(contextDO.getQueryText());
if (contextDO.getSemanticParse() != null && !contextDO.getSemanticParse().isEmpty()) {
SemanticParseInfo semanticParseInfo = JsonUtil.toObject(contextDO.getSemanticParse(),
SemanticParseInfo.class);
SemanticParseInfo semanticParseInfo =
JsonUtil.toObject(contextDO.getSemanticParse(), SemanticParseInfo.class);
chatContext.setParseInfo(semanticParseInfo);
}
return chatContext;

View File

@@ -38,5 +38,4 @@ public class ChatMemoryRepositoryImpl implements ChatMemoryRepository {
public List<ChatMemoryDO> getMemories(QueryWrapper<ChatMemoryDO> queryWrapper) {
return chatMemoryMapper.selectList(queryWrapper);
}
}

View File

@@ -7,6 +7,8 @@ import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
@@ -18,9 +20,7 @@ import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.PageUtils;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseTimeCostResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@@ -40,14 +40,11 @@ import java.util.stream.Collectors;
@Slf4j
public class ChatQueryRepositoryImpl implements ChatQueryRepository {
@Autowired
private ChatQueryDOMapper chatQueryDOMapper;
@Autowired private ChatQueryDOMapper chatQueryDOMapper;
@Autowired
private ChatParseMapper chatParseMapper;
@Autowired private ChatParseMapper chatParseMapper;
@Autowired
private ShowCaseCustomMapper showCaseCustomMapper;
@Autowired private ShowCaseCustomMapper showCaseCustomMapper;
@Override
public PageInfo<QueryResp> getChatQuery(PageQueryInfoReq pageQueryInfoReq, Long chatId) {
@@ -65,9 +62,9 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
queryWrapper.lambda().ne(ChatQueryDO::getQueryResult, "");
queryWrapper.lambda().orderByDesc(ChatQueryDO::getQuestionId);
PageInfo<ChatQueryDO> pageInfo = PageHelper.startPage(pageQueryInfoReq.getCurrent(),
pageQueryInfoReq.getPageSize())
.doSelectPageInfo(() -> chatQueryDOMapper.selectList(queryWrapper));
PageInfo<ChatQueryDO> pageInfo =
PageHelper.startPage(pageQueryInfoReq.getCurrent(), pageQueryInfoReq.getPageSize())
.doSelectPageInfo(() -> chatQueryDOMapper.selectList(queryWrapper));
PageInfo<QueryResp> chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo);
chatQueryVOPageInfo.setList(
@@ -104,24 +101,31 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
@Override
public List<QueryResp> queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId) {
return showCaseCustomMapper.queryShowCase(pageQueryInfoReq.getLimitStart(),
pageQueryInfoReq.getPageSize(), agentId, pageQueryInfoReq.getUserName())
.stream().map(this::convertTo)
return showCaseCustomMapper
.queryShowCase(
pageQueryInfoReq.getLimitStart(),
pageQueryInfoReq.getPageSize(),
agentId,
pageQueryInfoReq.getUserName())
.stream()
.map(this::convertTo)
.collect(Collectors.toList());
}
private QueryResp convertTo(ChatQueryDO chatQueryDO) {
QueryResp queryResp = new QueryResp();
BeanUtils.copyProperties(chatQueryDO, queryResp);
QueryResult queryResult = JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class);
QueryResult queryResult =
JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class);
if (queryResult != null) {
queryResult.setQueryId(chatQueryDO.getQuestionId());
queryResp.setQueryResult(queryResult);
}
queryResp.setSimilarQueries(JSONObject.parseArray(chatQueryDO.getSimilarQueries(),
SimilarQueryRecallResp.class));
queryResp.setParseTimeCost(JsonUtil.toObject(chatQueryDO.getParseTimeCost(),
ParseTimeCostResp.class));
queryResp.setSimilarQueries(
JSONObject.parseArray(
chatQueryDO.getSimilarQueries(), SimilarQueryRecallResp.class));
queryResp.setParseTimeCost(
JsonUtil.toObject(chatQueryDO.getParseTimeCost(), ParseTimeCostResp.class));
return queryResp;
}
@@ -143,8 +147,10 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
}
@Override
public List<ChatParseDO> batchSaveParseInfo(ChatParseReq chatParseReq,
ParseResp parseResult, List<SemanticParseInfo> candidateParses) {
public List<ChatParseDO> batchSaveParseInfo(
ChatParseReq chatParseReq,
ParseResp parseResult,
List<SemanticParseInfo> candidateParses) {
List<ChatParseDO> chatParseDOList = new ArrayList<>();
getChatParseDO(chatParseReq, parseResult.getQueryId(), candidateParses, chatParseDOList);
if (!CollectionUtils.isEmpty(candidateParses)) {
@@ -153,8 +159,11 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
return chatParseDOList;
}
public void getChatParseDO(ChatParseReq chatParseReq, Long queryId,
List<SemanticParseInfo> parses, List<ChatParseDO> chatParseDOList) {
public void getChatParseDO(
ChatParseReq chatParseReq,
Long queryId,
List<SemanticParseInfo> parses,
List<ChatParseDO> chatParseDOList) {
for (int i = 0; i < parses.size(); i++) {
ChatParseDO chatParseDO = new ChatParseDO();
chatParseDO.setChatId(chatParseReq.getChatId());
@@ -190,5 +199,4 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
public List<ChatParseDO> getParseInfoList(List<Long> questionIds) {
return chatParseMapper.getParseInfoList(questionIds);
}
}

View File

@@ -56,5 +56,4 @@ public class ChatRepositoryImpl implements ChatRepository {
public Boolean deleteChat(Long chatId, String userName) {
return chatMapper.deleteChat(chatId, userName);
}
}

View File

@@ -77,5 +77,4 @@ public class PluginRepositoryImpl implements PluginRepository {
public void deletePlugin(Long id) {
pluginDOMapper.deleteById(id);
}
}

View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.chat.server.plugin;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
@Data
@@ -14,37 +14,30 @@ public class ChatPlugin extends RecordInfo {
private Long id;
/***
* plugin type WEB_PAGE WEB_SERVICE
*/
/** * plugin type WEB_PAGE WEB_SERVICE */
private String type;
private List<Long> dataSetList = Lists.newArrayList();
/**
* description, for parsing
*/
/** description, for parsing */
private String pattern;
/**
* parse
*/
/** parse */
private ParseMode parseMode;
private String parseModeConfig;
private String name;
/**
* config for different plugin type
*/
/** config for different plugin type */
private String config;
private String comment;
public List<String> getExampleQuestionList() {
if (StringUtils.isNotBlank(parseModeConfig)) {
PluginParseConfig pluginParseConfig = JSONObject.parseObject(parseModeConfig, PluginParseConfig.class);
PluginParseConfig pluginParseConfig =
JSONObject.parseObject(parseModeConfig, PluginParseConfig.class);
return pluginParseConfig.getExamples();
}
return Lists.newArrayList();
@@ -57,5 +50,4 @@ public class ChatPlugin extends RecordInfo {
public Long getDefaultMode() {
return -1L;
}
}

View File

@@ -1,8 +1,6 @@
package com.tencent.supersonic.chat.server.plugin;
public enum ParseMode {
EMBEDDING_RECALL,
FUNCTION_CALL;
}
}

View File

@@ -46,11 +46,9 @@ import java.util.stream.Collectors;
@Component
public class PluginManager {
@Autowired
private EmbeddingConfig embeddingConfig;
@Autowired private EmbeddingConfig embeddingConfig;
@Autowired
private EmbeddingService embeddingService;
@Autowired private EmbeddingService embeddingService;
public static List<ChatPlugin> getPluginAgentCanSupport(ParseContext parseContext) {
PluginService pluginService = ContextUtils.getBean(PluginService.class);
@@ -59,14 +57,21 @@ public class PluginManager {
if (Objects.isNull(agent)) {
return plugins;
}
List<Long> pluginIds = getPluginTools(agent).stream().map(PluginTool::getPlugins)
.flatMap(Collection::stream).collect(Collectors.toList());
List<Long> pluginIds =
getPluginTools(agent).stream()
.map(PluginTool::getPlugins)
.flatMap(Collection::stream)
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(pluginIds)) {
return Lists.newArrayList();
}
plugins = plugins.stream().filter(plugin -> pluginIds.contains(plugin.getId()))
.collect(Collectors.toList());
log.info("plugins witch can be supported by cur agent :{} {}", agent.getName(),
plugins =
plugins.stream()
.filter(plugin -> pluginIds.contains(plugin.getId()))
.collect(Collectors.toList());
log.info(
"plugins witch can be supported by cur agent :{} {}",
agent.getName(),
plugins.stream().map(ChatPlugin::getName).collect(Collectors.toList()));
return plugins;
}
@@ -79,7 +84,8 @@ public class PluginManager {
if (CollectionUtils.isEmpty(tools)) {
return Lists.newArrayList();
}
return tools.stream().map(tool -> JSONObject.parseObject(tool, PluginTool.class))
return tools.stream()
.map(tool -> JSONObject.parseObject(tool, PluginTool.class))
.collect(Collectors.toList());
}
@@ -136,18 +142,23 @@ public class PluginManager {
public RetrieveQueryResult recognize(String embeddingText) {
RetrieveQuery retrieveQuery = RetrieveQuery.builder()
.queryTextsList(Collections.singletonList(embeddingText))
.build();
RetrieveQuery retrieveQuery =
RetrieveQuery.builder()
.queryTextsList(Collections.singletonList(embeddingText))
.build();
List<RetrieveQueryResult> resultList = embeddingService.retrieveQuery(embeddingConfig.getPresetCollection(),
retrieveQuery, embeddingConfig.getNResult());
List<RetrieveQueryResult> resultList =
embeddingService.retrieveQuery(
embeddingConfig.getPresetCollection(),
retrieveQuery,
embeddingConfig.getNResult());
if (CollectionUtils.isNotEmpty(resultList)) {
for (RetrieveQueryResult embeddingResp : resultList) {
List<Retrieval> embeddingRetrievals = embeddingResp.getRetrieval();
for (Retrieval embeddingRetrieval : embeddingRetrievals) {
embeddingRetrieval.setId(getPluginIdFromEmbeddingId(embeddingRetrieval.getId()));
embeddingRetrieval.setId(
getPluginIdFromEmbeddingId(embeddingRetrieval.getId()));
}
}
return resultList.get(0);
@@ -162,7 +173,8 @@ public class PluginManager {
int num = 0;
for (String pattern : exampleQuestions) {
TextSegment query = TextSegment.from(pattern);
TextSegmentConvert.addQueryId(query, generateUniqueEmbeddingId(num, plugin.getId()));
TextSegmentConvert.addQueryId(
query, generateUniqueEmbeddingId(num, plugin.getId()));
queries.add(query);
num++;
}
@@ -178,7 +190,7 @@ public class PluginManager {
return embeddingIdSet;
}
//num can not bigger than 100
// num can not bigger than 100
private String generateUniqueEmbeddingId(int num, Long pluginId) {
if (num < 10) {
return String.format("%s00%s", pluginId, num);
@@ -202,8 +214,8 @@ public class PluginManager {
return Pair.of(true, pluginMatchedDataSet);
}
Set<Long> matchedDataSet = Sets.newHashSet();
Map<Long, List<ParamOption>> paramOptionMap = paramOptions.stream()
.collect(Collectors.groupingBy(ParamOption::getDataSetId));
Map<Long, List<ParamOption>> paramOptionMap =
paramOptions.stream().collect(Collectors.groupingBy(ParamOption::getDataSetId));
for (Long dataSetId : paramOptionMap.keySet()) {
List<ParamOption> params = paramOptionMap.get(dataSetId);
if (CollectionUtils.isEmpty(params)) {
@@ -237,9 +249,13 @@ public class PluginManager {
if (org.springframework.util.CollectionUtils.isEmpty(schemaElementMatches)) {
return Sets.newHashSet();
}
return schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
return schemaElementMatches.stream()
.filter(
schemaElementMatch ->
SchemaElementType.VALUE.equals(
schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(
schemaElementMatch.getElement().getType()))
.map(SchemaElementMatch::getElement)
.map(SchemaElement::getId)
.collect(Collectors.toSet());
@@ -255,7 +271,9 @@ public class PluginManager {
return Lists.newArrayList();
}
return paramOptions.stream()
.filter(paramOption -> ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType()))
.filter(
paramOption ->
ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType()))
.collect(Collectors.toList());
}
@@ -273,5 +291,4 @@ public class PluginManager {
}
return pluginMatchedDataSet;
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.plugin;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@@ -22,5 +21,4 @@ public class PluginParseConfig implements Serializable {
private String name;
private String description;
}

View File

@@ -20,5 +20,4 @@ public class PluginQueryManager {
public static PluginSemanticQuery getPluginQuery(String queryMode) {
return pluginQueries.get(queryMode);
}
}

View File

@@ -20,5 +20,4 @@ public class PluginRecallResult {
private double score;
private double distance;
}

View File

@@ -22,16 +22,17 @@ public class ParamOption {
private Object value;
/**
* CUSTOM: the value is specified by the user
* SEMANTIC: the value of element
* FORWARD: only forward
* CUSTOM: the value is specified by the user SEMANTIC: the value of element FORWARD: only
* forward
*/
public enum ParamType {
CUSTOM, SEMANTIC, FORWARD
CUSTOM,
SEMANTIC,
FORWARD
}
public enum OptionType {
REQUIRED, OPTIONAL
REQUIRED,
OPTIONAL
}
}

View File

@@ -1,13 +1,13 @@
package com.tencent.supersonic.chat.server.plugin.build;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.server.plugin.PluginParseResult;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@@ -43,29 +43,40 @@ public abstract class PluginSemanticQuery {
protected Map<String, Object> getElementMap(PluginParseResult pluginParseResult) {
Map<String, Object> elementValueMap = new HashMap<>();
Map<Long, Object> filterValueMap = getFilterMap(pluginParseResult);
List<SchemaElementMatch> schemaElementMatchList = parseInfo.getElementMatches()
.stream().filter(schemaElementMatch -> schemaElementMatch.getFrequency() != null)
.sorted(Comparator.comparingLong(SchemaElementMatch::getFrequency).reversed())
.collect(Collectors.toList());
List<SchemaElementMatch> schemaElementMatchList =
parseInfo.getElementMatches().stream()
.filter(schemaElementMatch -> schemaElementMatch.getFrequency() != null)
.sorted(
Comparator.comparingLong(SchemaElementMatch::getFrequency)
.reversed())
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(schemaElementMatchList)) {
schemaElementMatchList.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
schemaElementMatchList.stream()
.filter(
schemaElementMatch ->
SchemaElementType.VALUE.equals(
schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(
schemaElementMatch.getElement().getType()))
.filter(schemaElementMatch -> schemaElementMatch.getSimilarity() == 1.0)
.forEach(schemaElementMatch -> {
Object queryFilterValue = filterValueMap.get(schemaElementMatch.getElement().getId());
if (queryFilterValue != null) {
if (String.valueOf(queryFilterValue).equals(String.valueOf(schemaElementMatch.getWord()))) {
elementValueMap.put(
String.valueOf(schemaElementMatch.getElement().getId()),
schemaElementMatch.getWord());
}
} else {
elementValueMap.computeIfAbsent(
String.valueOf(schemaElementMatch.getElement().getId()),
k -> schemaElementMatch.getWord());
}
});
.forEach(
schemaElementMatch -> {
Object queryFilterValue =
filterValueMap.get(schemaElementMatch.getElement().getId());
if (queryFilterValue != null) {
if (String.valueOf(queryFilterValue)
.equals(String.valueOf(schemaElementMatch.getWord()))) {
elementValueMap.put(
String.valueOf(
schemaElementMatch.getElement().getId()),
schemaElementMatch.getWord());
}
} else {
elementValueMap.computeIfAbsent(
String.valueOf(schemaElementMatch.getElement().getId()),
k -> schemaElementMatch.getWord());
}
});
}
return elementValueMap;
}
@@ -97,5 +108,4 @@ public abstract class PluginSemanticQuery {
public void setParseInfo(SemanticParseInfo parseInfo) {
this.parseInfo = parseInfo;
}
}

View File

@@ -15,5 +15,4 @@ public class WebBase {
public List<ParamOption> getParams() {
return paramOptions;
}
}

View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.chat.server.plugin.build.webpage;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.server.plugin.ChatPlugin;
import com.tencent.supersonic.chat.server.plugin.PluginParseResult;
import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
@@ -7,7 +8,6 @@ import com.tencent.supersonic.chat.server.plugin.build.PluginSemanticQuery;
import com.tencent.supersonic.chat.server.plugin.build.WebBase;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -41,12 +41,13 @@ public class WebPageQuery extends PluginSemanticQuery {
QueryResult queryResult = new QueryResult();
queryResult.setQueryMode(QUERY_MODE);
Map<String, Object> properties = parseInfo.getProperties();
PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)),
PluginParseResult.class);
PluginParseResult pluginParseResult =
JsonUtil.toObject(
JsonUtil.toString(properties.get(Constants.CONTEXT)),
PluginParseResult.class);
WebPageResp webPageResponse = buildResponse(pluginParseResult);
queryResult.setResponse(webPageResponse);
queryResult.setQueryState(QueryState.SUCCESS);
return queryResult;
}
}

View File

@@ -1,12 +1,10 @@
package com.tencent.supersonic.chat.server.plugin.build.webpage;
import com.tencent.supersonic.chat.server.plugin.build.WebBase;
import lombok.Data;
import java.util.List;
@Data
public class WebPageResp {
@@ -21,5 +19,4 @@ public class WebPageResp {
private WebBase webPage;
private List<WebBase> moreWebPage;
}

View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.server.plugin.build.webservice;
import com.alibaba.fastjson.JSON;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.server.plugin.ChatPlugin;
import com.tencent.supersonic.chat.server.plugin.PluginParseResult;
import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
@@ -11,7 +12,6 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpEntity;
@@ -28,7 +28,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
@Component
public class WebServiceQuery extends PluginSemanticQuery {
@@ -46,15 +45,18 @@ public class WebServiceQuery extends PluginSemanticQuery {
QueryResult queryResult = new QueryResult();
queryResult.setQueryMode(QUERY_MODE);
Map<String, Object> properties = parseInfo.getProperties();
PluginParseResult pluginParseResult = JsonUtil.toObject(
JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class);
PluginParseResult pluginParseResult =
JsonUtil.toObject(
JsonUtil.toString(properties.get(Constants.CONTEXT)),
PluginParseResult.class);
WebServiceResp webServiceResponse = buildResponse(pluginParseResult);
Object object = webServiceResponse.getResult();
// in order to show webServiceQuery result int frontend conveniently,
// webServiceResponse result format is consistent with queryByStruct result.
log.info("webServiceResponse result:{}", JsonUtil.toString(object));
try {
Map<String, Object> data = JsonUtil.toMap(JsonUtil.toString(object), String.class, Object.class);
Map<String, Object> data =
JsonUtil.toMap(JsonUtil.toString(object), String.class, Object.class);
if (data.get("resultList") != null) {
queryResult.setQueryResults((List<Map<String, Object>>) data.get("resultList"));
}
@@ -72,7 +74,9 @@ public class WebServiceQuery extends PluginSemanticQuery {
protected WebServiceResp buildResponse(PluginParseResult pluginParseResult) {
WebServiceResp webServiceResponse = new WebServiceResp();
ChatPlugin plugin = pluginParseResult.getPlugin();
WebBase webBase = fillWebBaseResult(JsonUtil.toObject(plugin.getConfig(), WebBase.class), pluginParseResult);
WebBase webBase =
fillWebBaseResult(
JsonUtil.toObject(plugin.getConfig(), WebBase.class), pluginParseResult);
webServiceResponse.setWebBase(webBase);
List<ParamOption> paramOptions = webBase.getParamOptions();
Map<String, Object> params = new HashMap<>();
@@ -86,7 +90,8 @@ public class WebServiceQuery extends PluginSemanticQuery {
Object objectResponse = null;
restTemplate = ContextUtils.getBean(RestTemplate.class);
try {
responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity, Object.class);
responseEntity =
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, Object.class);
objectResponse = responseEntity.getBody();
log.info("objectResponse:{}", objectResponse);
Map<String, Object> response = JsonUtil.objectToMap(objectResponse);
@@ -96,5 +101,4 @@ public class WebServiceQuery extends PluginSemanticQuery {
}
return webServiceResponse;
}
}

View File

@@ -1,15 +1,12 @@
package com.tencent.supersonic.chat.server.plugin.build.webservice;
import com.tencent.supersonic.chat.server.plugin.build.WebBase;
import lombok.Data;
@Data
public class WebServiceResp {
private WebBase webBase;
private Object result;
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.plugin.event;
import com.tencent.supersonic.chat.server.plugin.ChatPlugin;
import org.springframework.context.ApplicationEvent;

View File

@@ -22,5 +22,4 @@ public class PluginUpdateEvent extends ApplicationEvent {
public ChatPlugin getNewPlugin() {
return newPlugin;
}
}

View File

@@ -23,9 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* PluginParser defines the basic process and common methods for recalling plugins.
*/
/** PluginParser defines the basic process and common methods for recalling plugins. */
public abstract class PluginRecognizer {
public void recognize(ParseContext parseContext, ParseResp parseResp) {
@@ -43,16 +41,17 @@ public abstract class PluginRecognizer {
public abstract PluginRecallResult recallPlugin(ParseContext parseContext);
public void buildQuery(ParseContext parseContext, ParseResp parseResp,
PluginRecallResult pluginRecallResult) {
public void buildQuery(
ParseContext parseContext, ParseResp parseResp, PluginRecallResult pluginRecallResult) {
ChatPlugin plugin = pluginRecallResult.getPlugin();
Set<Long> dataSetIds = pluginRecallResult.getDataSetIds();
if (plugin.isContainsAllDataSet()) {
dataSetIds = Sets.newHashSet(-1L);
}
for (Long dataSetId : dataSetIds) {
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(dataSetId, plugin,
parseContext, pluginRecallResult.getDistance());
SemanticParseInfo semanticParseInfo =
buildSemanticParseInfo(
dataSetId, plugin, parseContext, pluginRecallResult.getDistance());
semanticParseInfo.setQueryMode(plugin.getType());
semanticParseInfo.setScore(pluginRecallResult.getScore());
parseResp.getSelectedParses().add(semanticParseInfo);
@@ -63,9 +62,10 @@ public abstract class PluginRecognizer {
return PluginManager.getPluginAgentCanSupport(parseContext);
}
protected SemanticParseInfo buildSemanticParseInfo(Long dataSetId, ChatPlugin plugin,
ParseContext parseContext, double distance) {
List<SchemaElementMatch> schemaElementMatches = parseContext.getMapInfo().getMatchedElements(dataSetId);
protected SemanticParseInfo buildSemanticParseInfo(
Long dataSetId, ChatPlugin plugin, ParseContext parseContext, double distance) {
List<SchemaElementMatch> schemaElementMatches =
parseContext.getMapInfo().getMatchedElements(dataSetId);
QueryFilters queryFilters = parseContext.getQueryFilters();
if (schemaElementMatches == null) {
schemaElementMatches = Lists.newArrayList();
@@ -96,18 +96,22 @@ public abstract class PluginRecognizer {
if (CollectionUtils.isEmpty(schemaElementMatches)) {
return;
}
schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.forEach(schemaElementMatch -> {
QueryFilter queryFilter = new QueryFilter();
queryFilter.setValue(schemaElementMatch.getWord());
queryFilter.setElementID(schemaElementMatch.getElement().getId());
queryFilter.setName(schemaElementMatch.getElement().getName());
queryFilter.setOperator(FilterOperatorEnum.EQUALS);
queryFilter.setBizName(schemaElementMatch.getElement().getBizName());
semanticParseInfo.getDimensionFilters().add(queryFilter);
});
schemaElementMatches.stream()
.filter(
schemaElementMatch ->
SchemaElementType.VALUE.equals(
schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(
schemaElementMatch.getElement().getType()))
.forEach(
schemaElementMatch -> {
QueryFilter queryFilter = new QueryFilter();
queryFilter.setValue(schemaElementMatch.getWord());
queryFilter.setElementID(schemaElementMatch.getElement().getId());
queryFilter.setName(schemaElementMatch.getElement().getName());
queryFilter.setOperator(FilterOperatorEnum.EQUALS);
queryFilter.setBizName(schemaElementMatch.getElement().getBizName());
semanticParseInfo.getDimensionFilters().add(queryFilter);
});
}
}

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.server.plugin.recognize.embedding;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.server.plugin.ParseMode;
import com.tencent.supersonic.chat.server.plugin.ChatPlugin;
import com.tencent.supersonic.chat.server.plugin.ParseMode;
import com.tencent.supersonic.chat.server.plugin.PluginManager;
import com.tencent.supersonic.chat.server.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer;
@@ -20,9 +20,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* EmbeddingRecallParser is an implementation of a recall plugin based on Embedding
*/
/** EmbeddingRecallParser is an implementation of a recall plugin based on Embedding */
@Slf4j
public class EmbeddingRecallRecognizer extends PluginRecognizer {
@@ -38,7 +36,8 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer {
return null;
}
List<ChatPlugin> plugins = getPluginList(parseContext);
Map<Long, ChatPlugin> pluginMap = plugins.stream().collect(Collectors.toMap(ChatPlugin::getId, p -> p));
Map<Long, ChatPlugin> pluginMap =
plugins.stream().collect(Collectors.toMap(ChatPlugin::getId, p -> p));
for (Retrieval embeddingRetrieval : embeddingRetrievals) {
ChatPlugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId()));
if (plugin == null) {
@@ -55,7 +54,11 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer {
double distance = embeddingRetrieval.getDistance();
double score = parseContext.getQueryText().length() * (1 - distance);
return PluginRecallResult.builder()
.plugin(plugin).dataSetIds(dataSetList).score(score).distance(distance).build();
.plugin(plugin)
.dataSetIds(dataSetList)
.score(score)
.distance(distance)
.build();
}
}
return null;
@@ -68,8 +71,10 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer {
List<Retrieval> embeddingRetrievals = embeddingResp.getRetrieval();
if (!CollectionUtils.isEmpty(embeddingRetrievals)) {
embeddingRetrievals = embeddingRetrievals.stream().sorted(Comparator.comparingDouble(o ->
Math.abs(o.getDistance()))).collect(Collectors.toList());
embeddingRetrievals =
embeddingRetrievals.stream()
.sorted(Comparator.comparingDouble(o -> Math.abs(o.getDistance())))
.collect(Collectors.toList());
embeddingResp.setRetrieval(embeddingRetrievals);
}
return embeddingRetrievals;
@@ -78,5 +83,4 @@ public class EmbeddingRecallRecognizer extends PluginRecognizer {
}
return Lists.newArrayList();
}
}

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.server.plugin.recognize.embedding;
import lombok.Data;
@Data
@@ -15,5 +14,4 @@ public class RecallRetrieval {
private String presetId;
private String query;
}

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat.server.plugin.recognize.embedding;
import lombok.Data;
import java.util.List;
@Data
@@ -10,6 +10,4 @@ public class RecallRetrievalResp {
private String query;
private List<RecallRetrieval> retrieval;
}

Some files were not shown because too many files have changed in this diff Show More