mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-12 20:51:48 +00:00
(improvement)(chat) Split chat into three modules: server, api, and core. (#594)
This commit is contained in:
@@ -1,23 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.api.pojo;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.NoArgsConstructor;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
@NoArgsConstructor
|
|
||||||
public class QueryContext {
|
|
||||||
|
|
||||||
private QueryReq request;
|
|
||||||
private List<SemanticQuery> candidateQueries = new ArrayList<>();
|
|
||||||
private SchemaMapInfo mapInfo = new SchemaMapInfo();
|
|
||||||
private SchemaModelClusterMapInfo modelClusterMapInfo = new SchemaModelClusterMapInfo();
|
|
||||||
|
|
||||||
public QueryContext(QueryReq request) {
|
|
||||||
this.request = request;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
package com.tencent.supersonic.chat.api.pojo;
|
package com.tencent.supersonic.chat.api.pojo;
|
||||||
|
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
import com.tencent.supersonic.common.pojo.ModelCluster;
|
import com.tencent.supersonic.common.pojo.ModelCluster;
|
||||||
import com.tencent.supersonic.common.pojo.Order;
|
import com.tencent.supersonic.common.pojo.Order;
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.api.pojo.response;
|
|||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq.TimeMode;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -26,6 +26,6 @@ public class ChatDefaultRichConfigResp {
|
|||||||
*/
|
*/
|
||||||
private String period = Constants.DAY;
|
private String period = Constants.DAY;
|
||||||
|
|
||||||
private ChatDefaultConfigReq.TimeMode timeMode;
|
private TimeMode timeMode;
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -22,6 +22,69 @@
|
|||||||
<artifactId>spring-context</artifactId>
|
<artifactId>spring-context</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.hankcs</groupId>
|
||||||
|
<artifactId>hanlp</artifactId>
|
||||||
|
<version>${hanlp.version}</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-client</artifactId>
|
||||||
|
<version>${hadoop.version}</version>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
|
<artifactId>zookeeper</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.servlet</groupId>
|
||||||
|
<artifactId>servlet-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-hdfs</artifactId>
|
||||||
|
<version>${hadoop.version}</version>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
|
<artifactId>zookeeper</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.servlet</groupId>
|
||||||
|
<artifactId>servlet-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-common</artifactId>
|
||||||
|
<version>${hadoop.version}</version>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-log4j12</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>log4j</groupId>
|
||||||
|
<artifactId>log4j</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
|
<artifactId>zookeeper</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.apache.curator</groupId>
|
||||||
|
<artifactId>*</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.servlet</groupId>
|
||||||
|
<artifactId>servlet-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.testng</groupId>
|
<groupId>org.testng</groupId>
|
||||||
<artifactId>testng</artifactId>
|
<artifactId>testng</artifactId>
|
||||||
@@ -70,11 +133,6 @@
|
|||||||
<artifactId>h2</artifactId>
|
<artifactId>h2</artifactId>
|
||||||
<version>${h2.version}</version>
|
<version>${h2.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<groupId>com.tencent.supersonic</groupId>
|
|
||||||
<artifactId>chat-knowledge</artifactId>
|
|
||||||
<version>${project.version}</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.tencent.supersonic</groupId>
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
<artifactId>headless-api</artifactId>
|
<artifactId>headless-api</artifactId>
|
||||||
@@ -104,6 +162,12 @@
|
|||||||
<version>${mockito-inline.version}</version>
|
<version>${mockito-inline.version}</version>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
|
<artifactId>headless-server</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.hankcs.hanlp.collection.trie.bintrie;
|
package com.hankcs.hanlp.collection.trie.bintrie;
|
||||||
|
|
||||||
import com.hankcs.hanlp.corpus.io.ByteArray;
|
import com.hankcs.hanlp.corpus.io.ByteArray;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.LoadRemoveService;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.knowledge.service.LoadRemoveService;
|
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.ObjectInput;
|
import java.io.ObjectInput;
|
||||||
@@ -3,7 +3,7 @@ package com.hankcs.hanlp.seg.common;
|
|||||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||||
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
||||||
import com.hankcs.hanlp.dictionary.CustomDictionary;
|
import com.hankcs.hanlp.dictionary.CustomDictionary;
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.ToString;
|
import lombok.ToString;
|
||||||
|
|
||||||
@@ -1,15 +1,18 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
import com.alibaba.fastjson.JSONObject;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.common.pojo.RecordInfo;
|
import com.tencent.supersonic.common.pojo.RecordInfo;
|
||||||
import java.util.Objects;
|
import java.util.Collection;
|
||||||
import lombok.Data;
|
import java.util.HashSet;
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public class Agent extends RecordInfo {
|
public class Agent extends RecordInfo {
|
||||||
@@ -23,6 +26,7 @@ public class Agent extends RecordInfo {
|
|||||||
private Integer status;
|
private Integer status;
|
||||||
private List<String> examples;
|
private List<String> examples;
|
||||||
private String agentConfig;
|
private String agentConfig;
|
||||||
|
|
||||||
public List<String> getTools(AgentToolType type) {
|
public List<String> getTools(AgentToolType type) {
|
||||||
Map map = JSONObject.parseObject(agentConfig, Map.class);
|
Map map = JSONObject.parseObject(agentConfig, Map.class);
|
||||||
if (CollectionUtils.isEmpty(map) || map.get("tools") == null) {
|
if (CollectionUtils.isEmpty(map) || map.get("tools") == null) {
|
||||||
@@ -45,4 +49,27 @@ public class Agent extends RecordInfo {
|
|||||||
return enableSearch != null && enableSearch == 1;
|
return enableSearch != null && enableSearch == 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean containsAllModel(Set<Long> detectModelIds) {
|
||||||
|
return !CollectionUtils.isEmpty(detectModelIds) && detectModelIds.contains(-1L);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<NL2SQLTool> getParserTools(AgentToolType agentToolType) {
|
||||||
|
List<String> tools = this.getTools(agentToolType);
|
||||||
|
if (CollectionUtils.isEmpty(tools)) {
|
||||||
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
|
return tools.stream().map(tool -> JSONObject.parseObject(tool, NL2SQLTool.class))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
public Set<Long> getModelIds(AgentToolType agentToolType) {
|
||||||
|
List<NL2SQLTool> commonAgentTools = getParserTools(agentToolType);
|
||||||
|
if (CollectionUtils.isEmpty(commonAgentTools)) {
|
||||||
|
return new HashSet<>();
|
||||||
|
}
|
||||||
|
return commonAgentTools.stream().map(NL2SQLTool::getModelIds)
|
||||||
|
.filter(modelIds -> !CollectionUtils.isEmpty(modelIds))
|
||||||
|
.flatMap(Collection::stream)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
public enum AgentToolType {
|
public enum AgentToolType {
|
||||||
NL2SQL_RULE,
|
NL2SQL_RULE,
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
|
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.SqlGenerationMode;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode;
|
||||||
import com.tencent.supersonic.common.service.SysParameterService;
|
import com.tencent.supersonic.common.service.SysParameterService;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -1,22 +1,14 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticCorrector;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.apache.commons.lang3.tuple.Pair;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -24,6 +16,10 @@ import java.util.Map;
|
|||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* basic semantic correction functionality, offering common methods and an
|
* basic semantic correction functionality, offering common methods and an
|
||||||
@@ -32,23 +28,23 @@ import java.util.stream.Collectors;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
||||||
|
|
||||||
public void correct(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void correct(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
try {
|
try {
|
||||||
if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getCorrectS2SQL())) {
|
if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getCorrectS2SQL())) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
doCorrect(queryReq, semanticParseInfo);
|
doCorrect(queryContext, semanticParseInfo);
|
||||||
log.info("sqlCorrection:{} sql:{}", this.getClass().getSimpleName(), semanticParseInfo.getSqlInfo());
|
log.info("sqlCorrection:{} sql:{}", this.getClass().getSimpleName(), semanticParseInfo.getSqlInfo());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error(String.format("correct error,sqlInfo:%s", semanticParseInfo.getSqlInfo()), e);
|
log.error(String.format("correct error,sqlInfo:%s", semanticParseInfo.getSqlInfo()), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo);
|
public abstract void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo);
|
||||||
|
|
||||||
protected Map<String, String> getFieldNameMap(Set<Long> modelIds) {
|
protected Map<String, String> getFieldNameMap(QueryContext queryContext, Set<Long> modelIds) {
|
||||||
|
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
|
|
||||||
List<SchemaElement> dbAllFields = new ArrayList<>();
|
List<SchemaElement> dbAllFields = new ArrayList<>();
|
||||||
dbAllFields.addAll(semanticSchema.getMetrics());
|
dbAllFields.addAll(semanticSchema.getMetrics());
|
||||||
@@ -101,12 +97,12 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
|||||||
semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceFields);
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceFields);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void addAggregateToMetric(SemanticParseInfo semanticParseInfo) {
|
protected void addAggregateToMetric(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
//add aggregate to all metric
|
//add aggregate to all metric
|
||||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||||
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
||||||
|
|
||||||
List<SchemaElement> metrics = getMetricElements(modelIds);
|
List<SchemaElement> metrics = getMetricElements(queryContext, modelIds);
|
||||||
|
|
||||||
Map<String, String> metricToAggregate = metrics.stream()
|
Map<String, String> metricToAggregate = metrics.stream()
|
||||||
.map(schemaElement -> {
|
.map(schemaElement -> {
|
||||||
@@ -131,8 +127,8 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
|||||||
semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql);
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<SchemaElement> getMetricElements(Set<Long> modelIds) {
|
protected List<SchemaElement> getMetricElements(QueryContext queryContext, Set<Long> modelIds) {
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
return semanticSchema.getMetrics(modelIds);
|
return semanticSchema.getMetrics(modelIds);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
/**
|
/**
|
||||||
@@ -11,7 +11,7 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
public class FromCorrector extends BaseSemanticCorrector {
|
public class FromCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
String modelName = semanticParseInfo.getModel().getName();
|
String modelName = semanticParseInfo.getModel().getName();
|
||||||
String correctSql = SqlParserReplaceHelper
|
String correctSql = SqlParserReplaceHelper
|
||||||
.replaceTable(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), modelName);
|
.replaceTable(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), modelName);
|
||||||
@@ -1,21 +1,18 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Perform SQL corrections on the "Group by" section in S2SQL.
|
* Perform SQL corrections on the "Group by" section in S2SQL.
|
||||||
@@ -24,19 +21,19 @@ import java.util.stream.Collectors;
|
|||||||
public class GroupByCorrector extends BaseSemanticCorrector {
|
public class GroupByCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
|
||||||
addGroupByFields(semanticParseInfo);
|
addGroupByFields(queryContext, semanticParseInfo);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addGroupByFields(SemanticParseInfo semanticParseInfo) {
|
private void addGroupByFields(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
||||||
|
|
||||||
//add dimension group by
|
//add dimension group by
|
||||||
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
|
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
|
||||||
String correctS2SQL = sqlInfo.getCorrectS2SQL();
|
String correctS2SQL = sqlInfo.getCorrectS2SQL();
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
//add alias field name
|
//add alias field name
|
||||||
Set<String> dimensions = semanticSchema.getDimensions(modelIds).stream()
|
Set<String> dimensions = semanticSchema.getDimensions(modelIds).stream()
|
||||||
.flatMap(
|
.flatMap(
|
||||||
@@ -77,15 +74,15 @@ public class GroupByCorrector extends BaseSemanticCorrector {
|
|||||||
.collect(Collectors.toSet());
|
.collect(Collectors.toSet());
|
||||||
semanticParseInfo.getSqlInfo().setCorrectS2SQL(SqlParserAddHelper.addGroupBy(correctS2SQL, groupByFields));
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(SqlParserAddHelper.addGroupBy(correctS2SQL, groupByFields));
|
||||||
|
|
||||||
addAggregate(semanticParseInfo);
|
addAggregate(queryContext, semanticParseInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addAggregate(SemanticParseInfo semanticParseInfo) {
|
private void addAggregate(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
List<String> sqlGroupByFields = SqlParserSelectHelper.getGroupByFields(
|
List<String> sqlGroupByFields = SqlParserSelectHelper.getGroupByFields(
|
||||||
semanticParseInfo.getSqlInfo().getCorrectS2SQL());
|
semanticParseInfo.getSqlInfo().getCorrectS2SQL());
|
||||||
if (CollectionUtils.isEmpty(sqlGroupByFields)) {
|
if (CollectionUtils.isEmpty(sqlGroupByFields)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
addAggregateToMetric(semanticParseInfo);
|
addAggregateToMetric(queryContext, semanticParseInfo);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,18 +1,14 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import net.sf.jsqlparser.expression.Expression;
|
import net.sf.jsqlparser.expression.Expression;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -24,20 +20,20 @@ import org.springframework.util.CollectionUtils;
|
|||||||
public class HavingCorrector extends BaseSemanticCorrector {
|
public class HavingCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
|
||||||
//add aggregate to all metric
|
//add aggregate to all metric
|
||||||
addHaving(semanticParseInfo);
|
addHaving(queryContext, semanticParseInfo);
|
||||||
|
|
||||||
//add having expression filed to select
|
//add having expression filed to select
|
||||||
addHavingToSelect(semanticParseInfo);
|
addHavingToSelect(semanticParseInfo);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addHaving(SemanticParseInfo semanticParseInfo) {
|
private void addHaving(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
||||||
|
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
|
|
||||||
Set<String> metrics = semanticSchema.getMetrics(modelIds).stream()
|
Set<String> metrics = semanticSchema.getMetrics(modelIds).stream()
|
||||||
.map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
|
.map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.ParseResult;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.ParseResult;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.AggregateEnum;
|
import com.tencent.supersonic.common.util.jsqlparser.AggregateEnum;
|
||||||
@@ -24,7 +24,7 @@ import org.springframework.util.CollectionUtils;
|
|||||||
public class SchemaCorrector extends BaseSemanticCorrector {
|
public class SchemaCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
|
||||||
correctAggFunction(semanticParseInfo);
|
correctAggFunction(semanticParseInfo);
|
||||||
|
|
||||||
@@ -34,7 +34,7 @@ public class SchemaCorrector extends BaseSemanticCorrector {
|
|||||||
|
|
||||||
updateFieldValueByLinkingValue(semanticParseInfo);
|
updateFieldValueByLinkingValue(semanticParseInfo);
|
||||||
|
|
||||||
correctFieldName(semanticParseInfo);
|
correctFieldName(queryContext, semanticParseInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void correctAggFunction(SemanticParseInfo semanticParseInfo) {
|
private void correctAggFunction(SemanticParseInfo semanticParseInfo) {
|
||||||
@@ -50,8 +50,8 @@ public class SchemaCorrector extends BaseSemanticCorrector {
|
|||||||
sqlInfo.setCorrectS2SQL(replaceAlias);
|
sqlInfo.setCorrectS2SQL(replaceAlias);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void correctFieldName(SemanticParseInfo semanticParseInfo) {
|
private void correctFieldName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
Map<String, String> fieldNameMap = getFieldNameMap(semanticParseInfo.getModel().getModelIds());
|
Map<String, String> fieldNameMap = getFieldNameMap(queryContext, semanticParseInfo.getModel().getModelIds());
|
||||||
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
|
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
|
||||||
String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap);
|
String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap);
|
||||||
sqlInfo.setCorrectS2SQL(sql);
|
sqlInfo.setCorrectS2SQL(sql);
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -14,7 +14,7 @@ import org.springframework.util.CollectionUtils;
|
|||||||
public class SelectCorrector extends BaseSemanticCorrector {
|
public class SelectCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||||
List<String> aggregateFields = SqlParserSelectHelper.getAggregateFields(correctS2SQL);
|
List<String> aggregateFields = SqlParserSelectHelper.getAggregateFields(correctS2SQL);
|
||||||
List<String> selectFields = SqlParserSelectHelper.getSelectFields(correctS2SQL);
|
List<String> selectFields = SqlParserSelectHelper.getSelectFields(correctS2SQL);
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.api.component;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A semantic corrector checks validity of extracted semantic information and
|
* A semantic corrector checks validity of extracted semantic information and
|
||||||
@@ -9,5 +9,5 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|||||||
*/
|
*/
|
||||||
public interface SemanticCorrector {
|
public interface SemanticCorrector {
|
||||||
|
|
||||||
void correct(QueryReq queryReq, SemanticParseInfo semanticParseInfo);
|
void correct(QueryContext queryContext, SemanticParseInfo semanticParseInfo);
|
||||||
}
|
}
|
||||||
@@ -1,20 +1,24 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
|
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.S2SqlDateHelper;
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.S2SqlDateHelper;
|
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.StringUtil;
|
import com.tencent.supersonic.common.util.StringUtil;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import net.sf.jsqlparser.JSQLParserException;
|
import net.sf.jsqlparser.JSQLParserException;
|
||||||
import net.sf.jsqlparser.expression.Expression;
|
import net.sf.jsqlparser.expression.Expression;
|
||||||
@@ -23,13 +27,6 @@ import org.apache.commons.lang3.StringUtils;
|
|||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Perform SQL corrections on the "Where" section in S2SQL.
|
* Perform SQL corrections on the "Where" section in S2SQL.
|
||||||
*/
|
*/
|
||||||
@@ -37,19 +34,19 @@ import java.util.stream.Collectors;
|
|||||||
public class WhereCorrector extends BaseSemanticCorrector {
|
public class WhereCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
|
||||||
addDateIfNotExist(semanticParseInfo);
|
addDateIfNotExist(queryContext, semanticParseInfo);
|
||||||
|
|
||||||
parserDateDiffFunction(semanticParseInfo);
|
parserDateDiffFunction(semanticParseInfo);
|
||||||
|
|
||||||
addQueryFilter(queryReq, semanticParseInfo);
|
addQueryFilter(queryContext, semanticParseInfo);
|
||||||
|
|
||||||
updateFieldValueByTechName(semanticParseInfo);
|
updateFieldValueByTechName(queryContext, semanticParseInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addQueryFilter(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
private void addQueryFilter(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
String queryFilter = getQueryFilter(queryReq.getQueryFilters());
|
String queryFilter = getQueryFilter(queryContext.getQueryFilters());
|
||||||
|
|
||||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||||
|
|
||||||
@@ -72,11 +69,11 @@ public class WhereCorrector extends BaseSemanticCorrector {
|
|||||||
semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL);
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addDateIfNotExist(SemanticParseInfo semanticParseInfo) {
|
private void addDateIfNotExist(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||||
List<String> whereFields = SqlParserSelectHelper.getWhereFields(correctS2SQL);
|
List<String> whereFields = SqlParserSelectHelper.getWhereFields(correctS2SQL);
|
||||||
if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
|
if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
|
||||||
String currentDate = S2SqlDateHelper.getReferenceDate(semanticParseInfo.getModelId());
|
String currentDate = S2SqlDateHelper.getReferenceDate(queryContext, semanticParseInfo.getModelId());
|
||||||
if (StringUtils.isNotBlank(currentDate)) {
|
if (StringUtils.isNotBlank(currentDate)) {
|
||||||
correctS2SQL = SqlParserAddHelper.addParenthesisToWhere(correctS2SQL);
|
correctS2SQL = SqlParserAddHelper.addParenthesisToWhere(correctS2SQL);
|
||||||
correctS2SQL = SqlParserAddHelper.addWhere(
|
correctS2SQL = SqlParserAddHelper.addWhere(
|
||||||
@@ -100,8 +97,8 @@ public class WhereCorrector extends BaseSemanticCorrector {
|
|||||||
.collect(Collectors.joining(Constants.AND_UPPER));
|
.collect(Collectors.joining(Constants.AND_UPPER));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void updateFieldValueByTechName(SemanticParseInfo semanticParseInfo) {
|
private void updateFieldValueByTechName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
||||||
List<SchemaElement> dimensions = semanticSchema.getDimensions(modelIds);
|
List<SchemaElement> dimensions = semanticSchema.getDimensions(modelIds);
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
|
||||||
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
public enum DictUpdateMode {
|
public enum DictUpdateMode {
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge.dictionary;
|
||||||
|
|
||||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||||
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.hankcs.hanlp.corpus.io.IIOAdapter;
|
import com.hankcs.hanlp.corpus.io.IIOAdapter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.tencent.supersonic.knowledge.service;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
import com.tencent.supersonic.chat.core.utils.NatureHelper;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.config.LocalFileConfig;
|
||||||
import java.io.BufferedWriter;
|
import java.io.BufferedWriter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
@@ -10,7 +11,6 @@ import java.nio.file.Paths;
|
|||||||
import java.nio.file.StandardCopyOption;
|
import java.nio.file.StandardCopyOption;
|
||||||
import java.nio.file.StandardOpenOption;
|
import java.nio.file.StandardOpenOption;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import static com.hankcs.hanlp.utility.Predefine.logger;
|
import static com.hankcs.hanlp.utility.Predefine.logger;
|
||||||
|
|
||||||
@@ -15,8 +15,8 @@ import com.hankcs.hanlp.seg.common.Term;
|
|||||||
import com.hankcs.hanlp.utility.LexiconUtility;
|
import com.hankcs.hanlp.utility.LexiconUtility;
|
||||||
import com.hankcs.hanlp.utility.Predefine;
|
import com.hankcs.hanlp.utility.Predefine;
|
||||||
import com.hankcs.hanlp.utility.TextUtility;
|
import com.hankcs.hanlp.utility.TextUtility;
|
||||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
import com.tencent.supersonic.chat.core.knowledge.dictionary.DictionaryAttributeUtil;
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
import java.io.BufferedOutputStream;
|
import java.io.BufferedOutputStream;
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
@@ -1,17 +1,19 @@
|
|||||||
package com.tencent.supersonic.knowledge.service;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.hankcs.hanlp.collection.trie.bintrie.BaseNode;
|
import com.hankcs.hanlp.collection.trie.bintrie.BaseNode;
|
||||||
import com.hankcs.hanlp.collection.trie.bintrie.BinTrie;
|
import com.hankcs.hanlp.collection.trie.bintrie.BinTrie;
|
||||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||||
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.dictionary.DictionaryAttributeUtil;
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictionaryAttributeUtil;
|
import java.util.ArrayList;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.PriorityQueue;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
@@ -156,5 +158,15 @@ public class SearchService {
|
|||||||
suffixTrie.remove(dictWord.getWord());
|
suffixTrie.remove(dictWord.getWord());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static List<String> getDimensionValue(DimensionValueReq dimensionValueReq) {
|
||||||
|
String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT
|
||||||
|
+ dimensionValueReq.getElementID();
|
||||||
|
PriorityQueue<Term> terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature);
|
||||||
|
if (org.apache.commons.collections.CollectionUtils.isEmpty(terms)) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
return terms.stream().map(term -> term.getWord()).collect(Collectors.toList());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -1,13 +1,11 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -1,13 +1,12 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -1,13 +1,11 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
@@ -1,13 +1,12 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
@@ -1,8 +1,7 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import com.google.common.cache.Cache;
|
import com.google.common.cache.Cache;
|
||||||
import com.google.common.cache.CacheBuilder;
|
import com.google.common.cache.CacheBuilder;
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
@@ -24,13 +24,12 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
|
|||||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||||
import com.tencent.supersonic.headless.server.service.QueryService;
|
import com.tencent.supersonic.headless.server.service.QueryService;
|
||||||
import com.tencent.supersonic.headless.server.service.SchemaService;
|
import com.tencent.supersonic.headless.server.service.SchemaService;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
|
public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
|
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
|
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
|
||||||
@@ -11,6 +11,7 @@ import com.google.gson.Gson;
|
|||||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||||
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.core.config.DefaultSemanticConfig;
|
||||||
import com.tencent.supersonic.common.pojo.ResultData;
|
import com.tencent.supersonic.common.pojo.ResultData;
|
||||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||||
import com.tencent.supersonic.common.pojo.enums.ReturnCode;
|
import com.tencent.supersonic.common.pojo.enums.ReturnCode;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.api.component;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
@@ -1,14 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.service.SemanticService;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@@ -68,10 +66,10 @@ public abstract class BaseMapper implements SchemaMapper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public SchemaElement getSchemaElement(Long modelId, SchemaElementType elementType, Long elementID) {
|
public SchemaElement getSchemaElement(Long modelId, SchemaElementType elementType, Long elementID,
|
||||||
|
SemanticSchema semanticSchema) {
|
||||||
SchemaElement element = new SchemaElement();
|
SchemaElement element = new SchemaElement();
|
||||||
SemanticService schemaService = ContextUtils.getBean(SemanticService.class);
|
ModelSchema modelSchema = semanticSchema.getModelSchemaMap().get(modelId);
|
||||||
ModelSchema modelSchema = schemaService.getModelSchema(modelId);
|
|
||||||
if (Objects.isNull(modelSchema)) {
|
if (Objects.isNull(modelSchema)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
import com.tencent.supersonic.chat.core.utils.NatureHelper;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -102,7 +102,7 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public List<T> getMatches(QueryContext queryContext, List<Term> terms) {
|
public List<T> getMatches(QueryContext queryContext, List<Term> terms) {
|
||||||
Set<Long> detectModelIds = mapperHelper.getModelIds(queryContext.getRequest());
|
Set<Long> detectModelIds = mapperHelper.getModelIds(queryContext.getRequest(), queryContext.getAgent());
|
||||||
terms = filterByModelIds(terms, detectModelIds);
|
terms = filterByModelIds(terms, detectModelIds);
|
||||||
Map<MatchText, List<T>> matchResult = match(queryContext, terms, detectModelIds);
|
Map<MatchText, List<T>> matchResult = match(queryContext, terms, detectModelIds);
|
||||||
List<T> matches = new ArrayList<>();
|
List<T> matches = new ArrayList<>();
|
||||||
@@ -1,13 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DatabaseMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -33,14 +32,12 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy<DatabaseMapResult>
|
|||||||
private OptimizationConfig optimizationConfig;
|
private OptimizationConfig optimizationConfig;
|
||||||
@Autowired
|
@Autowired
|
||||||
private MapperHelper mapperHelper;
|
private MapperHelper mapperHelper;
|
||||||
@Autowired
|
|
||||||
private SchemaService schemaService;
|
|
||||||
private List<SchemaElement> allElements;
|
private List<SchemaElement> allElements;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Map<MatchText, List<DatabaseMapResult>> match(QueryContext queryContext, List<Term> terms,
|
public Map<MatchText, List<DatabaseMapResult>> match(QueryContext queryContext, List<Term> terms,
|
||||||
Set<Long> detectModelIds) {
|
Set<Long> detectModelIds) {
|
||||||
this.allElements = getSchemaElements();
|
this.allElements = getSchemaElements(queryContext);
|
||||||
return super.match(queryContext, terms, detectModelIds);
|
return super.match(queryContext, terms, detectModelIds);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,7 +59,7 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy<DatabaseMapResult>
|
|||||||
if (StringUtils.isBlank(detectSegment)) {
|
if (StringUtils.isBlank(detectSegment)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Set<Long> modelIds = mapperHelper.getModelIds(queryContext.getRequest());
|
Set<Long> modelIds = mapperHelper.getModelIds(queryContext.getRequest(), queryContext.getAgent());
|
||||||
|
|
||||||
Double metricDimensionThresholdConfig = getThreshold(queryContext);
|
Double metricDimensionThresholdConfig = getThreshold(queryContext);
|
||||||
|
|
||||||
@@ -90,10 +87,10 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy<DatabaseMapResult>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<SchemaElement> getSchemaElements() {
|
private List<SchemaElement> getSchemaElements(QueryContext queryContext) {
|
||||||
List<SchemaElement> allElements = new ArrayList<>();
|
List<SchemaElement> allElements = new ArrayList<>();
|
||||||
allElements.addAll(schemaService.getSemanticSchema().getDimensions());
|
allElements.addAll(queryContext.getSemanticSchema().getDimensions());
|
||||||
allElements.addAll(schemaService.getSemanticSchema().getMetrics());
|
allElements.addAll(queryContext.getSemanticSchema().getMetrics());
|
||||||
return allElements;
|
return allElements;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
import com.alibaba.fastjson.JSONObject;
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.EmbeddingResult;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.EmbeddingResult;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder;
|
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
@@ -43,7 +43,8 @@ public class EmbeddingMapper extends BaseMapper {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
long modelId = Long.parseLong(modelIdStr);
|
long modelId = Long.parseLong(modelIdStr);
|
||||||
schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId);
|
schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId,
|
||||||
|
queryContext.getSemanticSchema());
|
||||||
if (schemaElement == null) {
|
if (schemaElement == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -1,28 +1,28 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.knowledge.EmbeddingResult;
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
|
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.common.util.ComponentFactory;
|
import com.tencent.supersonic.common.util.ComponentFactory;
|
||||||
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
||||||
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
|
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
|
||||||
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
|
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
|
||||||
import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
|
import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
|
||||||
import com.tencent.supersonic.headless.server.listener.MetaEmbeddingListener;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.EmbeddingResult;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.springframework.beans.BeanUtils;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.beans.BeanUtils;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* EmbeddingMatchStrategy uses vector database to perform
|
* EmbeddingMatchStrategy uses vector database to perform
|
||||||
@@ -35,6 +35,9 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy<EmbeddingResult> {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private OptimizationConfig optimizationConfig;
|
private OptimizationConfig optimizationConfig;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private EmbeddingConfig embeddingConfig;
|
||||||
|
|
||||||
private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore();
|
private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -86,7 +89,7 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy<EmbeddingResult> {
|
|||||||
.build();
|
.build();
|
||||||
// step2. retrieveQuery by detectSegment
|
// step2. retrieveQuery by detectSegment
|
||||||
List<RetrieveQueryResult> retrieveQueryResults = s2EmbeddingStore.retrieveQuery(
|
List<RetrieveQueryResult> retrieveQueryResults = s2EmbeddingStore.retrieveQuery(
|
||||||
MetaEmbeddingListener.COLLECTION_NAME, retrieveQuery, embeddingNumber);
|
embeddingConfig.getMetaCollectionName(), retrieveQuery, embeddingNumber);
|
||||||
|
|
||||||
if (CollectionUtils.isEmpty(retrieveQueryResults)) {
|
if (CollectionUtils.isEmpty(retrieveQueryResults)) {
|
||||||
return;
|
return;
|
||||||
@@ -1,21 +1,19 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.service.SemanticService;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A mapper capable of converting the VALUE of entity dimension values into ID types.
|
* A mapper capable of converting the VALUE of entity dimension values into ID types.
|
||||||
*/
|
*/
|
||||||
@@ -30,7 +28,7 @@ public class EntityMapper extends BaseMapper {
|
|||||||
if (CollectionUtils.isEmpty(schemaElementMatchList)) {
|
if (CollectionUtils.isEmpty(schemaElementMatchList)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
SchemaElement entity = getEntity(modelId);
|
SchemaElement entity = getEntity(modelId, queryContext);
|
||||||
if (entity == null || entity.getId() == null) {
|
if (entity == null || entity.getId() == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -66,9 +64,9 @@ public class EntityMapper extends BaseMapper {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private SchemaElement getEntity(Long modelId) {
|
private SchemaElement getEntity(Long modelId, QueryContext queryContext) {
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
|
ModelSchema modelSchema = semanticSchema.getModelSchemaMap().get(modelId);
|
||||||
if (modelSchema != null && modelSchema.getEntity() != null) {
|
if (modelSchema != null && modelSchema.getEntity() != null) {
|
||||||
return modelSchema.getEntity();
|
return modelSchema.getEntity();
|
||||||
}
|
}
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.SearchService;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -69,8 +69,7 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
|
|||||||
// step1. pre search
|
// step1. pre search
|
||||||
Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize();
|
Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize();
|
||||||
LinkedHashSet<HanlpMapResult> hanlpMapResults = SearchService.prefixSearch(detectSegment, oneDetectionMaxSize,
|
LinkedHashSet<HanlpMapResult> hanlpMapResults = SearchService.prefixSearch(detectSegment, oneDetectionMaxSize,
|
||||||
agentId,
|
agentId, detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
|
||||||
// step2. suffix search
|
// step2. suffix search
|
||||||
LinkedHashSet<HanlpMapResult> suffixHanlpMapResults = SearchService.suffixSearch(detectSegment,
|
LinkedHashSet<HanlpMapResult> suffixHanlpMapResults = SearchService.suffixSearch(detectSegment,
|
||||||
oneDetectionMaxSize, agentId, detectModelIds).stream()
|
oneDetectionMaxSize, agentId, detectModelIds).stream()
|
||||||
@@ -1,16 +1,16 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.NatureHelper;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DatabaseMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
|
||||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@@ -35,7 +35,7 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class);
|
HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class);
|
||||||
|
|
||||||
List<HanlpMapResult> hanlpMapResults = hanlpMatchStrategy.getMatches(queryContext, terms);
|
List<HanlpMapResult> hanlpMapResults = hanlpMatchStrategy.getMatches(queryContext, terms);
|
||||||
convertHanlpMapResultToMapInfo(hanlpMapResults, queryContext.getMapInfo(), terms);
|
convertHanlpMapResultToMapInfo(hanlpMapResults, queryContext, terms);
|
||||||
|
|
||||||
//2.database Match
|
//2.database Match
|
||||||
DatabaseMatchStrategy databaseMatchStrategy = ContextUtils.getBean(DatabaseMatchStrategy.class);
|
DatabaseMatchStrategy databaseMatchStrategy = ContextUtils.getBean(DatabaseMatchStrategy.class);
|
||||||
@@ -44,7 +44,7 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
convertDatabaseMapResultToMapInfo(queryContext, databaseResults);
|
convertDatabaseMapResultToMapInfo(queryContext, databaseResults);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void convertHanlpMapResultToMapInfo(List<HanlpMapResult> mapResults, SchemaMapInfo schemaMap,
|
private void convertHanlpMapResultToMapInfo(List<HanlpMapResult> mapResults, QueryContext queryContext,
|
||||||
List<Term> terms) {
|
List<Term> terms) {
|
||||||
if (CollectionUtils.isEmpty(mapResults)) {
|
if (CollectionUtils.isEmpty(mapResults)) {
|
||||||
return;
|
return;
|
||||||
@@ -65,7 +65,8 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
Long elementID = NatureHelper.getElementID(nature);
|
Long elementID = NatureHelper.getElementID(nature);
|
||||||
SchemaElement element = getSchemaElement(modelId, elementType, elementID);
|
SchemaElement element = getSchemaElement(modelId, elementType, elementID,
|
||||||
|
queryContext.getSemanticSchema());
|
||||||
if (element == null) {
|
if (element == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -81,7 +82,7 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
.detectWord(hanlpMapResult.getDetectWord())
|
.detectWord(hanlpMapResult.getDetectWord())
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
addToSchemaMap(schemaMap, modelId, schemaElementMatch);
|
addToSchemaMap(queryContext.getMapInfo(), modelId, schemaElementMatch);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,12 +1,11 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.algorithm.EditDistance;
|
import com.hankcs.hanlp.algorithm.EditDistance;
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.agent.Agent;
|
||||||
import com.tencent.supersonic.chat.service.AgentService;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.chat.core.utils.NatureHelper;
|
||||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -83,15 +82,12 @@ public class MapperHelper {
|
|||||||
detectSegment.length());
|
detectSegment.length());
|
||||||
}
|
}
|
||||||
|
|
||||||
public Set<Long> getModelIds(QueryReq request) {
|
public Set<Long> getModelIds(QueryReq request, Agent agent) {
|
||||||
|
|
||||||
Long modelId = request.getModelId();
|
Long modelId = request.getModelId();
|
||||||
|
Set<Long> detectModelIds = agent.getModelIds(null);
|
||||||
AgentService agentService = ContextUtils.getBean(AgentService.class);
|
|
||||||
|
|
||||||
Set<Long> detectModelIds = agentService.getModelIds(request.getAgentId(), null);
|
|
||||||
//contains all
|
//contains all
|
||||||
if (agentService.containsAllModel(detectModelIds)) {
|
if (agent.containsAllModel(detectModelIds)) {
|
||||||
if (Objects.nonNull(modelId) && modelId > 0) {
|
if (Objects.nonNull(modelId) && modelId > 0) {
|
||||||
Set<Long> result = new HashSet<>();
|
Set<Long> result = new HashSet<>();
|
||||||
result.add(modelId);
|
result.add(modelId);
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
@@ -1,16 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.utils.ModelClusterBuilder;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.ModelClusterBuilder;
|
||||||
import com.tencent.supersonic.common.pojo.ModelCluster;
|
import com.tencent.supersonic.common.pojo.ModelCluster;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -27,8 +23,7 @@ public class ModelClusterMapper implements SchemaMapper {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void map(QueryContext queryContext) {
|
public void map(QueryContext queryContext) {
|
||||||
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
|
|
||||||
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
|
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
|
||||||
List<ModelCluster> modelClusters = buildModelClusterMatched(schemaMapInfo, semanticSchema);
|
List<ModelCluster> modelClusters = buildModelClusterMatched(schemaMapInfo, semanticSchema);
|
||||||
Map<String, List<SchemaElementMatch>> modelClusterElementMatches = new HashMap<>();
|
Map<String, List<SchemaElementMatch>> modelClusterElementMatches = new HashMap<>();
|
||||||
@@ -46,7 +41,7 @@ public class ModelClusterMapper implements SchemaMapper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private List<ModelCluster> buildModelClusterMatched(SchemaMapInfo schemaMapInfo,
|
private List<ModelCluster> buildModelClusterMatched(SchemaMapInfo schemaMapInfo,
|
||||||
SemanticSchema semanticSchema) {
|
SemanticSchema semanticSchema) {
|
||||||
Set<Long> matchedModels = schemaMapInfo.getMatchedModels();
|
Set<Long> matchedModels = schemaMapInfo.getMatchedModels();
|
||||||
List<ModelCluster> modelClusters = ModelClusterBuilder.buildModelClusters(semanticSchema);
|
List<ModelCluster> modelClusters = ModelClusterBuilder.buildModelClusters(semanticSchema);
|
||||||
return modelClusters.stream().map(ModelCluster::getModelIds).peek(modelCluster -> {
|
return modelClusters.stream().map(ModelCluster::getModelIds).peek(modelCluster -> {
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
@@ -1,22 +1,21 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class QueryFilterMapper implements SchemaMapper {
|
public class QueryFilterMapper implements SchemaMapper {
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.tencent.supersonic.chat.api.component;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A schema mapper identifies references to schema elements(metrics/dimensions/entities/values)
|
* A schema mapper identifies references to schema elements(metrics/dimensions/entities/values)
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.SearchService;
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
package com.tencent.supersonic.chat.parser;
|
package com.tencent.supersonic.chat.core.parser;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.function.FunctionPromptGenerator;
|
import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionPromptGenerator;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.function.FunctionReq;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.OutputFormat;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.function.FunctionResp;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.SqlGeneration;
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.OutputFormat;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.SqlGenerationFactory;
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.SqlGeneration;
|
import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq;
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.SqlGenerationFactory;
|
import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.SqlGenerationMode;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import dev.langchain4j.model.chat.ChatLanguageModel;
|
import dev.langchain4j.model.chat.ChatLanguageModel;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package com.tencent.supersonic.chat.core.parser;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
|
import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq;
|
||||||
|
import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp;
|
||||||
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq;
|
||||||
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* LLMProxy encapsulates functions performed by LLMs so that multiple
|
||||||
|
* orchestration frameworks (e.g. LangChain in python, LangChain4j in java)
|
||||||
|
* could be used.
|
||||||
|
*/
|
||||||
|
public interface LLMProxy {
|
||||||
|
|
||||||
|
boolean isSkip(QueryContext queryContext);
|
||||||
|
|
||||||
|
LLMResp query2sql(LLMReq llmReq, String modelClusterKey);
|
||||||
|
|
||||||
|
FunctionResp requestFunction(FunctionReq functionReq);
|
||||||
|
|
||||||
|
}
|
||||||
@@ -1,14 +1,14 @@
|
|||||||
package com.tencent.supersonic.chat.parser;
|
package com.tencent.supersonic.chat.core.parser;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON;
|
import com.alibaba.fastjson.JSON;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.config.LLMParserConfig;
|
import com.tencent.supersonic.chat.core.config.LLMParserConfig;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.function.FunctionCallConfig;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.OutputFormat;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.function.FunctionReq;
|
import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionCallConfig;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.function.FunctionResp;
|
import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionReq;
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.OutputFormat;
|
import com.tencent.supersonic.chat.core.parser.plugin.function.FunctionResp;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMResp;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
@@ -1,30 +1,25 @@
|
|||||||
package com.tencent.supersonic.chat.parser;
|
package com.tencent.supersonic.chat.core.parser;
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery;
|
import com.tencent.supersonic.chat.core.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.service.SemanticService;
|
import com.tencent.supersonic.chat.core.query.SemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery;
|
||||||
|
import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery;
|
||||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.collections4.CollectionUtils;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* QueryTypeParser resolves query type as either METRIC or TAG, or ID.
|
* QueryTypeParser resolves query type as either METRIC or TAG, or ID.
|
||||||
@@ -40,14 +35,14 @@ public class QueryTypeParser implements SemanticParser {
|
|||||||
|
|
||||||
for (SemanticQuery semanticQuery : candidateQueries) {
|
for (SemanticQuery semanticQuery : candidateQueries) {
|
||||||
// 1.init S2SQL
|
// 1.init S2SQL
|
||||||
semanticQuery.initS2Sql(user);
|
semanticQuery.initS2Sql(queryContext.getSemanticSchema(), user);
|
||||||
// 2.set queryType
|
// 2.set queryType
|
||||||
QueryType queryType = getQueryType(semanticQuery);
|
QueryType queryType = getQueryType(queryContext, semanticQuery);
|
||||||
semanticQuery.getParseInfo().setQueryType(queryType);
|
semanticQuery.getParseInfo().setQueryType(queryType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private QueryType getQueryType(SemanticQuery semanticQuery) {
|
private QueryType getQueryType(QueryContext queryContext, SemanticQuery semanticQuery) {
|
||||||
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
|
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
|
||||||
SqlInfo sqlInfo = parseInfo.getSqlInfo();
|
SqlInfo sqlInfo = parseInfo.getSqlInfo();
|
||||||
if (Objects.isNull(sqlInfo) || StringUtils.isBlank(sqlInfo.getS2SQL())) {
|
if (Objects.isNull(sqlInfo) || StringUtils.isBlank(sqlInfo.getS2SQL())) {
|
||||||
@@ -55,13 +50,13 @@ public class QueryTypeParser implements SemanticParser {
|
|||||||
}
|
}
|
||||||
//1. entity queryType
|
//1. entity queryType
|
||||||
Set<Long> modelIds = parseInfo.getModel().getModelIds();
|
Set<Long> modelIds = parseInfo.getModel().getModelIds();
|
||||||
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) {
|
if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) {
|
||||||
//If all the fields in the SELECT statement are of tag type.
|
//If all the fields in the SELECT statement are of tag type.
|
||||||
List<String> whereFields = SqlParserSelectHelper.getWhereFields(sqlInfo.getS2SQL())
|
List<String> whereFields = SqlParserSelectHelper.getWhereFields(sqlInfo.getS2SQL())
|
||||||
.stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
|
.stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
|
||||||
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
|
|
||||||
if (CollectionUtils.isNotEmpty(whereFields)) {
|
if (CollectionUtils.isNotEmpty(whereFields)) {
|
||||||
Set<String> ids = semanticSchema.getEntities(modelIds).stream().map(SchemaElement::getName)
|
Set<String> ids = semanticSchema.getEntities(modelIds).stream().map(SchemaElement::getName)
|
||||||
.collect(Collectors.toSet());
|
.collect(Collectors.toSet());
|
||||||
@@ -77,7 +72,6 @@ public class QueryTypeParser implements SemanticParser {
|
|||||||
}
|
}
|
||||||
//2. metric queryType
|
//2. metric queryType
|
||||||
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getS2SQL());
|
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getS2SQL());
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
|
||||||
List<SchemaElement> metrics = semanticSchema.getMetrics(modelIds);
|
List<SchemaElement> metrics = semanticSchema.getMetrics(modelIds);
|
||||||
if (CollectionUtils.isNotEmpty(metrics)) {
|
if (CollectionUtils.isNotEmpty(metrics)) {
|
||||||
Set<String> metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet());
|
Set<String> metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet());
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
package com.tencent.supersonic.chat.parser;
|
package com.tencent.supersonic.chat.core.parser;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
import com.tencent.supersonic.chat.core.query.SemanticQuery;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.tencent.supersonic.chat.api.component;
|
package com.tencent.supersonic.chat.core.parser;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.ChatContext;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A semantic parser understands user queries and extracts semantic information.
|
* A semantic parser understands user queries and extracts semantic information.
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin;
|
package com.tencent.supersonic.chat.core.parser.plugin;
|
||||||
|
|
||||||
public enum ParseMode {
|
public enum ParseMode {
|
||||||
|
|
||||||
@@ -1,22 +1,22 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin;
|
package com.tencent.supersonic.chat.core.parser.plugin;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.google.common.collect.Sets;
|
import com.google.common.collect.Sets;
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
import com.tencent.supersonic.chat.core.parser.SemanticParser;
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
import com.tencent.supersonic.chat.core.query.SemanticQuery;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.core.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||||
import com.tencent.supersonic.chat.plugin.Plugin;
|
import com.tencent.supersonic.chat.core.plugin.Plugin;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
import com.tencent.supersonic.chat.core.plugin.PluginManager;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginParseResult;
|
import com.tencent.supersonic.chat.core.plugin.PluginParseResult;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
|
import com.tencent.supersonic.chat.core.plugin.PluginRecallResult;
|
||||||
import com.tencent.supersonic.chat.query.QueryManager;
|
import com.tencent.supersonic.chat.core.query.plugin.PluginSemanticQuery;
|
||||||
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
|
import com.tencent.supersonic.chat.core.query.QueryManager;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.common.pojo.ModelCluster;
|
import com.tencent.supersonic.common.pojo.ModelCluster;
|
||||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||||
@@ -75,7 +75,7 @@ public abstract class PluginParser implements SemanticParser {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected List<Plugin> getPluginList(QueryContext queryContext) {
|
protected List<Plugin> getPluginList(QueryContext queryContext) {
|
||||||
return PluginManager.getPluginAgentCanSupport(queryContext.getRequest().getAgentId());
|
return PluginManager.getPluginAgentCanSupport(queryContext);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected SemanticParseInfo buildSemanticParseInfo(Long modelId, Plugin plugin, QueryReq queryReq,
|
protected SemanticParseInfo buildSemanticParseInfo(Long modelId, Plugin plugin, QueryReq queryReq,
|
||||||
@@ -1,14 +1,14 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.embedding;
|
package com.tencent.supersonic.chat.core.parser.plugin.embedding;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.parser.PythonLLMProxy;
|
import com.tencent.supersonic.chat.core.plugin.Plugin;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.ParseMode;
|
import com.tencent.supersonic.chat.core.plugin.PluginManager;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.PluginParser;
|
import com.tencent.supersonic.chat.core.plugin.PluginRecallResult;
|
||||||
import com.tencent.supersonic.chat.plugin.Plugin;
|
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
import com.tencent.supersonic.chat.core.parser.PythonLLMProxy;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
|
import com.tencent.supersonic.chat.core.parser.plugin.ParseMode;
|
||||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
import com.tencent.supersonic.chat.core.parser.plugin.PluginParser;
|
||||||
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.embedding;
|
package com.tencent.supersonic.chat.core.parser.plugin.embedding;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.embedding;
|
package com.tencent.supersonic.chat.core.parser.plugin.embedding;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.function;
|
package com.tencent.supersonic.chat.core.parser.plugin.function;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -1,29 +1,26 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.function;
|
package com.tencent.supersonic.chat.core.parser.plugin.function;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.parser.PythonLLMProxy;
|
||||||
import com.tencent.supersonic.chat.parser.PythonLLMProxy;
|
import com.tencent.supersonic.chat.core.parser.plugin.ParseMode;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.ParseMode;
|
import com.tencent.supersonic.chat.core.parser.plugin.PluginParser;
|
||||||
import com.tencent.supersonic.chat.parser.plugin.PluginParser;
|
import com.tencent.supersonic.chat.core.plugin.Plugin;
|
||||||
import com.tencent.supersonic.chat.plugin.Plugin;
|
import com.tencent.supersonic.chat.core.plugin.PluginManager;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
import com.tencent.supersonic.chat.core.plugin.PluginParseConfig;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
|
import com.tencent.supersonic.chat.core.plugin.PluginRecallResult;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery;
|
||||||
import com.tencent.supersonic.chat.service.PluginService;
|
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
|
||||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.commons.lang3.tuple.Pair;
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* FunctionCallParser is an implementation of a recall plugin based on FunctionCall
|
* FunctionCallParser is an implementation of a recall plugin based on FunctionCall
|
||||||
*/
|
*/
|
||||||
@@ -45,19 +42,17 @@ public class FunctionCallParser extends PluginParser {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PluginRecallResult recallPlugin(QueryContext queryContext) {
|
public PluginRecallResult recallPlugin(QueryContext queryContext) {
|
||||||
PluginService pluginService = ContextUtils.getBean(PluginService.class);
|
|
||||||
FunctionResp functionResp = functionCall(queryContext);
|
FunctionResp functionResp = functionCall(queryContext);
|
||||||
if (skipFunction(functionResp)) {
|
if (skipFunction(functionResp)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
log.info("requestFunction result:{}", functionResp.getToolSelection());
|
log.info("requestFunction result:{}", functionResp.getToolSelection());
|
||||||
String toolSelection = functionResp.getToolSelection();
|
String toolSelection = functionResp.getToolSelection();
|
||||||
Optional<Plugin> pluginOptional = pluginService.getPluginByName(toolSelection);
|
Plugin plugin = queryContext.getNameToPlugin().get(toolSelection);
|
||||||
if (!pluginOptional.isPresent()) {
|
if (Objects.isNull(plugin)) {
|
||||||
log.info("pluginOptional is not exist:{}, skip the parse", toolSelection);
|
log.info("pluginOptional is not exist:{}, skip the parse", toolSelection);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
Plugin plugin = pluginOptional.get();
|
|
||||||
plugin.setParseMode(ParseMode.FUNCTION_CALL);
|
plugin.setParseMode(ParseMode.FUNCTION_CALL);
|
||||||
Pair<Boolean, Set<Long>> pluginResolveResult = PluginManager.resolve(plugin, queryContext);
|
Pair<Boolean, Set<Long>> pluginResolveResult = PluginManager.resolve(plugin, queryContext);
|
||||||
if (pluginResolveResult.getLeft()) {
|
if (pluginResolveResult.getLeft()) {
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.function;
|
package com.tencent.supersonic.chat.core.parser.plugin.function;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.function;
|
package com.tencent.supersonic.chat.core.parser.plugin.function;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.InputFormat;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.InputFormat;
|
||||||
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
|
import com.tencent.supersonic.chat.core.plugin.PluginParseConfig;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.function;
|
package com.tencent.supersonic.chat.core.parser.plugin.function;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.plugin.PluginParseConfig;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
|
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.function;
|
package com.tencent.supersonic.chat.core.parser.plugin.function;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.parser.plugin.function;
|
package com.tencent.supersonic.chat.core.parser.plugin.function;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.tencent.supersonic.chat.parser.sql.llm;
|
package com.tencent.supersonic.chat.core.parser.sql.llm;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
import com.tencent.supersonic.chat.core.query.SemanticQuery;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.core.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo;
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user