topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project] supersonic 0.7.0 version backend update (#20)

Browse Source 
Co-authored-by: kanedai <kanedai@tencent.com>
This commit is contained in:
daikon
2023-07-31 11:09:58 +08:00
committed by GitHub
parent 078a81038f
commit e2b2d31429
675 changed files with 13089 additions and 13536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SchemaMapper.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.api.component;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
/**
* This interface defines the contract for a schema mapper that identifies references to schema
@@ -11,5 +11,5 @@ import com.tencent.supersonic.chat.api.request.QueryContextReq;
*/
public interface SchemaMapper {
void map(QueryContextReq queryContext);
void map(QueryContext queryContext);
}

38
chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java
View File

@@ -2,10 +2,15 @@ package com.tencent.supersonic.chat.api.component;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.core.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.core.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.core.response.*;
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.List;
@@ -25,27 +30,16 @@ import java.util.List;
public interface SemanticLayer {
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user);
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user);
QueryResultWithSchemaResp queryBySql(QuerySqlReq querySqlReq, User user);
DomainSchemaResp getDomainSchemaInfo(Long domain, Boolean cacheEnable);
List<DomainSchemaResp> getDomainSchemaInfo(List<Long> ids);
List<DomainSchema> getDomainSchema();
List<DomainSchema> getDomainSchema(List<Long> ids);
DomainSchema getDomainSchema(Long domain, Boolean cacheEnable);
PageInfo<DimensionResp> getDimensionPage(PageDimensionReq pageDimensionCmd);
PageInfo<MetricResp> getMetricPage(PageMetricReq pageMetricCmd);
List<DomainResp> getDomainListForViewer();
List<DomainResp> getDomainListForAdmin();
PageInfo<DimensionResp> queryDimensionPage(PageDimensionReq pageDimensionCmd);
PageInfo<MetricResp> queryMetricPage(PageMetricReq pageMetricCmd);
// PageInfo<MetricResp> queryMetricPage(PageMetricReq pageMetricCmd);
//
// PageInfo<DimensionResp> queryDimensionPage(PageDimensionReq pageDimensionCmd);
//
// List<DomainResp> getDomainListForAdmin();
//
// List<DomainResp> getDomainListForViewer();
}

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticParser.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.api.component;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
/**
* This interface defines the contract for a semantic parser that can analyze natural language query
@@ -13,5 +13,5 @@ import com.tencent.supersonic.chat.api.request.QueryContextReq;
*/
public interface SemanticParser {
void parse(QueryContextReq queryContext, ChatContext chatContext);
void parse(QueryContext queryContext, ChatContext chatContext);
}

5
chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java
View File

@@ -2,7 +2,8 @@ package com.tencent.supersonic.chat.api.component;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import org.apache.calcite.sql.parser.SqlParseException;
/**
* This class defines the contract for a semantic query that executes specific type of
@@ -12,7 +13,7 @@ public interface SemanticQuery {
String getQueryMode();
QueryResultResp execute(User user);
QueryResult execute(User user) throws SqlParseException;
SemanticParseInfo getParseInfo();
}

45
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DomainSchema.java Normal file
View File

@@ -0,0 +1,45 @@
package com.tencent.supersonic.chat.api.pojo;
import lombok.Data;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
@Data
public class DomainSchema {
private SchemaElement domain;
private Set<SchemaElement> metrics = new HashSet<>();
private Set<SchemaElement> dimensions = new HashSet<>();
private Set<SchemaElement> dimensionValues = new HashSet<>();
private Set<SchemaElement> entities = new HashSet<>();
public SchemaElement getElement(SchemaElementType elementType, long elementID) {
Optional<SchemaElement> element = Optional.empty();
switch (elementType) {
case DOMAIN:
element = Optional.of(domain);
break;
case METRIC:
element = metrics.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case DIMENSION:
element = dimensions.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case ENTITY:
element = entities.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case VALUE:
element = dimensionValues.stream().filter(e -> e.getId() == elementID).findFirst();
default:
}
if (element.isPresent()) {
return element.get();
} else {
return null;
}
}
}

20
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java Normal file
View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.chat.api.pojo;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class QueryContext {
private QueryRequest request;
private List<SemanticQuery> candidateQueries = new ArrayList<>();
private SchemaMapInfo mapInfo = new SchemaMapInfo();
public QueryContext(QueryRequest request) {
this.request = request;
}
}

12
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryMatchInfo.java
View File

@@ -1,12 +0,0 @@
package com.tencent.supersonic.chat.api.pojo;
import lombok.Data;
@Data
public class QueryMatchInfo {
SchemaElementType elementType;
String detectWord;
private Integer count = 0;
private double maxSimilarity;
}

56
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java Normal file
View File

@@ -0,0 +1,56 @@
package com.tencent.supersonic.chat.api.pojo;
import com.google.common.base.Objects;
import java.io.Serializable;
import java.util.List;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class SchemaElement implements Serializable {
private Long domain;
private Long id;
private String name;
private String bizName;
private Long useCnt;
private SchemaElementType type;
private List<String> alias;
public SchemaElement() {
}
public SchemaElement(Long domain, Long id, String name, String bizName,
Long useCnt, SchemaElementType type, List<String> alias) {
this.domain = domain;
this.id = id;
this.name = name;
this.bizName = bizName;
this.useCnt = useCnt;
this.type = type;
this.alias = alias;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
SchemaElement schemaElement = (SchemaElement) o;
return Objects.equal(domain, schemaElement.domain) && Objects.equal(id,
schemaElement.id) && Objects.equal(name, schemaElement.name)
&& Objects.equal(bizName, schemaElement.bizName) && Objects.equal(
useCnt, schemaElement.useCnt) && Objects.equal(type, schemaElement.type);
}
@Override
public int hashCode() {
return Objects.hashCode(domain, id, name, bizName, useCnt, type);
}
}

8
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java
View File

@@ -13,15 +13,9 @@ import lombok.ToString;
@NoArgsConstructor
public class SchemaElementMatch {
SchemaElementType elementType;
int elementID;
SchemaElement element;
double similarity;
String detectWord;
String word;
Long frequency;
}

15
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaMapInfo.java
View File

@@ -7,26 +7,21 @@ import java.util.Set;
public class SchemaMapInfo {
private Map<Integer, List<SchemaElementMatch>> domainElementMatches = new HashMap<>();
private Map<Long, List<SchemaElementMatch>> domainElementMatches = new HashMap<>();
public Set<Integer> getMatchedDomains() {
public Set<Long> getMatchedDomains() {
return domainElementMatches.keySet();
}
public List<SchemaElementMatch> getMatchedElements(Integer domain) {
public List<SchemaElementMatch> getMatchedElements(Long domain) {
return domainElementMatches.get(domain);
}
public Map<Integer, List<SchemaElementMatch>> getDomainElementMatches() {
public Map<Long, List<SchemaElementMatch>> getDomainElementMatches() {
return domainElementMatches;
}
public void setDomainElementMatches(
Map<Integer, List<SchemaElementMatch>> domainElementMatches) {
this.domainElementMatches = domainElementMatches;
}
public void setMatchedElements(Integer domain, List<SchemaElementMatch> elementMatches) {
public void setMatchedElements(Long domain, List<SchemaElementMatch> elementMatches) {
domainElementMatches.put(domain, elementMatches);
}
}

43
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java
View File

@@ -1,34 +1,55 @@
package com.tencent.supersonic.chat.api.pojo;
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.Data;
@Data
public class SemanticParseInfo {
String queryMode;
AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
Long domainId = 0L;
String domainName;
SchemaElement domain;
Set<SchemaElement> metrics = new LinkedHashSet();
Set<SchemaElement> dimensions = new LinkedHashSet();
Long entity = 0L;
Set<SchemaItem> metrics = new LinkedHashSet();
Set<SchemaItem> dimensions = new LinkedHashSet();
Set<Filter> dimensionFilters = new LinkedHashSet();
Set<Filter> metricFilters = new LinkedHashSet();
AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
Set<QueryFilter> dimensionFilters = new LinkedHashSet();
Set<QueryFilter> metricFilters = new LinkedHashSet();
private Set<Order> orders = new LinkedHashSet();
private DateConf dateInfo;
private Long limit;
private Boolean nativeQuery = false;
private Double bonus = 0d;
private List<SchemaElementMatch> elementMatches = new ArrayList<>();
private Object info;
private Map<String, Object> properties;
public Long getDomainId() {
return domain != null ? domain.getId() : 0L;
}
public String getDomainName() {
return domain != null ? domain.getName() : "null";
}
public Set<SchemaElement> getMetrics() {
this.metrics = this.metrics.stream().sorted((o1, o2) -> {
int len1 = o1.getName().length();
int len2 = o2.getName().length();
if (len1 != len2) {
return len1 - len2;
} else {
return o1.getName().compareTo(o2.getName());
}
}).collect(Collectors.toCollection(LinkedHashSet::new));
return this.metrics;
}
}

54
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java Normal file
View File

@@ -0,0 +1,54 @@
package com.tencent.supersonic.chat.api.pojo;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class SemanticSchema implements Serializable {
private List<DomainSchema> domainSchemaList;
public SemanticSchema(List<DomainSchema> domainSchemaList) {
this.domainSchemaList = domainSchemaList;
}
public void add(DomainSchema schema) {
domainSchemaList.add(schema);
}
public Map<Long, String> getDomainIdToName() {
return domainSchemaList.stream()
.collect(Collectors.toMap(a -> a.getDomain().getId(), a -> a.getDomain().getName(), (k1, k2) -> k1));
}
public List<SchemaElement> getDimensionValues() {
List<SchemaElement> dimensionValues = new ArrayList<>();
domainSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues()));
return dimensionValues;
}
public List<SchemaElement> getDimensions() {
List<SchemaElement> dimensions = new ArrayList<>();
domainSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions()));
return dimensions;
}
public List<SchemaElement> getMetrics() {
List<SchemaElement> metrics = new ArrayList<>();
domainSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics()));
return metrics;
}
public List<SchemaElement> getDomains() {
List<SchemaElement> domains = new ArrayList<>();
domainSchemaList.stream().forEach(d -> domains.add(d.getDomain()));
return domains;
}
public List<SchemaElement> getEntities() {
List<SchemaElement> entities = new ArrayList<>();
domainSchemaList.stream().forEach(d -> entities.addAll(d.getEntities()));
return entities;
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/PageQueryInfoReq.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PageQueryInfoReq.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.chat;
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;

22
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java Normal file
View File

@@ -0,0 +1,22 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
@Data
public class PluginQueryReq {
private String showElementId;
//DASHBOARD WIDGET
private String showType;
private String type;
private String domain;
private String pattern;
}

24
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataRequest.java Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.chat.api.pojo.request;
import java.util.HashSet;
import java.util.Set;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import lombok.Data;
@Data
public class QueryDataRequest {
String queryMode;
SchemaElement domain;
Set<SchemaElement> metrics = new HashSet<>();
Set<SchemaElement> dimensions = new HashSet<>();
Set<QueryFilter> dimensionFilters = new HashSet<>();
Set<QueryFilter> metricFilters = new HashSet<>();
private Set<Order> orders = new HashSet<>();
private DateConf dateInfo;
private Long limit;
private Boolean nativeQuery = false;
}

6
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/Filter.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilter.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.api.pojo;
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import java.util.Objects;
@@ -7,7 +7,7 @@ import lombok.ToString;
@Data
@ToString(callSuper = true)
public class Filter {
public class QueryFilter {
private String bizName;
@@ -27,7 +27,7 @@ public class Filter {
if (o == null || getClass() != o.getClass()) {
return false;
}
Filter filter = (Filter) o;
QueryFilter filter = (QueryFilter) o;
return Objects.equals(bizName, filter.bizName) && Objects.equals(name, filter.name)
&& operator == filter.operator && Objects.equals(value, filter.value) && Objects.equals(
elementID, filter.elementID);

9
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryFilter.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryFilters.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.api.pojo;
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.ArrayList;
@@ -7,10 +7,7 @@ import java.util.List;
import java.util.Map;
@Data
public class QueryFilter {
private List<Filter> filters = new ArrayList<>();
public class QueryFilters {
private List<QueryFilter> filters = new ArrayList<>();
private Map<String, Object> params = new HashMap<>();
}

15
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryRequest.java Normal file
View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import lombok.Data;
@Data
public class QueryRequest {
private String queryText;
private Integer chatId;
private Long domainId = 0L;
private User user;
private QueryFilters queryFilters;
private boolean saveAnswer = true;
}

12
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestion.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class RecommendedQuestion {
private String question;
}

10
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/AggregateInfo.java Normal file
View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.chat.api.pojo.response;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class AggregateInfo {
private List<MetricInfo> metricInfos = new ArrayList<>();
}

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataInfo.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DataInfo.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.api.pojo;
package com.tencent.supersonic.chat.api.pojo.response;
import lombok.Data;

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DomainInfo.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DomainInfo.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.api.pojo;
package com.tencent.supersonic.chat.api.pojo.response;
import java.io.Serializable;
import java.util.List;

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/EntityInfo.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityInfo.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.api.pojo;
package com.tencent.supersonic.chat.api.pojo.response;
import java.util.ArrayList;
import java.util.List;

14
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/MetricInfo.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.chat.api.pojo.response;
import java.util.Map;
import lombok.Data;
@Data
public class MetricInfo {
private String name;
private String value;
private String date;
private Map<String, String> statistics;
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/ChatQueryVO.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResponse.java
View File

@@ -1,11 +1,10 @@
package com.tencent.supersonic.chat.domain.pojo.chat;
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import java.util.Date;
import lombok.Data;
@Data
public class ChatQueryVO {
public class QueryResponse {
private Long questionId;
private Date createTime;
@@ -13,5 +12,5 @@ public class ChatQueryVO {
private Integer score;
private String feedback;
private String queryText;
private QueryResultResp queryResponse;
private QueryResult queryResult;
}

12
chat/api/src/main/java/com/tencent/supersonic/chat/api/response/QueryResultResp.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java
View File

@@ -1,21 +1,21 @@
package com.tencent.supersonic.chat.api.response;
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.semantic.api.core.pojo.QueryAuthorization;
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class QueryResultResp {
public class QueryResult {
public EntityInfo entityInfo;
public AggregateInfo aggregateInfo;
private Long queryId;
private String queryMode;
private String querySql;
private int queryState;
private QueryState queryState = QueryState.EMPTY;
private List<QueryColumn> queryColumns;
private QueryAuthorization queryAuthorization;
private SemanticParseInfo chatContext;

8
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryState.java Normal file
View File

@@ -0,0 +1,8 @@
package com.tencent.supersonic.chat.api.pojo.response;
public enum QueryState {
SUCCESS,
SEARCH_EXCEPTION,
EMPTY,
INVALID;
}

14
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendQuestion.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
@Data
@AllArgsConstructor
public class RecommendQuestion {
private Long domainId;
private List<RecommendedQuestion> recommendedQuestions;
}

12
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResponse.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import lombok.Data;
import java.util.List;
@Data
public class RecommendResponse {
private List<SchemaElement> dimensions;
private List<SchemaElement> metrics;
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/search/SearchResponse.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResponse.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.search;
package com.tencent.supersonic.chat.api.pojo.response;
import java.util.List;
import lombok.Data;
@@ -6,8 +6,6 @@ import lombok.Getter;
import lombok.Setter;
@Data
@Setter
@Getter
public class SearchResponse {
private List<SearchResult> searchResults;

23
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/search/SearchResult.java → chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java
View File

@@ -1,7 +1,8 @@
package com.tencent.supersonic.chat.domain.pojo.search;
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import java.util.Objects;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
@@ -9,6 +10,7 @@ import lombok.Setter;
@Data
@Setter
@Getter
@Builder
public class SearchResult {
private String recommend;
@@ -17,29 +19,12 @@ public class SearchResult {
private String domainName;
private Integer domainId;
private Long domainId;
private SchemaElementType schemaElementType;
private boolean isComplete = true;
public SearchResult(String recommend, String subRecommend, String className, Integer domainId,
SchemaElementType schemaElementType) {
this.recommend = recommend;
this.subRecommend = subRecommend;
this.domainName = className;
this.domainId = domainId;
this.schemaElementType = schemaElementType;
}
public SearchResult(String recommend, String subRecommend, String className, Integer domainId, boolean isComplete) {
this.recommend = recommend;
this.subRecommend = subRecommend;
this.domainName = className;
this.domainId = domainId;
this.isComplete = isComplete;
}
@Override
public boolean equals(Object o) {
if (this == o) {

23
chat/api/src/main/java/com/tencent/supersonic/chat/api/request/QueryContextReq.java
View File

@@ -1,23 +0,0 @@
package com.tencent.supersonic.chat.api.request;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class QueryContextReq {
private String queryText;
private Integer chatId;
private Integer domainId = 0;
private User user;
private QueryFilter queryFilter;
private List<SemanticQuery> candidateQueries = new ArrayList<>();
private SchemaMapInfo mapInfo = new SchemaMapInfo();
private boolean saveAnswer = true;
}

12
chat/core/pom.xml
View File

@@ -113,12 +113,6 @@
<artifactId>semantic-api</artifactId>
<version>${project.version}</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.tencent.supersonic</groupId>-->
<!-- <artifactId>semantic-query</artifactId>-->
<!-- <version>${project.version}</version>-->
<!-- <scope>compile</scope>-->
<!-- </dependency>-->
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>semantic-query</artifactId>
@@ -137,6 +131,12 @@
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.xkzhangsan</groupId>
<artifactId>xk-time</artifactId>
<version>${xk.time.version}</version>
</dependency>
</dependencies>
</project>

213
chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java
View File

@@ -1,213 +0,0 @@
package com.tencent.supersonic.chat.application;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.pojo.DataInfo;
import com.tencent.supersonic.chat.api.pojo.DomainInfo;
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.Filter;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
import com.tencent.supersonic.chat.domain.pojo.config.ChatDefaultRichConfig;
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
@Slf4j
public class DomainEntityService {
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
@Autowired
private ConfigServiceImpl configService;
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
if (parseInfo != null && parseInfo.getDomainId() > 0) {
EntityInfo entityInfo = getEntityInfo(parseInfo.getDomainId());
if (parseInfo.getDimensionFilters().size() <= 0) {
entityInfo.setMetrics(null);
entityInfo.setDimensions(null);
return entityInfo;
}
if (entityInfo.getDomainInfo() != null && entityInfo.getDomainInfo().getPrimaryEntityBizName() != null) {
String domainInfoPrimaryName = entityInfo.getDomainInfo().getPrimaryEntityBizName();
String domainInfoId = "";
for (Filter chatFilter : parseInfo.getDimensionFilters()) {
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
.equals(domainInfoPrimaryName)) {
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
domainInfoId = chatFilter.getValue().toString();
}
if (chatFilter.getOperator().equals(FilterOperatorEnum.IN)) {
domainInfoId = ((List<String>) chatFilter.getValue()).get(0);
}
}
}
if (!"".equals(domainInfoId)) {
try {
setMainDomain(entityInfo, parseInfo.getDomainId(),
domainInfoId, user);
return entityInfo;
} catch (Exception e) {
log.error("setMaintDomain error {}", e);
}
}
}
}
return null;
}
public EntityInfo getEntityInfo(Long domain) {
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(domain);
if (Objects.isNull(chaConfigRichDesc) || Objects.isNull(chaConfigRichDesc.getChatDetailRichConfig())) {
return new EntityInfo();
}
return getEntityInfo(chaConfigRichDesc);
}
private EntityInfo getEntityInfo(ChatConfigRichResp chaConfigRichDesc) {
EntityInfo entityInfo = new EntityInfo();
EntityRichInfo entityDesc = chaConfigRichDesc.getChatDetailRichConfig().getEntity();
if (entityDesc != null && Objects.nonNull(chaConfigRichDesc.getDomainId())) {
DomainInfo domainInfo = new DomainInfo();
domainInfo.setItemId(Integer.valueOf(chaConfigRichDesc.getDomainId().intValue()));
domainInfo.setName(chaConfigRichDesc.getDomainName());
domainInfo.setWords(entityDesc.getNames());
domainInfo.setBizName(chaConfigRichDesc.getBizName());
if (Objects.nonNull(entityDesc.getDimItem())) {
domainInfo.setPrimaryEntityBizName(entityDesc.getDimItem().getBizName());
}
entityInfo.setDomainInfo(domainInfo);
List<DataInfo> dimensions = new ArrayList<>();
List<DataInfo> metrics = new ArrayList<>();
if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig())
&& Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig())) {
ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
if(!CollectionUtils.isEmpty(chatDefaultConfig.getDimensions())){
for (SchemaItem dimensionDesc : chatDefaultConfig.getDimensions()) {
DataInfo mainEntityDimension = new DataInfo();
mainEntityDimension.setItemId(dimensionDesc.getId().intValue());
mainEntityDimension.setName(dimensionDesc.getName());
mainEntityDimension.setBizName(dimensionDesc.getBizName());
dimensions.add(mainEntityDimension);
}
entityInfo.setDimensions(dimensions);
}
if(!CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())){
for (SchemaItem metricDesc : chatDefaultConfig.getMetrics()) {
DataInfo dataInfo = new DataInfo();
dataInfo.setName(metricDesc.getName());
dataInfo.setBizName(metricDesc.getBizName());
dataInfo.setItemId(metricDesc.getId().intValue());
metrics.add(dataInfo);
}
entityInfo.setMetrics(metrics);
}
}
}
return entityInfo;
}
public void setMainDomain(EntityInfo domainInfo, Long domain, String entity, User user) {
domainInfo.setEntityId(entity);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setDomainId(Long.valueOf(domain));
semanticParseInfo.setNativeQuery(true);
semanticParseInfo.setMetrics(getMetrics(domainInfo));
semanticParseInfo.setDimensions(getDimensions(domainInfo));
DateConf dateInfo = new DateConf();
dateInfo.setUnit(1);
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
semanticParseInfo.setDateInfo(dateInfo);
// add filter
Filter chatFilter = new Filter();
chatFilter.setValue(String.valueOf(entity));
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
chatFilter.setBizName(getEntityPrimaryName(domainInfo));
Set<Filter> chatFilters = new LinkedHashSet();
chatFilters.add(chatFilter);
semanticParseInfo.setDimensionFilters(chatFilters);
QueryResultWithSchemaResp queryResultWithColumns = null;
try {
queryResultWithColumns = semanticLayer.queryByStruct(SchemaInfoConverter.convertTo(semanticParseInfo),
user);
} catch (Exception e) {
log.warn("setMainDomain queryByStruct error, e:", e);
}
if (queryResultWithColumns != null) {
if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList())
&& queryResultWithColumns.getResultList().size() > 0) {
Map<String, Object> result = queryResultWithColumns.getResultList().get(0);
for (Map.Entry<String, Object> entry : result.entrySet()) {
String entryKey = getEntryKey(entry);
if (entry.getValue() == null || entryKey == null) {
continue;
}
domainInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
domainInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
}
}
}
}
private Set<SchemaItem> getDimensions(EntityInfo domainInfo) {
Set<SchemaItem> dimensions = new LinkedHashSet();
for (DataInfo mainEntityDimension : domainInfo.getDimensions()) {
SchemaItem dimension = new SchemaItem();
dimension.setBizName(mainEntityDimension.getBizName());
dimensions.add(dimension);
}
return dimensions;
}
private String getEntryKey(Map.Entry<String, Object> entry) {
// metric parser special handle, TODO delete
String entryKey = entry.getKey();
if (entryKey.contains("__")) {
entryKey = entryKey.split("__")[1];
}
return entryKey;
}
private Set<SchemaItem> getMetrics(EntityInfo domainInfo) {
Set<SchemaItem> metrics = new LinkedHashSet();
for (DataInfo metricValue : domainInfo.getMetrics()) {
SchemaItem metric = new SchemaItem();
metric.setBizName(metricValue.getBizName());
metrics.add(metric);
}
return metrics;
}
private String getEntityPrimaryName(EntityInfo domainInfo) {
return domainInfo.getDomainInfo().getPrimaryEntityBizName();
}
}

93
chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java
View File

@@ -1,93 +0,0 @@
package com.tencent.supersonic.chat.application;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.*;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import com.tencent.supersonic.chat.application.query.QuerySelector;
import com.tencent.supersonic.chat.domain.pojo.chat.QueryData;
import com.tencent.supersonic.chat.domain.pojo.search.QueryState;
import com.tencent.supersonic.chat.domain.service.QueryService;
import com.tencent.supersonic.chat.domain.service.ChatService;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.util.json.JsonUtil;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
@Service
@Component("chatQueryService")
@Primary
@Slf4j
public class QueryServiceImpl implements QueryService {
@Autowired
private ChatService chatService;
private List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
private List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
private QuerySelector querySelector = ComponentFactory.getQuerySelector();
@Override
public QueryResultResp executeQuery(QueryContextReq queryCtx) throws Exception {
schemaMappers.stream().forEach(s -> s.map(queryCtx));
// in order to support multi-turn conversation, we need to consider chat context
ChatContext chatCtx = chatService.getOrCreateContext(queryCtx.getChatId());
for (SemanticParser semanticParser : semanticParsers) {
log.info("semanticParser processing:[{}]", semanticParser.getClass().getName());
semanticParser.parse(queryCtx, chatCtx);
}
if (queryCtx.getCandidateQueries().size() > 0) {
log.info("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
Collectors.toList()));
SemanticQuery semanticQuery = querySelector.select(queryCtx.getCandidateQueries());
log.info("pick after [{}]", semanticQuery);
QueryResultResp queryResponse = semanticQuery.execute(queryCtx.getUser());
if (queryResponse != null) {
// update chat context after a successful semantic query
if (queryCtx.isSaveAnswer() && queryResponse.getQueryState() == QueryState.NORMAL.getState()) {
chatService.updateContext(chatCtx, queryCtx, semanticQuery.getParseInfo());
}
queryResponse.setChatContext(chatCtx.getParseInfo());
chatService.addQuery(queryResponse, queryCtx, chatCtx);
return queryResponse;
}
}
return null;
}
@Override
public SemanticParseInfo queryContext(QueryContextReq queryCtx) {
ChatContext context = chatService.getOrCreateContext(queryCtx.getChatId());
return context.getParseInfo();
}
@Override
public QueryResultResp executeDirectQuery(QueryData queryData, User user) throws Exception {
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
QueryResultResp queryResponse = new QueryResultResp();
BeanUtils.copyProperties(queryData, semanticParseInfo);
QueryResultWithSchemaResp resultWithColumns = semanticLayer.queryByStruct(
SchemaInfoConverter.convertTo(semanticParseInfo), user);
queryResponse.setQueryColumns(resultWithColumns.getColumns());
queryResponse.setQueryResults(resultWithColumns.getResultList());
return queryResponse;
}
}

55
chat/core/src/main/java/com/tencent/supersonic/chat/application/RecommendServiceImpl.java
View File

@@ -1,55 +0,0 @@
package com.tencent.supersonic.chat.application;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.chat.domain.pojo.chat.RecommendResponse;
import com.tencent.supersonic.chat.domain.service.RecommendService;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
/***
* Recommend Service impl
*/
@Service
public class RecommendServiceImpl implements RecommendService {
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
@Override
public RecommendResponse recommend(QueryContextReq queryCtx) {
Integer domainId = queryCtx.getDomainId();
if (Objects.isNull(domainId)) {
return new RecommendResponse();
}
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(
Long.valueOf(domainId), true);
List<RecommendResponse.Item> dimensions = domainSchemaDesc.getDimensions().stream().map(dimSchemaDesc -> {
RecommendResponse.Item item = new RecommendResponse.Item();
item.setDomain(domainId);
item.setName(dimSchemaDesc.getName());
item.setBizName(dimSchemaDesc.getBizName());
return item;
}).collect(Collectors.toList());
List<RecommendResponse.Item> metrics = domainSchemaDesc.getMetrics().stream().map(metricSchemaDesc -> {
RecommendResponse.Item item = new RecommendResponse.Item();
item.setDomain(domainId);
item.setName(metricSchemaDesc.getName());
item.setBizName(metricSchemaDesc.getBizName());
return item;
}).collect(Collectors.toList());
RecommendResponse response = new RecommendResponse();
response.setDimensions(dimensions);
response.setMetrics(metrics);
return response;
}
}

69
chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/ApplicationStartedInit.java
View File

@@ -1,69 +0,0 @@
package com.tencent.supersonic.chat.application.knowledge;
import com.tencent.supersonic.common.nlp.WordNature;
import com.tencent.supersonic.knowledge.domain.service.OnlineKnowledgeService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationStartedEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
@Slf4j
@Component
public class ApplicationStartedInit implements ApplicationListener<ApplicationStartedEvent> {
@Autowired
private OnlineKnowledgeService onlineKnowledgeService;
@Autowired
private WordNatureService wordNatureService;
@Override
public void onApplicationEvent(ApplicationStartedEvent event) {
try {
log.info("ApplicationStartedInit start");
List<WordNature> wordNatures = wordNatureService.getAllWordNature();
wordNatureService.setPreWordNatures(wordNatures);
onlineKnowledgeService.reloadAllData(wordNatures);
log.info("ApplicationStartedInit end");
} catch (Exception e) {
log.error("ApplicationStartedInit error", e);
}
}
/***
* reload knowledge task
*/
@Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}")
public void reloadKnowledge() {
log.info("reloadKnowledge start");
try {
List<WordNature> wordNatures = wordNatureService.getAllWordNature();
List<WordNature> preWordNatures = wordNatureService.getPreWordNatures();
if (CollectionUtils.isEqualCollection(wordNatures, preWordNatures)) {
log.debug("wordNatures is not change, reloadKnowledge end");
return;
}
log.info("wordNatures is change");
wordNatureService.setPreWordNatures(wordNatures);
onlineKnowledgeService.updateOnlineKnowledge(wordNatureService.getAllWordNature());
wordNatureService.getCache().refresh("");
} catch (Exception e) {
log.error("reloadKnowledge error", e);
}
log.info("reloadKnowledge end");
}
}

144
chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/NatureHelper.java
View File

@@ -1,144 +0,0 @@
package com.tencent.supersonic.chat.application.knowledge;
import com.hankcs.hanlp.corpus.tag.Nature;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.chat.application.mapper.HanlpSchemaMapper;
import com.tencent.supersonic.chat.domain.pojo.search.DomainInfoStat;
import com.tencent.supersonic.common.nlp.NatureType;
import com.tencent.supersonic.knowledge.application.online.BaseWordNature;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* nature parse helper
*/
@Slf4j
public class NatureHelper {
private static boolean isDomainOrEntity(Term term, Integer domain) {
return (NatureType.NATURE_SPILT + domain).equals(term.nature.toString()) || term.nature.toString()
.endsWith(NatureType.ENTITY.getType());
}
public static Integer getDomainByNature(Nature nature) {
if (nature.startsWith(NatureType.NATURE_SPILT)) {
String[] dimensionValues = nature.toString().split(NatureType.NATURE_SPILT);
if (StringUtils.isNumeric(dimensionValues[1])) {
return Integer.valueOf(dimensionValues[1]);
}
}
return 0;
}
public static Integer getDomain(String nature) {
try {
String[] split = nature.split(NatureType.NATURE_SPILT);
if (split.length <= 1) {
return null;
}
return Integer.valueOf(split[1]);
} catch (NumberFormatException e) {
log.error("", e);
}
return null;
}
public static boolean isDimensionValueClassId(String nature) {
if (StringUtils.isEmpty(nature)) {
return false;
}
if (!nature.startsWith(NatureType.NATURE_SPILT)) {
return false;
}
String[] split = nature.split(NatureType.NATURE_SPILT);
if (split.length <= 1) {
return false;
}
return !nature.endsWith(NatureType.METRIC.getType()) && !nature.endsWith(NatureType.DIMENSION.getType())
&& StringUtils.isNumeric(split[1]);
}
public static DomainInfoStat getDomainStat(List<Term> terms) {
DomainInfoStat stat = new DomainInfoStat();
stat.setDimensionDomainCount(getDimensionCount(terms));
stat.setMetricDomainCount(getMetricCount(terms));
stat.setDomainCount(getDomainCount(terms));
stat.setDimensionValueDomainCount(getDimensionValueCount(terms));
return stat;
}
private static long getDomainCount(List<Term> terms) {
return terms.stream().filter(term -> isDomainOrEntity(term, getDomainByNature(term.nature))).count();
}
private static long getDimensionValueCount(List<Term> terms) {
return terms.stream().filter(term -> isDimensionValueClassId(term.nature.toString())).count();
}
private static long getDimensionCount(List<Term> terms) {
return terms.stream().filter(term -> term.nature.startsWith(NatureType.NATURE_SPILT) && term.nature.toString()
.endsWith(NatureType.DIMENSION.getType())).count();
}
private static long getMetricCount(List<Term> terms) {
return terms.stream().filter(term -> term.nature.startsWith(NatureType.NATURE_SPILT) && term.nature.toString()
.endsWith(NatureType.METRIC.getType())).count();
}
/**
* Get the number of types of class parts of speech
* domainId -> (nature , natureCount)
*
* @param terms
* @return
*/
public static Map<Integer, Map<NatureType, Integer>> getDomainToNatureStat(List<Term> terms) {
Map<Integer, Map<NatureType, Integer>> domainToNature = new HashMap<>();
terms.stream().filter(
term -> term.nature.startsWith(NatureType.NATURE_SPILT)
).forEach(term -> {
NatureType natureType = NatureType.getNatureType(String.valueOf(term.nature));
Integer domain = getDomain(String.valueOf(term.nature));
Map<NatureType, Integer> natureTypeMap = new HashMap<>();
natureTypeMap.put(natureType, 1);
Map<NatureType, Integer> original = domainToNature.get(domain);
if (Objects.isNull(original)) {
domainToNature.put(domain, natureTypeMap);
} else {
Integer count = original.get(natureType);
if (Objects.isNull(count)) {
count = 1;
} else {
count = count + 1;
}
original.put(natureType, count);
}
});
return domainToNature;
}
public static List<Integer> selectPossibleDomains(List<Term> terms) {
Map<Integer, Map<NatureType, Integer>> domainToNatureStat = getDomainToNatureStat(terms);
Integer maxDomainTypeSize = domainToNatureStat.entrySet().stream()
.max(Comparator.comparingInt(o -> o.getValue().size())).map(entry -> entry.getValue().size())
.orElse(null);
if (Objects.isNull(maxDomainTypeSize) || maxDomainTypeSize == 0) {
return new ArrayList<>();
}
return domainToNatureStat.entrySet().stream().filter(entry -> entry.getValue().size() == maxDomainTypeSize)
.map(entry -> entry.getKey()).collect(Collectors.toList());
}
}

81
chat/core/src/main/java/com/tencent/supersonic/chat/application/knowledge/WordNatureService.java
View File

@@ -1,81 +0,0 @@
package com.tencent.supersonic.chat.application.knowledge;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.nlp.ItemDO;
import com.tencent.supersonic.common.nlp.NatureType;
import com.tencent.supersonic.common.nlp.WordNature;
import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
/**
* word nature service
**/
@Service
@Slf4j
public class WordNatureService {
private static final Integer META_CACHE_TIME = 5;
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
private List<WordNature> preWordNatures = new ArrayList<>();
private LoadingCache<String, DomainInfos> cache = CacheBuilder.newBuilder()
.expireAfterWrite(META_CACHE_TIME, TimeUnit.MINUTES)
.build(
new CacheLoader<String, DomainInfos>() {
@Override
public DomainInfos load(String key) {
log.info("load getDomainSchemaInfo cache [{}]", key);
return SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
}
}
);
public List<WordNature> getAllWordNature() {
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
DomainInfos domainInfos = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
List<WordNature> natures = new ArrayList<>();
addNatureToResult(NatureType.DIMENSION, domainInfos.getDimensions(), natures);
addNatureToResult(NatureType.METRIC, domainInfos.getMetrics(), natures);
addNatureToResult(NatureType.DOMAIN, domainInfos.getDomains(), natures);
addNatureToResult(NatureType.ENTITY, domainInfos.getEntities(), natures);
return natures;
}
private void addNatureToResult(NatureType value, List<ItemDO> metas, List<WordNature> natures) {
List<WordNature> natureList = WordNatureStrategyFactory.get(value).getWordNatureList(metas);
log.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
natures.addAll(natureList);
}
public List<WordNature> getPreWordNatures() {
return preWordNatures;
}
public void setPreWordNatures(List<WordNature> preWordNatures) {
this.preWordNatures = preWordNatures;
}
public LoadingCache<String, DomainInfos> getCache() {
return cache;
}
}

143
chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/DatabaseSchemaMapper.java
View File

@@ -1,143 +0,0 @@
package com.tencent.supersonic.chat.application.mapper;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.application.knowledge.WordNatureService;
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
import com.tencent.supersonic.common.nlp.ItemDO;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Slf4j
public class DatabaseSchemaMapper implements SchemaMapper {
@Override
public void map(QueryContextReq queryContext) {
log.debug("before db mapper,mapInfo:{}", queryContext.getMapInfo());
List<Term> terms = HanlpHelper.getTerms(queryContext.getQueryText());
WordNatureService wordNatureService = ContextUtils.getBean(WordNatureService.class);
DomainInfos domainInfos = wordNatureService.getCache().getUnchecked("");
detectAndAddToSchema(queryContext, terms, domainInfos.getDimensions(),
SchemaElementType.DIMENSION);
detectAndAddToSchema(queryContext, terms, domainInfos.getMetrics(), SchemaElementType.METRIC);
log.debug("after db mapper,mapInfo:{}", queryContext.getMapInfo());
}
private void detectAndAddToSchema(QueryContextReq queryContext, List<Term> terms, List<ItemDO> domains,
SchemaElementType schemaElementType) {
try {
String queryText = queryContext.getQueryText();
Map<String, Set<ItemDO>> domainResultSet = getResultSet(queryText, terms, domains);
addToSchemaMapInfo(domainResultSet, queryContext.getMapInfo(), schemaElementType);
} catch (Exception e) {
log.error("detectAndAddToSchema error", e);
}
}
private Map<String, Set<ItemDO>> getResultSet(String queryText, List<Term> terms, List<ItemDO> domains) {
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
Map<String, Set<ItemDO>> nameToItems = getNameToItems(domains);
Map<Integer, Integer> regOffsetToLength = terms.stream().sorted(Comparator.comparing(Term::length))
.collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2));
Map<String, Set<ItemDO>> domainResultSet = new HashMap<>();
for (Integer index = 0; index <= queryText.length() - 1; ) {
for (Integer i = index; i <= queryText.length(); ) {
i = mapperHelper.getStepIndex(regOffsetToLength, i);
if (i <= queryText.length()) {
String detectSegment = queryText.substring(index, i);
nameToItems.forEach(
(name, newItemDOs) -> {
if (name.contains(detectSegment)
&& mapperHelper.getSimilarity(detectSegment, name)
>= mapperHelper.getMetricDimensionThresholdConfig()) {
Set<ItemDO> preItemDOS = domainResultSet.putIfAbsent(detectSegment, newItemDOs);
if (Objects.nonNull(preItemDOS)) {
preItemDOS.addAll(newItemDOs);
}
}
}
);
}
}
index = mapperHelper.getStepIndex(regOffsetToLength, index);
}
return domainResultSet;
}
private Map<String, Set<ItemDO>> getNameToItems(List<ItemDO> domains) {
return domains.stream()
.collect(Collectors.toMap(ItemDO::getName, a -> {
Set<ItemDO> result = new HashSet<>();
result.add(a);
return result;
}, (k1, k2) -> {
k1.addAll(k2);
return k1;
}));
}
private void addToSchemaMapInfo(Map<String, Set<ItemDO>> mapResultRowSet, SchemaMapInfo schemaMap,
SchemaElementType schemaElementType) {
if (Objects.isNull(mapResultRowSet) || mapResultRowSet.size() <= 0) {
return;
}
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
for (Map.Entry<String, Set<ItemDO>> entry : mapResultRowSet.entrySet()) {
String detectWord = entry.getKey();
Set<ItemDO> itemDOS = entry.getValue();
for (ItemDO itemDO : itemDOS) {
List<SchemaElementMatch> elements = schemaMap.getMatchedElements(itemDO.getDomain());
if (CollectionUtils.isEmpty(elements)) {
elements = new ArrayList<>();
schemaMap.setMatchedElements(itemDO.getDomain(), elements);
}
Set<Integer> regElementSet = elements.stream()
.filter(elementMatch -> schemaElementType.equals(elementMatch.getElementType()))
.map(elementMatch -> elementMatch.getElementID())
.collect(Collectors.toSet());
if (regElementSet.contains(itemDO.getItemId())) {
continue;
}
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.elementID(itemDO.getItemId()).word(itemDO.getName()).frequency(10000L)
.elementType(schemaElementType).detectWord(detectWord)
.similarity(mapperHelper.getSimilarity(detectWord, itemDO.getName()))
.build();
log.info("schemaElementType:{},add to schema, elementMatch {}", schemaElementType, schemaElementMatch);
elements.add(schemaElementMatch);
}
}
}
}

103
chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java
View File

@@ -1,103 +0,0 @@
package com.tencent.supersonic.chat.application.mapper;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
import com.tencent.supersonic.chat.domain.utils.NatureConverter;
import com.tencent.supersonic.common.nlp.MapResult;
import com.tencent.supersonic.common.nlp.NatureType;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.knowledge.application.online.BaseWordNature;
import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory;
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@Slf4j
public class HanlpSchemaMapper implements SchemaMapper {
@Override
public void map(QueryContextReq queryContext) {
List<Term> terms = HanlpHelper.getTerms(queryContext.getQueryText());
terms.forEach(
item -> log.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(),
item.getFrequency())
);
QueryMatchStrategy matchStrategy = ContextUtils.getBean(QueryMatchStrategy.class);
Map<MatchText, List<MapResult>> matchResult = matchStrategy.match(queryContext.getQueryText(), terms,
queryContext.getDomainId());
List<MapResult> matches = new ArrayList<>();
if (Objects.nonNull(matchResult)) {
Optional<List<MapResult>> first = matchResult.entrySet().stream()
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
.map(entry -> entry.getValue()).findFirst();
if (first.isPresent()) {
matches = first.get();
}
}
HanlpHelper.transLetterOriginal(matches);
log.info("queryContext:{},matches:{}", queryContext, matches);
convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo(), terms);
}
private void convertTermsToSchemaMapInfo(List<MapResult> mapResults, SchemaMapInfo schemaMap, List<Term> terms) {
if (CollectionUtils.isEmpty(mapResults)) {
return;
}
Map<String, Long> wordNatureToFrequency = terms.stream().collect(
Collectors.toMap(entry -> entry.getWord() + entry.getNature(),
term -> Long.valueOf(term.getFrequency()), (value1, value2) -> value2));
for (MapResult mapResult : mapResults) {
for (String nature : mapResult.getNatures()) {
Integer domain = NatureHelper.getDomain(nature);
if (Objects.isNull(domain)) {
continue;
}
SchemaElementType elementType = NatureConverter.convertTo(nature);
if (Objects.isNull(elementType)) {
continue;
}
BaseWordNature baseWordNature = WordNatureStrategyFactory.get(NatureType.getNatureType(nature));
Integer elementID = baseWordNature.getElementID(nature);
Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature);
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.elementType(elementType)
.elementID(elementID)
.frequency(frequency)
.word(mapResult.getName())
.similarity(mapResult.getSimilarity())
.detectWord(mapResult.getDetectWord())
.build();
Map<Integer, List<SchemaElementMatch>> domainElementMatches = schemaMap.getDomainElementMatches();
List<SchemaElementMatch> schemaElementMatches = domainElementMatches.putIfAbsent(domain,
new ArrayList<>());
if (schemaElementMatches == null) {
schemaElementMatches = domainElementMatches.get(domain);
}
schemaElementMatches.add(schemaElementMatch);
}
}
}
}

66
chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryFilterMapper.java
View File

@@ -1,66 +0,0 @@
package com.tencent.supersonic.chat.application.mapper;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.common.constant.Constants;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Slf4j
public class QueryFilterMapper implements SchemaMapper {
private Long FREQUENCY = 9999999L;
private double SIMILARITY = 1.0;
@Override
public void map(QueryContextReq queryContext) {
Integer domainId = queryContext.getDomainId();
if (domainId == null || domainId <= 0 || queryContext.getQueryFilter() == null) {
return;
}
QueryFilter queryFilter = queryContext.getQueryFilter();
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
List<SchemaElementMatch> schemaElementMatches = schemaMapInfo.getMatchedElements(domainId);
clearOtherSchemaElementMatch(domainId, schemaMapInfo);
convertFilterToSchemaMapInfo(queryFilter.getFilters(), schemaElementMatches);
}
private void convertFilterToSchemaMapInfo(List<Filter> filters, List<SchemaElementMatch> schemaElementMatches) {
log.info("schemaElementMatches before queryFilerMapper:{}", schemaElementMatches);
if (CollectionUtils.isEmpty(schemaElementMatches)) {
schemaElementMatches = Lists.newArrayList();
}
List<String> words = schemaElementMatches.stream().map(SchemaElementMatch::getWord).collect(Collectors.toList());
for (Filter filter : filters) {
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.elementType(SchemaElementType.VALUE)
.elementID(filter.getElementID().intValue())
.frequency(FREQUENCY)
.word(String.valueOf(filter.getValue()))
.similarity(SIMILARITY)
.detectWord(Constants.EMPTY)
.build();
if (words.contains(schemaElementMatch.getWord())) {
continue;
}
schemaElementMatches.add(schemaElementMatch);
}
log.info("schemaElementMatches after queryFilerMapper:{}", schemaElementMatches);
}
private void clearOtherSchemaElementMatch(Integer domainId, SchemaMapInfo schemaMapInfo) {
for (Map.Entry<Integer, List<SchemaElementMatch>> entry : schemaMapInfo.getDomainElementMatches().entrySet()) {
if (!entry.getKey().equals(domainId)) {
entry.getValue().clear();
}
}
}
}

102
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/AggregateSemanticParser.java
View File

@@ -1,102 +0,0 @@
package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.application.query.EntityListFilter;
import com.tencent.supersonic.chat.application.query.MetricGroupBy;
import com.tencent.supersonic.chat.application.query.MetricOrderBy;
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class AggregateSemanticParser implements SemanticParser {
public static final Integer TOPN_LIMIT = 1000;
private static Map<AggregateTypeEnum, Pattern> aggregateRegexMap = new HashMap<>();
static {
aggregateRegexMap.put(AggregateTypeEnum.MAX, Pattern.compile("(?i)(最大值|最大|max|峰值|最高|最多)"));
aggregateRegexMap.put(AggregateTypeEnum.MIN, Pattern.compile("(?i)(最小值|最小|min|最低|最少)"));
aggregateRegexMap.put(AggregateTypeEnum.SUM, Pattern.compile("(?i)(汇总|总和|sum)"));
aggregateRegexMap.put(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|日均|平均|avg)"));
aggregateRegexMap.put(AggregateTypeEnum.TOPN, Pattern.compile("(?i)(top)"));
aggregateRegexMap.put(AggregateTypeEnum.DISTINCT, Pattern.compile("(?i)(uv)"));
aggregateRegexMap.put(AggregateTypeEnum.COUNT, Pattern.compile("(?i)(总数|pv)"));
aggregateRegexMap.put(AggregateTypeEnum.NONE, Pattern.compile("(?i)(明细)"));
}
public static AggregateTypeEnum resolveAggregateType(String queryText) {
Map<AggregateTypeEnum, Integer> aggregateCount = new HashMap<>(aggregateRegexMap.size());
for (Map.Entry<AggregateTypeEnum, Pattern> entry : aggregateRegexMap.entrySet()) {
Matcher matcher = entry.getValue().matcher(queryText);
int count = 0;
while (matcher.find()) {
count++;
}
if (count > 0) {
aggregateCount.put(entry.getKey(), count);
}
}
return aggregateCount.entrySet().stream().max(Map.Entry.comparingByValue()).map(entry -> entry.getKey())
.orElse(null);
}
@Override
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
AggregateTypeEnum aggregateType = resolveAggregateType(queryContext.getQueryText());
for (SemanticQuery semanticQuery : queryContext.getCandidateQueries()) {
SemanticParseInfo semanticParse = semanticQuery.getParseInfo();
semanticParse.setNativeQuery(getNativeQuery(aggregateType, semanticParse));
semanticParse.setAggType(aggregateType);
if (Objects.isNull(semanticParse.getLimit()) || semanticParse.getLimit() <= 0) {
semanticParse.setLimit(Long.valueOf(TOPN_LIMIT));
}
resetQueryModeByAggregateType(semanticParse, aggregateType);
}
}
/**
* query mode reset by the AggregateType
*
* @param parseInfo
* @param aggregateType
*/
private void resetQueryModeByAggregateType(SemanticParseInfo parseInfo,
AggregateTypeEnum aggregateType) {
String queryMode = parseInfo.getQueryMode();
if (MetricGroupBy.QUERY_MODE.equals(queryMode) || MetricGroupBy.QUERY_MODE.equals(queryMode)) {
if (AggregateTypeEnum.MAX.equals(aggregateType) || AggregateTypeEnum.MIN.equals(aggregateType)
|| AggregateTypeEnum.TOPN.equals(aggregateType)) {
parseInfo.setQueryMode(MetricOrderBy.QUERY_MODE);
} else {
parseInfo.setQueryMode(MetricGroupBy.QUERY_MODE);
}
log.info("queryMode mode [{}]->[{}]", queryMode, parseInfo.getQueryMode());
}
}
private boolean getNativeQuery(AggregateTypeEnum aggregateType, SemanticParseInfo semanticParse) {
if (AggregateTypeEnum.TOPN.equals(aggregateType)) {
return true;
}
if (EntityListFilter.QUERY_MODE.equals(semanticParse.getQueryMode()) && (semanticParse.getMetrics() == null
|| semanticParse.getMetrics().isEmpty())) {
return true;
}
return semanticParse.getNativeQuery();
}
}

19
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainResolver.java
View File

@@ -1,19 +0,0 @@
package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import java.util.Map;
public interface DomainResolver {
Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq queryCtx, ChatContext chatCtx,
SchemaMapInfo schemaMap);
boolean isDomainSwitch(ChatContext chatCtx, SemanticParseInfo semanticParseInfo);
}

250
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java
View File

@@ -1,250 +0,0 @@
package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.application.query.EntitySemanticQuery;
import com.tencent.supersonic.chat.application.query.MetricSemanticQuery;
import com.tencent.supersonic.chat.application.query.RuleSemanticQuery;
import com.tencent.supersonic.chat.application.query.RuleSemanticQueryManager;
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigResp;
import com.tencent.supersonic.chat.domain.service.ConfigService;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import com.tencent.supersonic.chat.domain.utils.DefaultMetricUtils;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.util.context.ContextUtils;
import java.util.*;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Slf4j
public class DomainSemanticParser implements SemanticParser {
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
@Override
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
DomainInfos domainInfosDb = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
Map<Integer, String> domainToName = domainInfosDb.getDomainToName();
SchemaMapInfo mapInfo = queryContext.getMapInfo();
// iterate all schemaElementMatches to resolve semantic query
for (Integer domainId : mapInfo.getMatchedDomains()) {
List<SchemaElementMatch> elementMatches = mapInfo.getMatchedElements(domainId);
List<RuleSemanticQuery> queries = resolveQuery(elementMatches, queryContext);
for (RuleSemanticQuery query : queries) {
if (useBlackItem(query, domainId)) {
log.info("useBlackItem, skip query:{}", query);
continue;
}
addCandidateQuery(queryContext, chatContext, domainId.longValue(),
domainToName.get(domainId), query);
}
}
// if no candidates have been found yet, count in chat context and try again
if (queryContext.getCandidateQueries().size() <= 0) {
if (chatContext.getParseInfo() != null && chatContext.getParseInfo().getDomainId() > 0) {
Integer chatDomainId = Integer.valueOf(chatContext.getParseInfo().getDomainId().intValue());
if (mapInfo.getMatchedDomains().contains(chatDomainId)) {
List<SchemaElementMatch> elementMatches = mapInfo.getMatchedElements(chatDomainId);
List<RuleSemanticQuery> queries = tryParseByContext(elementMatches, chatContext, queryContext);
for (RuleSemanticQuery query : queries) {
addCandidateQuery(queryContext, chatContext, chatDomainId.longValue(),
domainToName.get(chatDomainId), query);
}
}
}
}
}
private boolean useBlackItem(RuleSemanticQuery query, Integer domainId) {
if (Objects.isNull(domainId)) {
return false;
}
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigResp chatConfigResp = configService.fetchConfigByDomainId(domainId.longValue());
if (Objects.nonNull(chatConfigResp) && Objects.nonNull(query) && Objects.nonNull(query.getParseInfo())) {
List<SchemaElementMatch> elementMatches = query.getParseInfo().getElementMatches();
if (!CollectionUtils.isEmpty(elementMatches)) {
return useBlackItemInternal(elementMatches, chatConfigResp, query);
}
}
return false;
}
private boolean useBlackItemInternal(List<SchemaElementMatch> elementMatches, ChatConfigResp chatConfigResp, RuleSemanticQuery query) {
if (Objects.isNull(chatConfigResp)) {
return false;
}
List<Long> blackDimIdList = new ArrayList<>();
List<Long> blackMetricIdList = new ArrayList<>();
if (query instanceof EntitySemanticQuery
&& Objects.nonNull(chatConfigResp.getChatDetailConfig())
&& Objects.nonNull(chatConfigResp.getChatDetailConfig().getVisibility())) {
log.info("useBlackItem, handle EntitySemanticQuery blackList logic");
blackDimIdList = chatConfigResp.getChatDetailConfig().getVisibility().getBlackDimIdList();
blackMetricIdList = chatConfigResp.getChatDetailConfig().getVisibility().getBlackMetricIdList();
}
if (query instanceof MetricSemanticQuery
&& Objects.nonNull(chatConfigResp.getChatAggConfig())
&& Objects.nonNull(chatConfigResp.getChatAggConfig().getVisibility())) {
log.info("useBlackItem, handle MetricSemanticQuery blackList logic");
blackDimIdList = chatConfigResp.getChatAggConfig().getVisibility().getBlackDimIdList();
blackMetricIdList = chatConfigResp.getChatAggConfig().getVisibility().getBlackMetricIdList();
}
return useBlackItemWithElementMatches(elementMatches, blackDimIdList, blackMetricIdList);
}
private boolean useBlackItemWithElementMatches(List<SchemaElementMatch> elementMatches, List<Long> blackDimIdList, List<Long> blackMetricIdList) {
Set<Long> dimIds = elementMatches.stream()
.filter(element -> SchemaElementType.VALUE.equals(element.getElementType()) || SchemaElementType.DIMENSION.equals(element.getElementType()))
.map(element -> Long.valueOf(element.getElementID())).collect(Collectors.toSet());
Set<Long> metricIds = elementMatches.stream()
.filter(element -> SchemaElementType.METRIC.equals(element.getElementType()))
.map(element -> Long.valueOf(element.getElementID())).collect(Collectors.toSet());
return useBlackItemWithIds(dimIds, metricIds, blackDimIdList, blackMetricIdList);
}
private boolean useBlackItemWithIds(Set<Long> dimIds, Set<Long> metricIds, List<Long> blackDimIdList, List<Long> blackMetricIdList) {
if (!CollectionUtils.isEmpty(blackDimIdList) && !CollectionUtils.isEmpty(dimIds)) {
if (blackDimIdList.stream().anyMatch(dimIds::contains)) {
log.info("useBlackItem, blackDimIdList:{}", blackDimIdList.stream().filter(dimIds::contains).collect(Collectors.toList()));
return true;
}
}
if (!CollectionUtils.isEmpty(blackMetricIdList) && !CollectionUtils.isEmpty(metricIds)) {
if (blackMetricIdList.stream().anyMatch(metricIds::contains)) {
log.info("useBlackItem, blackMetricIdList:{}", blackMetricIdList.stream().filter(metricIds::contains).collect(Collectors.toList()));
return true;
}
}
return false;
}
private void addCandidateQuery(QueryContextReq queryContext, ChatContext chatContext,
Long domainId, String domainName, RuleSemanticQuery semanticQuery) {
if (semanticQuery != null) {
DefaultMetricUtils defaultMetricUtils = ContextUtils.getBean(DefaultMetricUtils.class);
defaultMetricUtils.fillParseInfo(semanticQuery, domainId, domainName);
inheritContext(semanticQuery, chatContext);
defaultMetricUtils.fillDefaultMetric(semanticQuery.getParseInfo(), queryContext, chatContext);
queryContext.getCandidateQueries().add(semanticQuery);
}
}
protected void inheritContext(RuleSemanticQuery semanticQuery, ChatContext chatContext) {
// is domain switch
SemanticParseInfo semanticParse = semanticQuery.getParseInfo();
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
if (!domainResolver.isDomainSwitch(chatContext, semanticParse)) {
semanticQuery.inheritContext(chatContext);
}
}
/**
* try to add ChatContext to SchemaMatch and look if match QueryMode
*
* @param elementMatches
* @param chatCtx
* @return
*/
private List<RuleSemanticQuery> tryParseByContext(List<SchemaElementMatch> elementMatches,
ChatContext chatCtx, QueryContextReq queryCtx) {
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getEntity() > 0) {
Long entityCount = elementMatches.stream().filter(i -> SchemaElementType.ENTITY.equals(i.getElementType()))
.count();
Long metricCount = elementMatches.stream().filter(i -> SchemaElementType.METRIC.equals(i.getElementType()))
.count();
if (entityCount <= 0 && metricCount <= 0 && ContextHelper.hasEntityId(chatCtx)) {
// try entity parse
SchemaElementMatch entityElementMatch = SchemaElementMatch.builder()
.elementType(SchemaElementType.ENTITY).build();
List<SchemaElementMatch> newSchemaMatches = new ArrayList<>();
if (!CollectionUtils.isEmpty(elementMatches)) {
newSchemaMatches.addAll(elementMatches);
}
newSchemaMatches.add(entityElementMatch);
List<RuleSemanticQuery> queries = doParseByContext(newSchemaMatches, chatCtx, queryCtx);
if (queries.size() > 0) {
return queries;
}
}
}
return doParseByContext(elementMatches, chatCtx, queryCtx);
}
private List<RuleSemanticQuery> doParseByContext(List<SchemaElementMatch> elementMatches,
ChatContext chatCtx, QueryContextReq queryContext) {
SemanticParseInfo contextSemanticParse = chatCtx.getParseInfo();
if (contextSemanticParse != null) {
List<SchemaElementMatch> newElementMatches = new ArrayList<>();
List<List<SchemaElementType>> trySchemaElementTypes = new LinkedList<>();
// try DIMENSION+METRIC+VALUE
// try DIMENSION+METRIC METRIC+VALUE DIMENSION+VALUE
// try DIMENSION METRIC VALUE single
trySchemaElementTypes.add(new ArrayList<>(
Arrays.asList(SchemaElementType.DIMENSION, SchemaElementType.METRIC, SchemaElementType.VALUE)));
trySchemaElementTypes.add(
new ArrayList<>(Arrays.asList(SchemaElementType.METRIC, SchemaElementType.VALUE)));
trySchemaElementTypes.add(
new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION, SchemaElementType.METRIC)));
trySchemaElementTypes.add(
new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION, SchemaElementType.VALUE)));
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.METRIC)));
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.VALUE)));
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION)));
for (List<SchemaElementType> schemaTypes : trySchemaElementTypes) {
newElementMatches.clear();
if (!CollectionUtils.isEmpty(elementMatches)) {
newElementMatches.addAll(elementMatches);
}
ContextHelper.mergeContextSchemaElementMatch(newElementMatches, elementMatches, schemaTypes,
contextSemanticParse);
List<RuleSemanticQuery> queries = resolveQuery(newElementMatches, queryContext);
if (queries.size() > 0) {
return queries;
}
}
}
return new ArrayList<>();
}
private List<RuleSemanticQuery> resolveQuery(List<SchemaElementMatch> candidateElementMatches,
QueryContextReq queryContext) {
List<RuleSemanticQuery> matchedQueries = new ArrayList<>();
for (RuleSemanticQuery semanticQuery : RuleSemanticQueryManager.getSemanticQueries()) {
List<SchemaElementMatch> matches = semanticQuery.match(candidateElementMatches, queryContext);
if (matches.size() > 0) {
log.info("resolve match [{}:{}] ", semanticQuery.getQueryMode(), matches.size());
RuleSemanticQuery query = RuleSemanticQueryManager.create(semanticQuery.getQueryMode());
query.getParseInfo().getElementMatches().addAll(matches);
matchedQueries.add(query);
}
}
return matchedQueries;
}
}

147
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/HeuristicDomainResolver.java
View File

@@ -1,147 +0,0 @@
package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import lombok.extern.slf4j.Slf4j;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
public class HeuristicDomainResolver implements DomainResolver {
protected static Integer selectDomainBySchemaElementCount(Map<Integer, SemanticQuery> domainQueryModes,
SchemaMapInfo schemaMap) {
Map<Integer, QueryMatchInfo> domainTypeMap = getDomainTypeMap(schemaMap);
if (domainTypeMap.size() == 1) {
Integer domainSelect = domainTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey();
if (domainQueryModes.containsKey(domainSelect)) {
log.info("selectDomain from domainTypeMap not order [{}]", domainSelect);
return domainSelect;
}
} else {
Map.Entry<Integer, QueryMatchInfo> maxDomain = domainTypeMap.entrySet().stream()
.filter(entry -> domainQueryModes.containsKey(entry.getKey()))
.sorted(ContextHelper.DomainStatComparator).findFirst().orElse(null);
if (maxDomain != null) {
log.info("selectDomain from domainTypeMap order [{}]", maxDomain.getKey());
return maxDomain.getKey();
}
}
return 0;
}
/**
* to check can switch domain if context exit domain
*
* @return false will use context domain, true will use other domain , maybe include context domain
*/
protected static boolean isAllowSwitch(Map<Integer, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap,
ChatContext chatCtx, QueryContextReq searchCtx, Integer domainId) {
if (!Objects.nonNull(domainId) || domainId <= 0) {
return true;
}
// except content domain, calculate the number of types for each domain, if numbers<=1 will not switch
Map<Integer, QueryMatchInfo> domainTypeMap = getDomainTypeMap(schemaMap);
log.info("isAllowSwitch domainTypeMap [{}]", domainTypeMap);
long otherDomainTypeNumBigOneCount = domainTypeMap.entrySet().stream()
.filter(entry -> domainQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(domainId))
.filter(entry -> entry.getValue().getCount() > 1).count();
if (otherDomainTypeNumBigOneCount >= 1) {
return true;
}
// if query text only contain time , will not switch
for (SemanticQuery semanticQuery : domainQueryModes.values()) {
SemanticParseInfo semanticParseInfo = semanticQuery.getParseInfo();
if (semanticParseInfo == null) {
continue;
}
if (searchCtx.getQueryText() != null && semanticParseInfo.getDateInfo() != null) {
if (semanticParseInfo.getDateInfo().getText() != null) {
if (semanticParseInfo.getDateInfo().getText().equalsIgnoreCase(searchCtx.getQueryText())) {
log.info("timeParseResults is not null , can not switch context , timeParseResults:{},",
semanticParseInfo.getDateInfo());
return false;
}
}
}
}
// if context domain not in schemaMap , will switch
if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) {
log.info("domainId not in schemaMap ");
return true;
}
// other will not switch
return false;
}
public static Map<Integer, QueryMatchInfo> getDomainTypeMap(SchemaMapInfo schemaMap) {
Map<Integer, QueryMatchInfo> domainCount = new HashMap<>();
for (Map.Entry<Integer, List<SchemaElementMatch>> entry : schemaMap.getDomainElementMatches().entrySet()) {
List<SchemaElementMatch> schemaElementMatches = schemaMap.getMatchedElements(entry.getKey());
if (schemaElementMatches != null && schemaElementMatches.size() > 0) {
if (!domainCount.containsKey(entry.getKey())) {
domainCount.put(entry.getKey(), new QueryMatchInfo());
}
QueryMatchInfo queryMatchInfo = domainCount.get(entry.getKey());
Set<SchemaElementType> schemaElementTypes = new HashSet<>();
schemaElementMatches.stream()
.forEach(schemaElementMatch -> schemaElementTypes.add(schemaElementMatch.getElementType()));
SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream()
.sorted(ContextHelper.schemaElementMatchComparatorBySimilarity).findFirst().orElse(null);
if (schemaElementMatchMax != null) {
queryMatchInfo.setMaxSimilarity(schemaElementMatchMax.getSimilarity());
}
queryMatchInfo.setCount(schemaElementTypes.size());
}
}
return domainCount;
}
@Override
public boolean isDomainSwitch(ChatContext chatCtx, SemanticParseInfo semanticParseInfo) {
Long contextDomain = chatCtx.getParseInfo().getDomainId();
Long currentDomain = semanticParseInfo.getDomainId();
boolean noSwitch =
currentDomain == null || contextDomain == null || contextDomain.equals(currentDomain);
log.debug("ChatContext isDomainSwitch [{}] [{}]",
semanticParseInfo.getQueryMode(), !noSwitch);
return !noSwitch;
}
@Override
public Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
ChatContext chatCtx, SchemaMapInfo schemaMap) {
Integer selectDomain = selectDomain(domainQueryModes, searchCtx, chatCtx, schemaMap);
if (selectDomain > 0) {
log.info("selectDomain {} ", selectDomain);
return selectDomain;
}
// get the max SchemaElementType number
return selectDomainBySchemaElementCount(domainQueryModes, schemaMap);
}
public Integer selectDomain(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
ChatContext chatCtx,
SchemaMapInfo schemaMap) {
// if QueryContext has domainId and in domainQueryModes
if (domainQueryModes.containsKey(searchCtx.getDomainId())) {
log.info("selectDomain from QueryContext [{}]", searchCtx.getDomainId());
return searchCtx.getDomainId();
}
// if ChatContext has domainId and in domainQueryModes
if (chatCtx.getParseInfo().getDomainId() > 0) {
Integer domainId = Integer.valueOf(chatCtx.getParseInfo().getDomainId().intValue());
if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, searchCtx, domainId)) {
log.info("selectDomain from ChatContext [{}]", domainId);
return domainId;
}
}
// default 0
return 0;
}
}

160
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/LLMSemanticParser.java
View File

@@ -1,160 +0,0 @@
package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.application.knowledge.WordNatureService;
import com.tencent.supersonic.chat.application.query.LLMSemanticQuery;
import com.tencent.supersonic.chat.domain.config.LLMConfig;
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
import com.tencent.supersonic.chat.domain.pojo.chat.LLMReq;
import com.tencent.supersonic.chat.domain.pojo.chat.LLMResp;
import com.tencent.supersonic.chat.domain.pojo.chat.LLMSchema;
import com.tencent.supersonic.chat.domain.utils.DslToSemanticInfo;
import com.tencent.supersonic.chat.domain.utils.SemanticSatisfactionChecker;
import com.tencent.supersonic.common.nlp.ItemDO;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.common.util.json.JsonUtil;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.io.support.SpringFactoriesLoader;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
@Slf4j
public class LLMSemanticParser implements SemanticParser {
private DslToSemanticInfo dslToSemanticInfo = new DslToSemanticInfo();
@Override
public void parse(QueryContextReq queryContext, ChatContext chatCtx) {
String queryText = queryContext.getQueryText();
if (SemanticSatisfactionChecker.check(queryContext)) {
log.info("There is no need parse by llm , queryText:{}", queryText);
return;
}
try {
Integer domainId = getDomainId(queryContext, chatCtx);
LLMResp llmResp = requestLLM(queryContext, domainId);
if (Objects.isNull(llmResp)) {
return;
}
LLMSemanticQuery semanticQuery = new LLMSemanticQuery();
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
String sql = convertToSql(llmResp, parseInfo, domainId);
parseInfo.setInfo(sql);
parseInfo.setDomainId(Long.valueOf(domainId));
parseInfo.setBonus(queryText.length() * 1.0);
parseInfo.setQueryMode(LLMSemanticQuery.QUERY_MODE);
queryContext.getCandidateQueries().add(semanticQuery);
return;
} catch (Exception e) {
log.error("llm parse error , skip the parser. error:", e);
}
}
protected String convertToSql(LLMResp llmResp, SemanticParseInfo parseInfo, Integer domainId)
throws SqlParseException {
return dslToSemanticInfo.convert(parseInfo, llmResp, domainId);
}
protected LLMResp requestLLM(QueryContextReq queryContext, Integer domainId) {
final LLMConfig llmConfig = ContextUtils.getBean(LLMConfig.class);
if (StringUtils.isEmpty(llmConfig.getUrl())) {
log.warn("llmConfig url is null, skip llm parser");
return null;
}
DomainInfos domainInfos = ContextUtils.getBean(WordNatureService.class).getCache().getUnchecked("");
Map<Integer, String> domainIdToName = domainInfos.getDomains().stream()
.collect(Collectors.toMap(ItemDO::getDomain, a -> a.getName(), (k1, k2) -> k1));
Map<Integer, String> itemIdToName = domainInfos.getDimensions().stream()
.filter(entry -> domainId.equals(entry.getDomain()))
.collect(Collectors.toMap(ItemDO::getItemId, ItemDO::getName, (value1, value2) -> value2));
String domainName = domainIdToName.get(domainId);
LLMReq llmReq = new LLMReq();
llmReq.setQueryText(queryContext.getQueryText());
List<SchemaElementMatch> matchedElements = queryContext.getMapInfo().getMatchedElements(domainId);
Set<String> fieldNameList = matchedElements.stream()
.filter(schemaElementMatch ->
SchemaElementType.METRIC.equals(schemaElementMatch.getElementType()) ||
SchemaElementType.DIMENSION.equals(schemaElementMatch.getElementType()) ||
SchemaElementType.VALUE.equals(schemaElementMatch.getElementType()))
.map(schemaElementMatch -> {
if (!SchemaElementType.VALUE.equals(schemaElementMatch.getElementType())) {
return schemaElementMatch.getWord();
}
return itemIdToName.get(schemaElementMatch.getElementID());
})
.filter(name -> StringUtils.isNotEmpty(name) && !name.contains("%"))
.collect(Collectors.toSet());
LLMSchema llmSchema = new LLMSchema();
llmSchema.setDomainName(domainName);
llmSchema.setFieldNameList(new ArrayList<>(fieldNameList));
llmReq.setSchema(llmSchema);
log.info("requestLLM request, domainId:{},llmReq:{}", domainId, llmReq);
String questUrl = llmConfig.getUrl() + llmConfig.getQueryToSqlPath();
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
HttpEntity<String> entity = new HttpEntity<>(JsonUtil.toString(llmReq), headers);
ResponseEntity<LLMResp> responseEntity = restTemplate.exchange(questUrl, HttpMethod.POST, entity,
LLMResp.class);
log.info("requestLLM response, questUrl:{} \n entity:{} \n body:{}", questUrl, entity,
responseEntity.getBody());
return responseEntity.getBody();
}
protected Integer getDomainId(QueryContextReq queryContext, ChatContext chatCtx) {
SchemaMapInfo mapInfo = queryContext.getMapInfo();
Set<Integer> matchedDomains = mapInfo.getMatchedDomains();
Map<Integer, SemanticQuery> domainQueryModes = new HashMap<>();
for (Integer matchedDomain : matchedDomains) {
domainQueryModes.put(matchedDomain, new LLMSemanticQuery());
}
List<DomainResolver> domainResolverList = SpringFactoriesLoader.loadFactories(DomainResolver.class,
Thread.currentThread().getContextClassLoader());
Optional<Integer> domainId = domainResolverList.stream()
.map(domainResolver -> domainResolver.resolve(domainQueryModes, queryContext, chatCtx,
queryContext.getMapInfo())).filter(d -> d > 0).findFirst();
if (domainId.isPresent()) {
return domainId.get();
}
return 0;
}
}

144
chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParser.java
View File

@@ -1,144 +0,0 @@
package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.application.query.MetricSemanticQuery;
import com.tencent.supersonic.chat.application.query.RuleSemanticQuery;
import com.tencent.supersonic.chat.application.query.RuleSemanticQueryManager;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.core.enums.TimeDimensionEnum;
import java.time.LocalDate;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.util.Strings;
public class TimeSemanticParser implements SemanticParser {
private static final Pattern recentPeriodPattern = Pattern.compile(
".*(?<periodStr>(近|过去)((?<enNum>\\d+)|(?<zhNum>[一二三四五六七八九十百千万亿]+))个?(?<zhPeriod>[天周月年])).*");
private int zhNumParse(String zhNumStr) {
Stack<Integer> stack = new Stack<>();
String numStr = "一二三四五六七八九";
String unitStr = "十百千万亿";
String[] ssArr = zhNumStr.split("");
for (String e : ssArr) {
int numIndex = numStr.indexOf(e);
int unitIndex = unitStr.indexOf(e);
if (numIndex != -1) {
stack.push(numIndex + 1);
} else if (unitIndex != -1) {
int unitNum = (int) Math.pow(10, unitIndex + 1);
if (stack.isEmpty()) {
stack.push(unitNum);
} else {
stack.push(stack.pop() * unitNum);
}
}
}
return stack.stream().mapToInt(s -> s).sum();
}
@Override
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
Matcher m = recentPeriodPattern.matcher(queryContext.getQueryText());
if (m.matches()) {
int num = 0;
String enNum = m.group("enNum");
String zhNum = m.group("zhNum");
if (enNum != null) {
num = Integer.parseInt(enNum);
} else if (zhNum != null) {
num = zhNumParse(zhNum);
}
if (num > 0) {
DateConf info = new DateConf();
String zhPeriod = m.group("zhPeriod");
int days;
switch (zhPeriod) {
case "周":
days = 7;
info.setPeriod(Constants.WEEK);
break;
case "月":
days = 30;
info.setPeriod(Constants.MONTH);
break;
case "年":
days = 365;
info.setPeriod(Constants.YEAR);
break;
default:
days = 1;
info.setPeriod(Constants.DAY);
}
days = days * num;
info.setDateMode(DateConf.DateMode.RECENT_UNITS);
String text = "近" + num + zhPeriod;
if (Strings.isNotEmpty(m.group("periodStr"))) {
text = m.group("periodStr");
}
info.setText(text);
info.setStartDate(LocalDate.now().minusDays(days).toString());
info.setUnit(num);
//queryContext.getParseInfo().setDateInfo(info);
for (SemanticQuery query : queryContext.getCandidateQueries()) {
if (query instanceof MetricSemanticQuery) {
query.getParseInfo().setDateInfo(info);
}
}
doParseOnlyTime(queryContext, chatContext, info);
}
}
}
protected void doParseOnlyTime(QueryContextReq queryContext, ChatContext chatContext, DateConf info) {
if (!queryContext.getCandidateQueries().isEmpty() || chatContext.getParseInfo() == null || Objects.isNull(
info.getText())) {
return;
}
if (info.getText().equals(queryContext.getQueryText()) && queryContext.getMapInfo().getDomainElementMatches()
.isEmpty()
) {
if (Objects.nonNull(chatContext.getParseInfo().getQueryMode()) && Objects.nonNull(
chatContext.getParseInfo().getDomainId()) && chatContext.getParseInfo().getDomainId() > 0) {
if (Objects.nonNull(chatContext.getParseInfo().getDateInfo()) && !chatContext.getParseInfo()
.getDateInfo().getPeriod().equals(info.getPeriod())) {
if (!CollectionUtils.isEmpty(chatContext.getParseInfo().getDimensions())) {
String dateField = TimeDimensionEnum.DAY.getName();
if (Constants.MONTH.equals(chatContext.getParseInfo().getDateInfo().getPeriod())) {
dateField = TimeDimensionEnum.MONTH.getName();
}
if (Constants.WEEK.equals(chatContext.getParseInfo().getDateInfo().getPeriod())) {
dateField = TimeDimensionEnum.WEEK.getName();
}
Set<SchemaItem> dimensions = new HashSet<>();
for (SchemaItem schemaItem : chatContext.getParseInfo().getDimensions()) {
if (schemaItem.getBizName().equals(dateField)) {
continue;
}
dimensions.add(schemaItem);
}
chatContext.getParseInfo().setDimensions(dimensions);
}
}
chatContext.getParseInfo().setDateInfo(info);
RuleSemanticQuery semanticQuery = RuleSemanticQueryManager.create(
chatContext.getParseInfo().getQueryMode());
semanticQuery.setParseInfo(chatContext.getParseInfo());
queryContext.getCandidateQueries().add(semanticQuery);
}
}
}
}

33
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityDetail.java
View File

@@ -1,33 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import org.springframework.stereotype.Component;
@Component
public class EntityDetail extends EntitySemanticQuery {
public static String QUERY_MODE = "ENTITY_DETAIL";
public EntityDetail() {
super();
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1)
.addOption(VALUE, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public void inheritContext(ChatContext chatContext) {
ContextHelper.addIfEmpty(chatContext.getParseInfo().getDimensionFilters(),
parseInfo.getDimensionFilters());
}
}

128
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java
View File

@@ -1,128 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
import static com.tencent.supersonic.common.constant.Constants.DAY;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
import com.tencent.supersonic.chat.domain.pojo.config.ChatDefaultRichConfig;
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
import com.tencent.supersonic.chat.domain.service.ConfigService;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class EntityListFilter extends EntitySemanticQuery {
public static String QUERY_MODE = "ENTITY_LIST_FILTER";
private static Long entityListLimit = 200L;
public EntityListFilter() {
super();
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1)
.addOption(ENTITY, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public void inheritContext(ChatContext chatContext) {
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
parseInfo.setLimit(entityListLimit);
this.fillDateEntityFilter(parseInfo);
this.addEntityDetailAndOrderByMetric(parseInfo);
this.dealNativeQuery(parseInfo, true);
}
private void fillDateEntityFilter(SemanticParseInfo semanticParseInfo) {
DateConf dateInfo = new DateConf();
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
dateInfo.setUnit(1);
dateInfo.setPeriod(DAY);
dateInfo.setText(String.format("近1天"));
semanticParseInfo.setDateInfo(dateInfo);
}
private void addEntityDetailAndOrderByMetric(SemanticParseInfo semanticParseInfo) {
if (semanticParseInfo.getDomainId() > 0L) {
ConfigService configService = ContextUtils.getBean(ConfigService.class);
ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(
semanticParseInfo.getDomainId());
if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null
&& chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) {
// SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
// EntityRichInfo entity = chaConfigRichDesc.getChatDetailRichConfig().getEntity();
Set<SchemaItem> dimensions = new LinkedHashSet();
// Set<String> primaryDimensions = this.addPrimaryDimension(entity, dimensions);
Set<SchemaItem> metrics = new LinkedHashSet();
Set<Order> orders = new LinkedHashSet();
ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
if (chatDefaultConfig != null) {
chatDefaultConfig.getMetrics().stream()
.forEach(metric -> {
metrics.add(metric);
orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER));
});
chatDefaultConfig.getDimensions().stream()
// .filter((m) -> !primaryDimensions.contains(m.getBizName()))
.forEach(dimension -> dimensions.add(dimension));
}
semanticParseInfo.setDimensions(dimensions);
semanticParseInfo.setMetrics(metrics);
semanticParseInfo.setOrders(orders);
}
}
}
private Set<String> addPrimaryDimension(EntityRichInfo entity, Set<SchemaItem> dimensions) {
Set<String> primaryDimensions = new HashSet();
DimSchemaResp dimItem = entity.getDimItem();
if (Objects.nonNull(entity) && Objects.nonNull(dimItem)) {
SchemaItem dimension = new SchemaItem();
BeanUtils.copyProperties(dimItem, dimension);
dimensions.add(dimension);
primaryDimensions.add(dimItem.getBizName());
return primaryDimensions;
} else {
return primaryDimensions;
}
}
private void dealNativeQuery(SemanticParseInfo semanticParseInfo, boolean isNativeQuery) {
if (Objects.nonNull(semanticParseInfo)) {
semanticParseInfo.setNativeQuery(isNativeQuery);
}
}
}

35
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListTopN.java
View File

@@ -1,35 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import org.springframework.stereotype.Component;
@Component
public class EntityListTopN extends EntitySemanticQuery {
public static String QUERY_MODE = "ENTITY_LIST_TOPN";
public EntityListTopN() {
super();
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1)
.setSupportOrderBy(true);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public void inheritContext(ChatContext chatContext) {
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
}
}

35
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityMetricFilter.java
View File

@@ -1,35 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import org.springframework.stereotype.Component;
@Component
public class EntityMetricFilter extends EntitySemanticQuery {
public static String QUERY_MODE = "ENTITY_METRIC_FILTER";
public EntityMetricFilter() {
super();
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1)
.addOption(VALUE, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public void inheritContext(ChatContext chatContext) {
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
}
}

13
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntitySemanticQuery.java
View File

@@ -1,13 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
public abstract class EntitySemanticQuery extends RuleSemanticQuery {
public EntitySemanticQuery() {
super();
queryMatcher.addOption(ENTITY, REQUIRED, AT_LEAST, 1);
}
}

71
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/LLMSemanticQuery.java
View File

@@ -1,71 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import com.tencent.supersonic.chat.application.DomainEntityService;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Slf4j
public class LLMSemanticQuery implements SemanticQuery {
public static String QUERY_MODE = "DSL";
private SemanticParseInfo semanticParse = new SemanticParseInfo();
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public QueryResultResp execute(User user) {
String queryMode = semanticParse.getQueryMode();
if (semanticParse.getDomainId() < 0 || StringUtils.isEmpty(queryMode)) {
// reach here some error may happen
log.error("not find QueryMode");
throw new RuntimeException("not find QueryMode");
}
QueryResultResp queryResponse = new QueryResultResp();
QueryResultWithSchemaResp queryResult = semanticLayer.queryBySql(
SchemaInfoConverter.convertToQuerySqlReq(semanticParse), user);
if (queryResult != null) {
queryResponse.setQueryAuthorization(queryResult.getQueryAuthorization());
}
String sql = queryResult == null ? null : queryResult.getSql();
List<Map<String, Object>> resultList = queryResult == null ? new ArrayList<>()
: queryResult.getResultList();
List<QueryColumn> columns = queryResult == null ? new ArrayList<>() : queryResult.getColumns();
queryResponse.setQuerySql(sql);
queryResponse.setQueryResults(resultList);
queryResponse.setQueryColumns(columns);
queryResponse.setQueryMode(queryMode);
// add domain info
EntityInfo entityInfo = ContextUtils.getBean(DomainEntityService.class)
.getEntityInfo(semanticParse, user);
queryResponse.setEntityInfo(entityInfo);
return queryResponse;
}
@Override
public SemanticParseInfo getParseInfo() {
return semanticParse;
}
}

120
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java
View File

@@ -1,120 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_MOST;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.OPTIONAL;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.Filter;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class MetricCompare extends MetricSemanticQuery {
public static String QUERY_MODE = "METRIC_COMPARE";
public static Pattern intentWordPattern = Pattern.compile("(?i)(比较|对比)");
public MetricCompare() {
super();
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 2)
.addOption(ENTITY, OPTIONAL, AT_MOST, 1);
queryMatcher.setSupportCompare(true);
queryMatcher.setSupportOrderBy(true);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches, QueryContextReq queryCtx) {
if (intentWordPattern.matcher(queryCtx.getQueryText()).find()) {
return super.match(candidateElementMatches, queryCtx);
} else {
return new ArrayList<>();
}
}
@Override
public void inheritContext(ChatContext chatContext) {
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
mergeAppend(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
addCompareDimension(parseInfo);
parseInfo.setBonus(2 * 1.0);
}
private void addCompareDimension(SemanticParseInfo semanticParseInfo) {
if (!semanticParseInfo.getDimensionFilters().isEmpty()) {
Set<String> dimensions = semanticParseInfo.getDimensions().stream().map(d -> d.getBizName()).collect(
Collectors.toSet());
log.info("addCompareDimension before [{}]", dimensions);
semanticParseInfo.getDimensionFilters().stream().filter(d -> d.getOperator().equals(FilterOperatorEnum.IN))
.forEach(
d -> {
if (!dimensions.contains(d.getBizName())) {
SchemaItem schemaItem = new SchemaItem();
schemaItem.setBizName(d.getBizName());
schemaItem.setId(d.getElementID());
semanticParseInfo.getDimensions().add(schemaItem);
dimensions.add(d.getBizName());
}
}
);
log.info("addCompareDimension after [{}]", dimensions);
}
}
private void mergeAppend(Set<Filter> from, Set<Filter> to) {
if (!from.isEmpty()) {
for (Filter filter : from) {
if (FilterOperatorEnum.EQUALS.equals(filter.getOperator()) || FilterOperatorEnum.IN.equals(
filter.getOperator())) {
Optional<Filter> toAdd = to.stream()
.filter(t -> t.getBizName().equalsIgnoreCase(filter.getBizName())).findFirst();
if (toAdd.isPresent()) {
if (FilterOperatorEnum.EQUALS.equals(toAdd.get().getOperator()) || FilterOperatorEnum.IN.equals(
toAdd.get().getOperator())) {
Set<Object> vals = new HashSet<>();
if (toAdd.get().getOperator().equals(FilterOperatorEnum.IN)) {
vals.addAll((List<Object>) (toAdd.get().getValue()));
} else {
vals.add(toAdd.get().getValue());
}
if (filter.getOperator().equals(FilterOperatorEnum.IN)) {
vals.addAll((List<Object>) (filter.getValue()));
} else {
vals.add(filter.getValue());
}
toAdd.get().setValue(new ArrayList<>(vals));
toAdd.get().setOperator(FilterOperatorEnum.IN);
continue;
}
}
}
to.add(filter);
}
}
}
}

35
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricDomain.java
View File

@@ -1,35 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import org.springframework.stereotype.Component;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
@Component
public class MetricDomain extends MetricSemanticQuery {
public static String QUERY_MODE = "METRIC_DOMAIN";
public MetricDomain() {
super();
queryMatcher.addOption(DOMAIN, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public void inheritContext(ChatContext chatContext) {
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
}
}

37
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricFilter.java
View File

@@ -1,37 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import org.springframework.stereotype.Component;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
@Component
public class MetricFilter extends MetricSemanticQuery {
public static String QUERY_MODE = "METRIC_FILTER";
public MetricFilter() {
super();
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1)
.addOption(ENTITY, OPTIONAL, AT_MOST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public void inheritContext(ChatContext chatContext) {
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
}
}

35
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricGroupBy.java
View File

@@ -1,35 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import org.springframework.stereotype.Component;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
@Component
public class MetricGroupBy extends MetricSemanticQuery {
public static String QUERY_MODE = "METRIC_GROUPBY";
public MetricGroupBy() {
super();
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public void inheritContext(ChatContext chatContext) {
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
ContextHelper.addIfEmpty(chatParseInfo.getDimensions(), parseInfo.getDimensions());
}
}

32
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricOrderBy.java
View File

@@ -1,32 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import org.springframework.stereotype.Component;
@Component
public class MetricOrderBy extends RuleSemanticQuery {
public static String QUERY_MODE = "METRIC_ORDERBY";
public MetricOrderBy() {
super();
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public void inheritContext(ChatContext chatContext) {
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
ContextHelper.addIfEmpty(chatParseInfo.getDimensions(), parseInfo.getDimensions());
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
}
}

13
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricSemanticQuery.java
View File

@@ -1,13 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
public abstract class MetricSemanticQuery extends RuleSemanticQuery {
public MetricSemanticQuery() {
super();
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1);
}
}

89
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/RuleSemanticQuery.java
View File

@@ -1,89 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import com.tencent.supersonic.chat.application.DomainEntityService;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
import java.util.*;
@Slf4j
@ToString
public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
protected SemanticParseInfo parseInfo = new SemanticParseInfo();
protected QueryMatcher queryMatcher = new QueryMatcher();
protected SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
public RuleSemanticQuery() {
RuleSemanticQueryManager.register(this);
}
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
QueryContextReq queryCtx) {
return queryMatcher.match(candidateElementMatches);
}
public abstract void inheritContext(ChatContext chatContext);
@Override
public QueryResultResp execute(User user) {
String queryMode = parseInfo.getQueryMode();
if (parseInfo.getDomainId() < 0 || StringUtils.isEmpty(queryMode)) {
// reach here some error may happen
log.error("not find QueryMode");
throw new RuntimeException("not find QueryMode");
}
List<String> semanticQueryModes = RuleSemanticQueryManager.getSemanticQueryModes();
if (!semanticQueryModes.contains(parseInfo.getQueryMode())) {
return null;
}
QueryResultResp queryResponse = new QueryResultResp();
QueryResultWithSchemaResp queryResult = semanticLayer.queryByStruct(
SchemaInfoConverter.convertTo(parseInfo), user);
if (queryResult != null) {
queryResponse.setQueryAuthorization(queryResult.getQueryAuthorization());
}
String sql = queryResult == null ? null : queryResult.getSql();
List<Map<String, Object>> resultList = queryResult == null ? new ArrayList<>()
: queryResult.getResultList();
List<QueryColumn> columns = queryResult == null ? new ArrayList<>() : queryResult.getColumns();
queryResponse.setQuerySql(sql);
queryResponse.setQueryResults(resultList);
queryResponse.setQueryColumns(columns);
queryResponse.setQueryMode(queryMode);
// add domain info
EntityInfo entityInfo = ContextUtils.getBean(DomainEntityService.class)
.getEntityInfo(parseInfo, user);
queryResponse.setEntityInfo(entityInfo);
return queryResponse;
}
@Override
public SemanticParseInfo getParseInfo() {
return parseInfo;
}
public void setParseInfo(SemanticParseInfo parseInfo) {
this.parseInfo = parseInfo;
}
}

39
chat/core/src/main/java/com/tencent/supersonic/chat/application/query/RuleSemanticQueryManager.java
View File

@@ -1,39 +0,0 @@
package com.tencent.supersonic.chat.application.query;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* RuleSemanticQueryManager
*/
public class RuleSemanticQueryManager {
private static Map<String, RuleSemanticQuery> semanticQueryMap = new ConcurrentHashMap<>();
public static RuleSemanticQuery create(String queryMode) {
RuleSemanticQuery semanticQuery = semanticQueryMap.get(queryMode);
if (Objects.isNull(semanticQuery)) {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
try {
return semanticQuery.getClass().getDeclaredConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
}
public static void register(RuleSemanticQuery query) {
semanticQueryMap.put(query.getQueryMode(), query);
}
public static List<RuleSemanticQuery> getSemanticQueries() {
return new ArrayList<>(semanticQueryMap.values());
}
public static List<String> getSemanticQueryModes() {
return new ArrayList<>(semanticQueryMap.keySet());
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatAggConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import lombok.Data;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatAggRichConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggRichConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import lombok.Data;

14
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfig.java
View File

@@ -1,11 +1,13 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.enums.StatusEnum;
import com.tencent.supersonic.common.util.RecordInfo;
import java.util.List;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.Data;
import lombok.ToString;
import java.util.List;
@Data
@ToString
public class ChatConfig {
@@ -27,6 +29,8 @@ public class ChatConfig {
*/
private ChatAggConfig chatAggConfig;
private List<RecommendedQuestion> recommendedQuestions;
/**
* available status
*/
@@ -37,4 +41,4 @@ public class ChatConfig {
*/
private RecordInfo recordInfo;
}
}

17
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatConfigBaseReq.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigBaseReq.java
View File

@@ -1,12 +1,13 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.enums.StatusEnum;
import java.util.List;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import lombok.ToString;
import java.util.List;
/**
* extended information command about domain
*/
@@ -26,9 +27,15 @@ public class ChatConfigBaseReq {
*/
private ChatAggConfig chatAggConfig;
/**
* the recommended questions about the domain
*/
private List<RecommendedQuestion> recommendedQuestions;
/**
* available status
*/
private StatusEnum status;
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatConfigEditReqReq.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigEditReqReq.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import lombok.Data;

4
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatConfigFilter.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigFilter.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data;
import lombok.NoArgsConstructor;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatConfigFilterInternal.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigFilterInternal.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import lombok.Data;

10
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatConfigResp.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigResp.java
View File

@@ -1,7 +1,11 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.enums.StatusEnum;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data
@@ -15,6 +19,8 @@ public class ChatConfigResp {
private ChatAggConfig chatAggConfig;
private List<RecommendedQuestion> recommendedQuestions;
/**
* available status
*/

12
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatConfigRichResp.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigRich.java
View File

@@ -1,12 +1,14 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.enums.StatusEnum;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data
public class ChatConfigRichResp {
public class ChatConfigRich {
private Long id;
@@ -19,6 +21,8 @@ public class ChatConfigRichResp {
private ChatDetailRichConfig chatDetailRichConfig;
private List<RecommendedQuestion> recommendedQuestions;
/**
* available status
*/
@@ -28,4 +32,4 @@ public class ChatConfigRichResp {
private String updatedBy;
private Date createdAt;
private Date updatedAt;
}
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatDefaultConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultConfig.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data;
import java.util.ArrayList;

11
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatDefaultRichConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultRichConfig.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data;
import java.util.List;
@@ -10,8 +10,9 @@ import java.util.List;
@Data
public class ChatDefaultRichConfig {
private List<SchemaItem> dimensions;
private List<SchemaItem> metrics;
private List<SchemaElement> dimensions;
private List<SchemaElement> metrics;
/**
* default time span unit

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatDetailConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import lombok.Data;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ChatDetailRichConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailRichConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import lombok.Data;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/DefaultMetric.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/DefaultMetric.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import lombok.AllArgsConstructor;
import lombok.Data;

4
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/DefaultMetricInfo.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/DefaultMetricInfo.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data;
import lombok.ToString;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/Dim4Dict.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/Dim4Dict.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import java.util.List;
import lombok.AllArgsConstructor;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/Entity.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/Entity.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import java.util.List;
import lombok.AllArgsConstructor;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/EntityDetailData.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/EntityDetailData.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import java.util.List;
import lombok.Data;

14
chat/core/src/main/java/com/tencent/supersonic/chat/config/EntityInternalDetail.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import java.util.List;
import lombok.Data;
@Data
public class EntityInternalDetail {
List<DimSchemaResp> dimensionList;
List<MetricSchemaResp> metricList;
}

16
chat/core/src/main/java/com/tencent/supersonic/chat/config/EntityRichInfo.java Normal file
View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import java.util.List;
import lombok.Data;
@Data
public class EntityRichInfo {
/**
* entity alias
*/
private List<String> names;
private SchemaElement dimItem;
}

12
chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallConfig.java Normal file
View File

@@ -0,0 +1,12 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@Configuration
@Data
public class FunctionCallConfig {
@Value("${functionCall.url:}")
private String url;
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ItemVisibility.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibility.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import java.util.ArrayList;
import java.util.List;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/ItemVisibilityInfo.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibilityInfo.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import java.util.List;
import lombok.Data;

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/KnowledgeAdvancedConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeAdvancedConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import lombok.Data;

7
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/KnowledgeInfo.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeInfo.java
View File

@@ -1,8 +1,7 @@
package com.tencent.supersonic.chat.domain.pojo.config;
package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.common.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import java.util.List;
import javax.validation.constraints.NotNull;
import lombok.Data;
@@ -35,4 +34,4 @@ public class KnowledgeInfo {
private KnowledgeAdvancedConfig knowledgeAdvancedConfig;
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/domain/config/LLMConfig.java → chat/core/src/main/java/com/tencent/supersonic/chat/config/LLMConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.domain.config;
package com.tencent.supersonic.chat.config;
import lombok.Data;

32
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/DomainInfos.java
View File

@@ -1,32 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.chat;
import com.tencent.supersonic.common.nlp.ItemDO;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
/**
* DomainInfos
*/
@Data
public class DomainInfos implements Serializable {
private List<ItemDO> domains = new ArrayList<>();
private List<ItemDO> dimensions = new ArrayList<>();
private List<ItemDO> metrics = new ArrayList<>();
private List<ItemDO> entities = new ArrayList<>();
public Map<Integer, String> getDomainToName() {
if (CollectionUtils.isEmpty(domains)) {
return new HashMap();
}
return domains.stream().collect(
Collectors.toMap(ItemDO::getDomain, ItemDO::getName, (value1, value2) -> value2)
);
}
}

11
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/LLMReq.java
View File

@@ -1,11 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.chat;
import lombok.Data;
@Data
public class LLMReq {
private String queryText;
private LLMSchema schema;
}

13
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/LLMSchema.java
View File

@@ -1,13 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.chat;
import java.util.List;
import lombok.Data;
@Data
public class LLMSchema {
private String domainName;
private List<String> fieldNameList;
}

29
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/QueryData.java
View File

@@ -1,29 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.chat;
import com.tencent.supersonic.chat.api.pojo.Filter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.SchemaItem;
import lombok.Data;
@Data
public class QueryData {
Long domainId = 0L;
Set<SchemaItem> metrics = new HashSet<>();
Set<SchemaItem> dimensions = new HashSet<>();
Set<Filter> dimensionFilters = new HashSet<>();
Set<Filter> metricFilters = new HashSet<>();
private Set<Order> orders = new HashSet<>();
private DateConf dateInfo;
private Long limit;
private Boolean nativeQuery = false;
}

48
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/RecommendResponse.java
View File

@@ -1,48 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.chat;
import java.io.Serializable;
import java.util.List;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
@Data
@Setter
@Getter
public class RecommendResponse {
private List<Item> dimensions;
private List<Item> metrics;
public static class Item implements Serializable {
private Integer domain;
private String name;
private String bizName;
public Integer getDomain() {
return domain;
}
public void setDomain(Integer domain) {
this.domain = domain;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getBizName() {
return bizName;
}
public void setBizName(String bizName) {
this.bizName = bizName;
}
}
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/chat/SchemaElementOption.java
View File

@@ -1,7 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.chat;
public enum SchemaElementOption {
REQUIRED,
OPTIONAL,
UNUSED
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/EntityInternalDetail.java
View File

@@ -1,14 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.config;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
import java.util.List;
import lombok.Data;
@Data
public class EntityInternalDetail {
List<DimSchemaResp> dimensionList;
List<MetricSchemaResp> metricList;
}

20
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/config/EntityRichInfo.java
View File

@@ -1,20 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.config;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import java.util.List;
import lombok.Data;
@Data
public class EntityRichInfo {
// private Long domainId;
// private String domainName;
// private String domainBizName;
/**
* entity alias
*/
private List<String> names;
private DimSchemaResp dimItem;
}

22
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/search/NameNature.java
View File

@@ -1,22 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.search;
import java.io.Serializable;
import java.util.List;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class NameNature implements Serializable {
private String name;
private List<String> natures;
public NameNature() {
}
public NameNature(String name, List<String> natures) {
this.name = name;
this.natures = natures;
}
}

18
chat/core/src/main/java/com/tencent/supersonic/chat/domain/pojo/search/QueryState.java
View File

@@ -1,18 +0,0 @@
package com.tencent.supersonic.chat.domain.pojo.search;
public enum QueryState {
NORMAL(0),
SEARCH_EXCEPTION(1),
EMPTY(2),
INVALID(3);
private int state;
QueryState(int state) {
this.state = state;
}
public int getState() {
return state;
}
}

20
chat/core/src/main/java/com/tencent/supersonic/chat/domain/repository/ChatQueryRepository.java
View File

@@ -1,20 +0,0 @@
package com.tencent.supersonic.chat.domain.repository;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.api.response.QueryResultResp;
import com.tencent.supersonic.chat.domain.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.domain.pojo.chat.ChatQueryVO;
import com.tencent.supersonic.chat.domain.pojo.chat.PageQueryInfoReq;
public interface ChatQueryRepository {
PageInfo<ChatQueryVO> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId);
void createChatQuery(QueryResultResp queryResponse, QueryContextReq queryContext, ChatContext chatCtx);
ChatQueryDO getLastChatQuery(long chatId);
int updateChatQuery(ChatQueryDO chatQueryDO);
}

26
chat/core/src/main/java/com/tencent/supersonic/chat/domain/service/ConfigService.java
View File

@@ -1,26 +0,0 @@
package com.tencent.supersonic.chat.domain.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigBaseReq;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigEditReqReq;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigFilter;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigResp;
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
import java.util.List;
public interface ConfigService {
Long addConfig(ChatConfigBaseReq extendBaseCmd, User user);
Long editConfig(ChatConfigEditReqReq extendEditCmd, User user);
List<ChatConfigResp> search(ChatConfigFilter filter, User user);
ChatConfigRichResp getConfigRichInfo(Long domainId);
ChatConfigResp fetchConfigByDomainId(Long domainId);
List<ChatConfigRichResp> getAllChatRichConfig();
}

Some files were not shown because too many files have changed in this diff Show More