[improvement][project] supersonic 0.7.2 version backend update (#28)

Co-authored-by: jipengli <jipengli@tencent.com>
This commit is contained in:
jipeli
2023-08-15 08:56:18 +08:00
committed by GitHub
parent 27283001a8
commit b1952d64ab
461 changed files with 18548 additions and 11939 deletions

View File

@@ -2,17 +2,18 @@ package com.tencent.supersonic.chat.api.component;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.DomainSchema;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.List;
/**
@@ -30,16 +31,22 @@ import java.util.List;
public interface SemanticLayer {
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user);
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user);
List<DomainSchema> getDomainSchema();
List<DomainSchema> getDomainSchema(List<Long> ids);
DomainSchema getDomainSchema(Long domain, Boolean cacheEnable);
List<ModelSchema> getModelSchema();
List<ModelSchema> getModelSchema(List<Long> ids);
ModelSchema getModelSchema(Long model, Boolean cacheEnable);
PageInfo<DimensionResp> getDimensionPage(PageDimensionReq pageDimensionCmd);
PageInfo<MetricResp> getMetricPage(PageMetricReq pageMetricCmd);
List<DomainResp> getDomainListForViewer();
List<DomainResp> getDomainListForAdmin();
List<DomainResp> getDomainList(User user);
List<ModelResp> getModelList(AuthType authType, Long domainId, User user);
}

View File

@@ -1,15 +1,14 @@
package com.tencent.supersonic.chat.api.pojo;
import lombok.Data;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import lombok.Data;
@Data
public class DomainSchema {
public class ModelSchema {
private SchemaElement domain;
private SchemaElement model;
private Set<SchemaElement> metrics = new HashSet<>();
private Set<SchemaElement> dimensions = new HashSet<>();
private Set<SchemaElement> dimensionValues = new HashSet<>();
@@ -22,8 +21,8 @@ public class DomainSchema {
case ENTITY:
element = Optional.ofNullable(entity);
break;
case DOMAIN:
element = Optional.of(domain);
case MODEL:
element = Optional.of(model);
break;
case METRIC:
element = metrics.stream().filter(e -> e.getId() == elementID).findFirst();

View File

@@ -2,12 +2,13 @@ package com.tencent.supersonic.chat.api.pojo;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class QueryContext {
private QueryReq request;

View File

@@ -1,11 +1,12 @@
package com.tencent.supersonic.chat.api.pojo;
import com.google.common.base.Objects;
import java.io.Serializable;
import java.util.List;
import lombok.*;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Data
@Getter
@@ -14,7 +15,7 @@ import lombok.*;
//@AllArgsConstructor
public class SchemaElement implements Serializable {
private Long domain;
private Long model;
private Long id;
private String name;
private String bizName;
@@ -25,9 +26,9 @@ public class SchemaElement implements Serializable {
// public SchemaElement() {
// }
public SchemaElement(Long domain, Long id, String name, String bizName,
Long useCnt, SchemaElementType type, List<String> alias) {
this.domain = domain;
public SchemaElement(Long model, Long id, String name, String bizName,
Long useCnt, SchemaElementType type, List<String> alias) {
this.model = model;
this.id = id;
this.name = name;
this.bizName = bizName;
@@ -45,7 +46,7 @@ public class SchemaElement implements Serializable {
return false;
}
SchemaElement schemaElement = (SchemaElement) o;
return Objects.equal(domain, schemaElement.domain) && Objects.equal(id,
return Objects.equal(model, schemaElement.model) && Objects.equal(id,
schemaElement.id) && Objects.equal(name, schemaElement.name)
&& Objects.equal(bizName, schemaElement.bizName) && Objects.equal(
useCnt, schemaElement.useCnt) && Objects.equal(type, schemaElement.type);
@@ -53,6 +54,6 @@ public class SchemaElement implements Serializable {
@Override
public int hashCode() {
return Objects.hashCode(domain, id, name, bizName, useCnt, type);
return Objects.hashCode(model, id, name, bizName, useCnt, type);
}
}

View File

@@ -18,10 +18,6 @@ public class SchemaElementMatch {
String detectWord;
String word;
Long frequency;
MatchMode mode = MatchMode.CURRENT;
boolean isInherited;
public enum MatchMode {
CURRENT,
INHERIT
}
}

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat.api.pojo;
public enum SchemaElementType {
DOMAIN,
MODEL,
METRIC,
DIMENSION,
VALUE,

View File

@@ -7,21 +7,21 @@ import java.util.Set;
public class SchemaMapInfo {
private Map<Long, List<SchemaElementMatch>> domainElementMatches = new HashMap<>();
private Map<Long, List<SchemaElementMatch>> modelElementMatches = new HashMap<>();
public Set<Long> getMatchedDomains() {
return domainElementMatches.keySet();
public Set<Long> getMatchedModels() {
return modelElementMatches.keySet();
}
public List<SchemaElementMatch> getMatchedElements(Long domain) {
return domainElementMatches.get(domain);
public List<SchemaElementMatch> getMatchedElements(Long model) {
return modelElementMatches.get(model);
}
public Map<Long, List<SchemaElementMatch>> getDomainElementMatches() {
return domainElementMatches;
public Map<Long, List<SchemaElementMatch>> getModelElementMatches() {
return modelElementMatches;
}
public void setMatchedElements(Long domain, List<SchemaElementMatch> elementMatches) {
domainElementMatches.put(domain, elementMatches);
public void setMatchedElements(Long model, List<SchemaElementMatch> elementMatches) {
modelElementMatches.put(model, elementMatches);
}
}

View File

@@ -1,19 +1,25 @@
package com.tencent.supersonic.chat.api.pojo;
import java.util.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import lombok.Data;
@Data
public class SemanticParseInfo {
private String queryMode;
private SchemaElement domain;
private SchemaElement model;
private Set<SchemaElement> metrics = new TreeSet<>(new SchemaNameLengthComparator());
private Set<SchemaElement> dimensions = new LinkedHashSet();
private SchemaElement entity;
@@ -28,15 +34,16 @@ public class SemanticParseInfo {
private List<SchemaElementMatch> elementMatches = new ArrayList<>();
private Map<String, Object> properties = new HashMap<>();
public Long getDomainId() {
return domain != null ? domain.getId() : 0L;
public Long getModelId() {
return model != null ? model.getId() : 0L;
}
public String getDomainName() {
return domain != null ? domain.getName() : "null";
public String getModelName() {
return model != null ? model.getName() : "null";
}
private static class SchemaNameLengthComparator implements Comparator<SchemaElement> {
@Override
public int compare(SchemaElement o1, SchemaElement o2) {
int len1 = o1.getName().length();
@@ -49,4 +56,11 @@ public class SemanticParseInfo {
}
}
public Set<SchemaElement> getMetrics() {
Set<SchemaElement> metricSet = new TreeSet<>(new SchemaNameLengthComparator());
metricSet.addAll(metrics);
metrics = metricSet;
return metrics;
}
}

View File

@@ -7,48 +7,49 @@ import java.util.Map;
import java.util.stream.Collectors;
public class SemanticSchema implements Serializable {
private List<DomainSchema> domainSchemaList;
public SemanticSchema(List<DomainSchema> domainSchemaList) {
this.domainSchemaList = domainSchemaList;
private List<ModelSchema> modelSchemaList;
public SemanticSchema(List<ModelSchema> modelSchemaList) {
this.modelSchemaList = modelSchemaList;
}
public void add(DomainSchema schema) {
domainSchemaList.add(schema);
public void add(ModelSchema schema) {
modelSchemaList.add(schema);
}
public Map<Long, String> getDomainIdToName() {
return domainSchemaList.stream()
.collect(Collectors.toMap(a -> a.getDomain().getId(), a -> a.getDomain().getName(), (k1, k2) -> k1));
public Map<Long, String> getModelIdToName() {
return modelSchemaList.stream()
.collect(Collectors.toMap(a -> a.getModel().getId(), a -> a.getModel().getName(), (k1, k2) -> k1));
}
public List<SchemaElement> getDimensionValues() {
List<SchemaElement> dimensionValues = new ArrayList<>();
domainSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues()));
modelSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues()));
return dimensionValues;
}
public List<SchemaElement> getDimensions() {
List<SchemaElement> dimensions = new ArrayList<>();
domainSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions()));
modelSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions()));
return dimensions;
}
public List<SchemaElement> getMetrics() {
List<SchemaElement> metrics = new ArrayList<>();
domainSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics()));
modelSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics()));
return metrics;
}
public List<SchemaElement> getDomains() {
List<SchemaElement> domains = new ArrayList<>();
domainSchemaList.stream().forEach(d -> domains.add(d.getDomain()));
return domains;
public List<SchemaElement> getModels() {
List<SchemaElement> models = new ArrayList<>();
modelSchemaList.stream().forEach(d -> models.add(d.getModel()));
return models;
}
public List<SchemaElement> getEntities() {
List<SchemaElement> entities = new ArrayList<>();
domainSchemaList.stream().forEach(d -> entities.add(d.getEntity()));
modelSchemaList.stream().forEach(d -> entities.add(d.getEntity()));
return entities;
}
}

View File

@@ -1,8 +1,7 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.List;
import lombok.Data;
@Data
public class ChatAggConfigReq {
@@ -13,7 +12,7 @@ public class ChatAggConfigReq {
private ItemVisibility visibility;
/**
* information about dictionary about the domain
* information about dictionary about the model
*/
private List<KnowledgeInfoReq> knowledgeInfos;

View File

@@ -1,34 +1,32 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.List;
import lombok.Data;
import lombok.ToString;
import java.util.List;
/**
* extended information command about domain
* extended information command about model
*/
@Data
@ToString
public class ChatConfigBaseReq {
private Long domainId;
private Long modelId;
/**
* the chatDetailConfig about the domain
* the chatDetailConfig about the model
*/
private ChatDetailConfigReq chatDetailConfig;
/**
* the chatAggConfig about the domain
* the chatAggConfig about the model
*/
private ChatAggConfigReq chatAggConfig;
/**
* the recommended questions about the domain
* the recommended questions about the model
*/
private List<RecommendedQuestionReq> recommendedQuestions;

View File

@@ -10,6 +10,6 @@ import lombok.NoArgsConstructor;
public class ChatConfigFilter {
private Long id;
private Long domainId;
private Long modelId;
private StatusEnum status = StatusEnum.ONLINE;
}

View File

@@ -2,10 +2,9 @@ package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class ChatDefaultConfigReq {

View File

@@ -1,8 +1,7 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.List;
import lombok.Data;
@Data
public class ChatDetailConfigReq {
@@ -13,7 +12,7 @@ public class ChatDetailConfigReq {
private ItemVisibility visibility;
/**
* information about dictionary about the domain
* information about dictionary about the model
*/
private List<KnowledgeInfoReq> knowledgeInfos;

View File

@@ -7,7 +7,7 @@ import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* the entity info about the domain
* the entity info about the model
*/
@Data
@AllArgsConstructor

View File

@@ -7,6 +7,7 @@ import lombok.Data;
@Data
public class ExecuteQueryReq {
private User user;
private Integer chatId;
private String queryText;

View File

@@ -1,9 +1,8 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
/**
* advanced knowledge config

View File

@@ -1,20 +1,18 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import javax.validation.constraints.NotNull;
import lombok.Data;
/**
* information about dictionary about the domain
* information about dictionary about the model
*/
@Data
public class KnowledgeInfoReq {
/**
* metricId、DimensionId、domainId
* metricId、DimensionId、modelId
*/
private Long itemId;

View File

@@ -13,7 +13,7 @@ public class PluginQueryReq {
private String type;
private String domain;
private String model;
private String pattern;

View File

@@ -1,18 +1,18 @@
package com.tencent.supersonic.chat.api.pojo.request;
import java.util.HashSet;
import java.util.Set;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import java.util.HashSet;
import java.util.Set;
import lombok.Data;
@Data
public class QueryDataReq {
String queryMode;
SchemaElement domain;
SchemaElement model;
Set<SchemaElement> metrics = new HashSet<>();
Set<SchemaElement> dimensions = new HashSet<>();
Set<QueryFilter> dimensionFilters = new HashSet<>();

View File

@@ -1,13 +1,14 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class QueryFilters {
private List<QueryFilter> filters = new ArrayList<>();
private Map<String, Object> params = new HashMap<>();
}

View File

@@ -5,9 +5,10 @@ import lombok.Data;
@Data
public class QueryReq {
private String queryText;
private Integer chatId;
private Long domainId = 0L;
private Long modelId = 0L;
private User user;
private QueryFilters queryFilters;
private boolean saveAnswer = true;

View File

@@ -6,5 +6,6 @@ import lombok.Data;
@Data
public class AggregateInfo {
private List<MetricInfo> metricInfos = new ArrayList<>();
private List<MetricInfo> metricInfos = new ArrayList<>();
}

View File

@@ -2,9 +2,8 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import lombok.Data;
import java.util.List;
import lombok.Data;
@Data
public class ChatAggRichConfigResp {
@@ -15,7 +14,7 @@ public class ChatAggRichConfigResp {
private ItemVisibilityInfo visibility;
/**
* information about dictionary about the domain
* information about dictionary about the model
*/
private List<KnowledgeInfoReq> knowledgeInfos;

View File

@@ -4,10 +4,8 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data
@@ -15,7 +13,7 @@ public class ChatConfigResp {
private Long id;
private Long domainId;
private Long modelId;
private ChatDetailConfigReq chatDetailConfig;

View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date;
import java.util.List;
import lombok.Data;
@Data
@@ -12,9 +11,9 @@ public class ChatConfigRichResp {
private Long id;
private Long domainId;
private Long modelId;
private String domainName;
private String modelName;
private String bizName;
private ChatAggRichConfigResp chatAggRichConfig;

View File

@@ -4,9 +4,8 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data;
import java.util.List;
import lombok.Data;
@Data
public class ChatDefaultRichConfigResp {

View File

@@ -2,9 +2,8 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import lombok.Data;
import java.util.List;
import lombok.Data;
@Data
public class ChatDetailRichConfigResp {
@@ -15,7 +14,7 @@ public class ChatDetailRichConfigResp {
private ItemVisibilityInfo visibility;
/**
* information about dictionary about the domain
* information about dictionary about the model
*/
private List<KnowledgeInfoReq> knowledgeInfos;

View File

@@ -7,7 +7,7 @@ import lombok.Data;
@Data
public class EntityInfo {
private DomainInfo domainInfo = new DomainInfo();
private ModelInfo modelInfo = new ModelInfo();
private List<DataInfo> dimensions = new ArrayList<>();
private List<DataInfo> metrics = new ArrayList<>();
private String entityId;

View File

@@ -1,14 +1,14 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import java.util.List;
import lombok.Data;
@Data
public class EntityRichInfoResp {
/**
* entity alias
* entity alias
*/
private List<String> names;

View File

@@ -5,7 +5,7 @@ import java.util.List;
import lombok.Data;
@Data
public class DomainInfo extends DataInfo implements Serializable {
public class ModelInfo extends DataInfo implements Serializable {
private List<String> words;
private String primaryEntityBizName;

View File

@@ -1,9 +1,12 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import lombok.*;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Data
@Getter
@@ -11,6 +14,7 @@ import java.util.List;
@NoArgsConstructor
@AllArgsConstructor
public class ParseResp {
private Integer chatId;
private String queryText;
private ParseState state;

View File

@@ -1,14 +1,14 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
@Data
@AllArgsConstructor
public class RecommendQuestionResp {
private Long domainId;
private Long modelId;
private List<RecommendedQuestionReq> recommendedQuestions;
}

View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import lombok.Data;
import java.util.List;
import lombok.Data;
@Data
public class RecommendResp {
private List<SchemaElement> dimensions;
private List<SchemaElement> metrics;
}

View File

@@ -2,8 +2,6 @@ package com.tencent.supersonic.chat.api.pojo.response;
import java.util.List;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
@Data
public class SearchResp {

View File

@@ -17,9 +17,9 @@ public class SearchResult {
private String subRecommend;
private String domainName;
private String modelName;
private Long domainId;
private Long modelId;
private SchemaElementType schemaElementType;
@@ -34,12 +34,12 @@ public class SearchResult {
return false;
}
SearchResult searchResult1 = (SearchResult) o;
return Objects.equals(recommend, searchResult1.recommend) && Objects.equals(domainName,
searchResult1.domainName);
return Objects.equals(recommend, searchResult1.recommend) && Objects.equals(modelName,
searchResult1.modelName);
}
@Override
public int hashCode() {
return Objects.hash(recommend, domainName);
return Objects.hash(recommend, modelName);
}
}