(improvement)(Chat) Move chat-core to headless (#805)

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-03-12 22:20:30 +08:00
committed by GitHub
parent f152deeb81
commit f93bee81cb
301 changed files with 2256 additions and 4527 deletions

View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class DataInfo {
private Integer itemId;
private String name;
private String bizName;
private String value;
}

View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
import java.io.Serializable;
import java.util.List;
@Data
public class DataSetInfo extends DataInfo implements Serializable {
private List<String> words;
private String primaryKey;
}

View File

@@ -0,0 +1,85 @@
package com.tencent.supersonic.headless.api.pojo;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import lombok.Data;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
@Data
public class DataSetSchema {
private SchemaElement dataSet;
private Set<SchemaElement> metrics = new HashSet<>();
private Set<SchemaElement> dimensions = new HashSet<>();
private Set<SchemaElement> dimensionValues = new HashSet<>();
private Set<SchemaElement> tags = new HashSet<>();
private Set<SchemaElement> tagValues = new HashSet<>();
private SchemaElement entity = new SchemaElement();
private QueryConfig queryConfig;
private QueryType queryType;
public SchemaElement getElement(SchemaElementType elementType, long elementID) {
Optional<SchemaElement> element = Optional.empty();
switch (elementType) {
case ENTITY:
element = Optional.ofNullable(entity);
break;
case DATASET:
element = Optional.of(dataSet);
break;
case METRIC:
element = metrics.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case DIMENSION:
element = dimensions.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case VALUE:
element = dimensionValues.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case TAG:
element = tags.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case TAG_VALUE:
element = tagValues.stream().filter(e -> e.getId() == elementID).findFirst();
break;
default:
}
if (element.isPresent()) {
return element.get();
} else {
return null;
}
}
public TimeDefaultConfig getTagTypeTimeDefaultConfig() {
if (queryConfig == null) {
return null;
}
if (queryConfig.getTagTypeDefaultConfig() == null) {
return null;
}
return queryConfig.getTagTypeDefaultConfig().getTimeDefaultConfig();
}
public TimeDefaultConfig getMetricTypeTimeDefaultConfig() {
if (queryConfig == null) {
return null;
}
if (queryConfig.getMetricTypeDefaultConfig() == null) {
return null;
}
return queryConfig.getMetricTypeDefaultConfig().getTimeDefaultConfig();
}
public TagTypeDefaultConfig getTagTypeDefaultConfig() {
if (queryConfig == null) {
return null;
}
return queryConfig.getTagTypeDefaultConfig();
}
}

View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class EntityInfo {
private DataSetInfo dataSetInfo = new DataSetInfo();
private List<DataInfo> dimensions = new ArrayList<>();
private List<DataInfo> metrics = new ArrayList<>();
private String entityId;
}

View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@ToString
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class SchemaElementMatch {
SchemaElement element;
double similarity;
String detectWord;
String word;
Long frequency;
boolean isInherited;
}

View File

@@ -0,0 +1,33 @@
package com.tencent.supersonic.headless.api.pojo;
import com.google.common.collect.Lists;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class SchemaMapInfo {
private Map<Long, List<SchemaElementMatch>> dataSetElementMatches = new HashMap<>();
public Set<Long> getMatchedDataSetInfos() {
return dataSetElementMatches.keySet();
}
public List<SchemaElementMatch> getMatchedElements(Long dataSet) {
return dataSetElementMatches.getOrDefault(dataSet, Lists.newArrayList());
}
public Map<Long, List<SchemaElementMatch>> getDataSetElementMatches() {
return dataSetElementMatches;
}
public void setDataSetElementMatches(Map<Long, List<SchemaElementMatch>> dataSetElementMatches) {
this.dataSetElementMatches = dataSetElementMatches;
}
public void setMatchedElements(Long dataSet, List<SchemaElementMatch> elementMatches) {
dataSetElementMatches.put(dataSet, elementMatches);
}
}

View File

@@ -0,0 +1,22 @@
package com.tencent.supersonic.headless.api.pojo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class SemanticCorrectInfo {
private QueryFilters queryFilters;
private SemanticParseInfo parseInfo;
private String sql;
private String preSql;
}

View File

@@ -0,0 +1,77 @@
package com.tencent.supersonic.headless.api.pojo;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.FilterType;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import lombok.Data;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@Data
public class SemanticParseInfo {
private Integer id;
private String queryMode;
private SchemaElement dataSet;
private Set<SchemaElement> metrics = new TreeSet<>(new SchemaNameLengthComparator());
private Set<SchemaElement> dimensions = new LinkedHashSet();
private SchemaElement entity;
private AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
private FilterType filterType = FilterType.UNION;
private Set<QueryFilter> dimensionFilters = new LinkedHashSet();
private Set<QueryFilter> metricFilters = new LinkedHashSet();
private Set<Order> orders = new LinkedHashSet();
private DateConf dateInfo;
private Long limit;
private double score;
private List<SchemaElementMatch> elementMatches = new ArrayList<>();
private Map<String, Object> properties = new HashMap<>();
private SqlInfo sqlInfo = new SqlInfo();
private QueryType queryType = QueryType.ID;
private EntityInfo entityInfo;
private static class SchemaNameLengthComparator implements Comparator<SchemaElement> {
@Override
public int compare(SchemaElement o1, SchemaElement o2) {
if (o1.getOrder() != o2.getOrder()) {
if (o1.getOrder() < o2.getOrder()) {
return -1;
} else {
return 1;
}
}
int len1 = o1.getName().length();
int len2 = o2.getName().length();
if (len1 != len2) {
return len1 - len2;
} else {
return o1.getName().compareTo(o2.getName());
}
}
}
public Set<SchemaElement> getMetrics() {
Set<SchemaElement> metricSet = new TreeSet<>(new SchemaNameLengthComparator());
metricSet.addAll(metrics);
metrics = metricSet;
return metrics;
}
public Long getDataSetId() {
if (dataSet == null) {
return null;
}
return dataSet.getDataSet();
}
}

View File

@@ -0,0 +1,170 @@
package com.tencent.supersonic.headless.api.pojo;
import org.springframework.util.CollectionUtils;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
public class SemanticSchema implements Serializable {
private List<DataSetSchema> dataSetSchemaList;
public SemanticSchema(List<DataSetSchema> dataSetSchemaList) {
this.dataSetSchemaList = dataSetSchemaList;
}
public void add(DataSetSchema schema) {
dataSetSchemaList.add(schema);
}
public SchemaElement getElement(SchemaElementType elementType, long elementID) {
Optional<SchemaElement> element = Optional.empty();
switch (elementType) {
case ENTITY:
element = getElementsById(elementID, getEntities());
break;
case DATASET:
element = getElementsById(elementID, getDataSets());
break;
case METRIC:
element = getElementsById(elementID, getMetrics());
break;
case DIMENSION:
element = getElementsById(elementID, getDimensions());
break;
case VALUE:
element = getElementsById(elementID, getDimensionValues());
break;
case TAG:
element = getElementsById(elementID, getTags());
break;
case TAG_VALUE:
element = getElementsById(elementID, getTagValues());
break;
default:
}
if (element.isPresent()) {
return element.get();
} else {
return null;
}
}
public Map<Long, String> getDataSetIdToName() {
return dataSetSchemaList.stream()
.collect(Collectors.toMap(a -> a.getDataSet().getId(), a -> a.getDataSet().getName(), (k1, k2) -> k1));
}
public List<SchemaElement> getDimensionValues() {
List<SchemaElement> dimensionValues = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> dimensionValues.addAll(d.getDimensionValues()));
return dimensionValues;
}
public List<SchemaElement> getDimensions() {
List<SchemaElement> dimensions = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> dimensions.addAll(d.getDimensions()));
return dimensions;
}
public List<SchemaElement> getDimensions(Long dataSetId) {
List<SchemaElement> dimensions = getDimensions();
return getElementsByDataSetId(dataSetId, dimensions);
}
public SchemaElement getDimension(Long id) {
List<SchemaElement> dimensions = getDimensions();
Optional<SchemaElement> dimension = getElementsById(id, dimensions);
return dimension.orElse(null);
}
public List<SchemaElement> getTags() {
List<SchemaElement> tags = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> tags.addAll(d.getTags()));
return tags;
}
public List<SchemaElement> getTags(Long dataSetId) {
List<SchemaElement> tags = getTags();
return getElementsByDataSetId(dataSetId, tags);
}
public List<SchemaElement> getTagValues() {
List<SchemaElement> tags = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> tags.addAll(d.getTagValues()));
return tags;
}
public List<SchemaElement> getTagValues(Long dataSetId) {
List<SchemaElement> tags = getTagValues();
return getElementsByDataSetId(dataSetId, tags);
}
public List<SchemaElement> getMetrics() {
List<SchemaElement> metrics = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> metrics.addAll(d.getMetrics()));
return metrics;
}
public List<SchemaElement> getMetrics(Long dataSetId) {
List<SchemaElement> metrics = getMetrics();
return getElementsByDataSetId(dataSetId, metrics);
}
public List<SchemaElement> getEntities() {
List<SchemaElement> entities = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> entities.add(d.getEntity()));
return entities;
}
public List<SchemaElement> getEntities(Long dataSetId) {
List<SchemaElement> entities = getEntities();
return getElementsByDataSetId(dataSetId, entities);
}
private List<SchemaElement> getElementsByDataSetId(Long dataSetId, List<SchemaElement> elements) {
return elements.stream()
.filter(schemaElement -> dataSetId.equals(schemaElement.getDataSet()))
.collect(Collectors.toList());
}
private Optional<SchemaElement> getElementsById(Long id, List<SchemaElement> elements) {
return elements.stream()
.filter(schemaElement -> id.equals(schemaElement.getId()))
.findFirst();
}
public SchemaElement getDataSet(Long dataSetId) {
List<SchemaElement> dataSets = getDataSets();
return getElementsById(dataSetId, dataSets).orElse(null);
}
public List<SchemaElement> getDataSets() {
List<SchemaElement> dataSets = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> dataSets.add(d.getDataSet()));
return dataSets;
}
public Map<String, String> getBizNameToName(Long dataSetId) {
List<SchemaElement> allElements = new ArrayList<>();
allElements.addAll(getDimensions(dataSetId));
allElements.addAll(getMetrics(dataSetId));
return allElements.stream()
.collect(Collectors.toMap(SchemaElement::getBizName, SchemaElement::getName, (k1, k2) -> k1));
}
public Map<Long, DataSetSchema> getDataSetSchemaMap() {
if (CollectionUtils.isEmpty(dataSetSchemaList)) {
return new HashMap<>();
}
return dataSetSchemaList.stream().collect(Collectors.toMap(dataSetSchema
-> dataSetSchema.getDataSet().getDataSet(), dataSetSchema -> dataSetSchema));
}
}

View File

@@ -0,0 +1,11 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
@Data
public class SqlInfo {
private String s2SQL;
private String correctS2SQL;
private String querySQL;
}

View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.headless.api.pojo.enums;
public enum CostType {
MAPPER(1, "mapper"),
PARSER(2, "parser"),
QUERY(3, "query"),
PROCESSOR(4, "processor");
private Integer type;
private String name;
CostType(Integer type, String name) {
this.type = type;
this.name = name;
}
public Integer getType() {
return type;
}
public String getName() {
return name;
}
}

View File

@@ -1,19 +1,19 @@
package com.tencent.supersonic.headless.api.pojo.enums;
public enum QueryType {
public enum QueryMethod {
SQL("SQL"),
STRUCT("STRUCT");
private String value;
QueryType(String value) {
QueryMethod(String value) {
this.value = value;
}
public static QueryType of(String src) {
for (QueryType operatorEnum : QueryType.values()) {
public static QueryMethod of(String src) {
for (QueryMethod operatorEnum : QueryMethod.values()) {
if (src.toUpperCase().contains(operatorEnum.value)) {
return operatorEnum;
}

View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import lombok.Builder;
import lombok.Data;
@Builder
@Data
public class ExecuteQueryReq {
private User user;
private Long queryId;
private Integer chatId;
private String queryText;
private SemanticParseInfo parseInfo;
private boolean saveAnswer;
}

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.tencent.supersonic.headless.api.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.enums.QueryMethod;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@@ -14,7 +14,7 @@ import lombok.ToString;
@NoArgsConstructor
public class ExplainSqlReq<T> {
private QueryType queryTypeEnum;
private QueryMethod queryTypeEnum;
private T queryReq;
}

View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import lombok.Data;
import java.util.HashSet;
import java.util.Set;
@Data
public class QueryDataReq {
private User user;
private Set<SchemaElement> metrics = new HashSet<>();
private Set<SchemaElement> dimensions = new HashSet<>();
private Set<QueryFilter> dimensionFilters = new HashSet<>();
private Set<QueryFilter> metricFilters = new HashSet<>();
private DateConf dateInfo;
private Long queryId;
private SemanticParseInfo parseInfo;
}

View File

@@ -0,0 +1,43 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.base.Objects;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import lombok.Data;
import lombok.ToString;
@Data
@ToString(callSuper = true)
public class QueryFilter {
private String bizName;
private String name;
private FilterOperatorEnum operator = FilterOperatorEnum.EQUALS;
private Object value;
private Long elementID;
private String function;
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
QueryFilter that = (QueryFilter) o;
return Objects.equal(bizName, that.bizName) && Objects.equal(name,
that.name) && operator == that.operator && Objects.equal(value, that.value)
&& Objects.equal(elementID, that.elementID) && Objects.equal(
function, that.function);
}
@Override
public int hashCode() {
return Objects.hashCode(bizName, name, operator, value, elementID, function);
}
}

View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.headless.api.pojo.request;
import lombok.Data;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Data
public class QueryFilters {
private List<QueryFilter> filters = new ArrayList<>();
private Map<String, Object> params = new HashMap<>();
}

View File

@@ -0,0 +1,17 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import lombok.Data;
import java.util.Set;
@Data
public class QueryReq {
private String queryText;
private Integer chatId;
private Set<Long> dataSetIds = Sets.newHashSet();
private User user;
private QueryFilters queryFilters;
private boolean saveAnswer = true;
}

View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import lombok.Data;
import java.util.List;
@Data
public class ParseResp {
private Integer chatId;
private String queryText;
private Long queryId;
private ParseState state;
private List<SemanticParseInfo> selectedParses = Lists.newArrayList();
private ParseTimeCostResp parseTimeCost = new ParseTimeCostResp();
public enum ParseState {
COMPLETED,
PENDING,
FAILED
}
}

View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.headless.api.pojo.response;
import lombok.Data;
@Data
public class ParseTimeCostResp {
private long parseStartTime;
private long parseTime;
private long sqlTime;
public ParseTimeCostResp() {
this.parseStartTime = System.currentTimeMillis();
}
}

View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import lombok.Data;
import java.util.Date;
import java.util.List;
@Data
public class QueryResp {
private Long questionId;
private Date createTime;
private Long chatId;
private Integer score;
private String feedback;
private String queryText;
private QueryResult queryResult;
private List<SemanticParseInfo> parseInfos;
}

View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
public class QueryResult {
private Long queryId;
private String queryMode;
private String querySql;
private QueryState queryState = QueryState.EMPTY;
private List<QueryColumn> queryColumns;
private QueryAuthorization queryAuthorization;
private SemanticParseInfo chatContext;
private Object response;
private List<Map<String, Object>> queryResults;
private Long queryTimeCost;
private EntityInfo entityInfo;
}

View File

@@ -0,0 +1,8 @@
package com.tencent.supersonic.headless.api.pojo.response;
public enum QueryState {
SUCCESS,
SEARCH_EXCEPTION,
EMPTY,
INVALID;
}

View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.headless.api.pojo.response;
import java.util.List;
import lombok.Data;
@Data
public class SearchResp {
private List<SearchResult> searchResults;
public SearchResp(List<SearchResult> searchResults) {
this.searchResults = searchResults;
}
}

View File

@@ -0,0 +1,46 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import java.util.Objects;
@Data
@Setter
@Getter
@Builder
public class SearchResult {
private String recommend;
private String subRecommend;
private String modelName;
private Long modelId;
private SchemaElementType schemaElementType;
private boolean isComplete = true;
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
SearchResult searchResult1 = (SearchResult) o;
return Objects.equals(recommend, searchResult1.recommend) && Objects.equals(modelName,
searchResult1.modelName);
}
@Override
public int hashCode() {
return Objects.hash(recommend, modelName);
}
}