[improvement][headless]Deprecate and remove entity-related abstraction and logic.#1876

This commit is contained in:
jerryjzhang
2024-11-04 00:55:07 +08:00
parent 6a4458a572
commit 1e5bf7909e
49 changed files with 61 additions and 1081 deletions

View File

@@ -4,7 +4,7 @@
# SuperSonic
SuperSonic is the next-generation BI platform that unifies **Chat BI** (powered by LLM) and **Headless BI** (powered by semantic layer) paradigms. This unification ensures that Chat BI has access to the same curated and governed semantic data models as traditional BI. Furthermore, the implementation of both paradigms benefit from each other:
SuperSonic is the next-generation AI+BI platform that unifies **Chat BI** (powered by LLM) and **Headless BI** (powered by semantic layer) paradigms. This unification ensures that Chat BI has access to the same curated and governed semantic data models as traditional BI. Furthermore, the implementation of both paradigms benefit from each other:
- Chat BI's Text2SQL gets augmented with context-retrieval from semantic models.
- Headless BI's query interface gets extended with natural language API.

View File

@@ -1,22 +0,0 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.List;
/** the entity info about the model */
@Data
@AllArgsConstructor
@ToString
@NoArgsConstructor
public class Entity {
/** uniquely identifies an entity */
private Long entityId;
/** entity name list */
private List<String> names;
}

View File

@@ -1,14 +0,0 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import lombok.Data;
import java.util.List;
@Data
public class EntityRichInfoResp {
/** entity alias */
private List<String> names;
private SchemaElement dimItem;
}

View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.headless.api.pojo.AggregateInfo;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
@@ -26,7 +25,6 @@ public class QueryResult {
private String textResult;
private String textSummary;
private Long queryTimeCost;
private EntityInfo entityInfo;
private List<SchemaElement> recommendedDimensions;
private AggregateInfo aggregateInfo;
private String errorMsg;

View File

@@ -100,6 +100,9 @@ public class NL2SQLParser implements ChatQueryParser {
queryNLReq.setMapModeEnum(MapModeEnum.LOOSE);
doParse(queryNLReq, parseResp);
}
if (parseResp.getSelectedParses().isEmpty()) {
return;
}
// for one dataset select the top 1 parse after sorting
SemanticParseInfo.sort(parseResp.getSelectedParses());
candidateParses.add(parseResp.getSelectedParses().get(0));

View File

@@ -5,9 +5,6 @@ import lombok.Data;
@Data
public class QueryDO {
public String aggregator = "trend";
public String startTime;
public String endTime;
private long id;
private long questionId;
private String createTime;
@@ -25,7 +22,6 @@ public class QueryDO {
private int topNum;
private String querySql;
private Object queryColumn;
private Object entityInfo;
private int score;
private String feedback;
}

View File

@@ -25,11 +25,9 @@ import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
@@ -198,7 +196,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
handleRuleQueryMode(semanticQuery, dataSetSchema, user);
}
return executeQuery(semanticQuery, user, dataSetSchema);
return executeQuery(semanticQuery, user);
}
private List<String> getFieldsFromSql(SemanticParseInfo parseInfo) {
@@ -236,15 +234,11 @@ public class ChatQueryServiceImpl implements ChatQueryService {
semanticQuery.initS2Sql(dataSetSchema, user);
}
private QueryResult executeQuery(SemanticQuery semanticQuery, User user,
DataSetSchema dataSetSchema) throws Exception {
private QueryResult executeQuery(SemanticQuery semanticQuery, User user) throws Exception {
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
QueryResult queryResult = doExecution(semanticQueryReq, parseInfo.getQueryMode(), user);
queryResult.setChatContext(semanticQuery.getParseInfo());
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
queryResult.setEntityInfo(entityInfo);
parseInfo.getSqlInfo().setQuerySQL(queryResult.getQuerySql());
return queryResult;
}

View File

@@ -7,7 +7,6 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.Entity;
import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo;
import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
@@ -16,7 +15,6 @@ import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ChatDetailRichConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp;
import com.tencent.supersonic.chat.api.pojo.response.ItemVisibilityInfo;
import com.tencent.supersonic.chat.server.config.ChatConfig;
import com.tencent.supersonic.chat.server.persistence.repository.ChatConfigRepository;
@@ -240,19 +238,6 @@ public class ConfigServiceImpl implements ConfigService {
return detailRichConfig;
}
private EntityRichInfoResp generateRichEntity(Entity entity, DataSetSchema modelSchema) {
EntityRichInfoResp entityRichInfo = new EntityRichInfoResp();
if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) {
return entityRichInfo;
}
BeanUtils.copyProperties(entity, entityRichInfo);
Map<Long, SchemaElement> dimIdAndRespPair = modelSchema.getDimensions().stream().collect(
Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
entityRichInfo.setDimItem(dimIdAndRespPair.get(entity.getEntityId()));
return entityRichInfo;
}
private ChatAggRichConfigResp fillChatAggRichConfig(DataSetSchema modelSchema,
ChatConfigResp chatConfigResp) {
if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) {

View File

@@ -5,9 +5,7 @@ public enum QueryType {
/** queries with aggregation (optionally slice and dice by dimensions) */
AGGREGATE,
/** queries with field selection */
DETAIL,
/** queries with ID-based entity selection */
ID;
DETAIL;
public boolean isNativeAggQuery() {
return DETAIL.equals(this);

View File

@@ -1,5 +1,5 @@
package com.tencent.supersonic.common.pojo.enums;
public enum TypeEnums {
METRIC, DIMENSION, TAG_OBJECT, TAG, DOMAIN, ENTITY, DATASET, MODEL, UNKNOWN
METRIC, DIMENSION, TAG_OBJECT, TAG, DOMAIN, DATASET, MODEL, UNKNOWN
}

View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
@@ -9,7 +8,6 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -31,9 +29,6 @@ public class DataSetSchema implements Serializable {
Optional<SchemaElement> element = Optional.empty();
switch (elementType) {
case ENTITY:
element = Optional.ofNullable(entity);
break;
case DATASET:
element = Optional.of(dataSet);
break;
@@ -55,11 +50,7 @@ public class DataSetSchema implements Serializable {
default:
}
if (element.isPresent()) {
return element.get();
} else {
return null;
}
return element.orElse(null);
}
public Map<String, String> getBizNameToName() {
@@ -70,7 +61,7 @@ public class DataSetSchema implements Serializable {
SchemaElement::getName, (k1, k2) -> k1));
}
public TimeDefaultConfig getTagTypeTimeDefaultConfig() {
public TimeDefaultConfig getDetailTypeTimeDefaultConfig() {
if (queryConfig == null) {
return null;
}
@@ -97,38 +88,6 @@ public class DataSetSchema implements Serializable {
return queryConfig.getDetailTypeDefaultConfig();
}
public List<SchemaElement> getTagDefaultDimensions() {
DetailTypeDefaultConfig detailTypeDefaultConfig = getTagTypeDefaultConfig();
if (Objects.isNull(detailTypeDefaultConfig)
|| Objects.isNull(detailTypeDefaultConfig.getDefaultDisplayInfo())) {
return new ArrayList<>();
}
if (CollectionUtils
.isNotEmpty(detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) {
return detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds().stream()
.map(id -> {
SchemaElement metric = getElement(SchemaElementType.METRIC, id);
return metric;
}).filter(Objects::nonNull).collect(Collectors.toList());
}
return new ArrayList<>();
}
public List<SchemaElement> getTagDefaultMetrics() {
DetailTypeDefaultConfig detailTypeDefaultConfig = getTagTypeDefaultConfig();
if (Objects.isNull(detailTypeDefaultConfig)
|| Objects.isNull(detailTypeDefaultConfig.getDefaultDisplayInfo())) {
return new ArrayList<>();
}
if (CollectionUtils
.isNotEmpty(detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) {
return detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> getElement(SchemaElementType.DIMENSION, id)).filter(Objects::nonNull)
.collect(Collectors.toList());
}
return new ArrayList<>();
}
public boolean containsPartitionDimensions() {
return dimensions.stream().anyMatch(SchemaElement::isPartitionTime);
}

View File

@@ -1,16 +0,0 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@Data
public class DefaultDisplayInfo implements Serializable {
// When displaying tag selection results, the information displayed by default
private List<Long> dimensionIds = new ArrayList<>();
private List<Long> metricIds = new ArrayList<>();
}

View File

@@ -8,8 +8,6 @@ import java.io.Serializable;
@Data
public class DetailTypeDefaultConfig implements Serializable {
private DefaultDisplayInfo defaultDisplayInfo;
// default time to filter tag selection results
private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig();

View File

@@ -1,21 +0,0 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.List;
@Data
@ToString
@AllArgsConstructor
@NoArgsConstructor
public class Entity {
/** uniquely identifies an entity */
private Long entityId;
/** entity name list */
private List<String> names;
}

View File

@@ -1,16 +0,0 @@
package com.tencent.supersonic.headless.api.pojo;
import lombok.Data;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@Data
public class EntityInfo implements Serializable {
private DataSetInfo dataSetInfo = new DataSetInfo();
private List<DataInfo> dimensions = new ArrayList<>();
private List<DataInfo> metrics = new ArrayList<>();
private String entityId;
}

View File

@@ -18,8 +18,6 @@ public class Identify {
private String bizName;
private List<String> entityNames;
private Integer isCreateDimension = 0;
public Identify(String name, String type, String bizName) {
@@ -28,13 +26,6 @@ public class Identify {
this.bizName = bizName;
}
public Identify(String name, String type, String bizName, Integer isCreateDimension) {
this.name = name;
this.type = type;
this.bizName = bizName;
this.isCreateDimension = isCreateDimension;
}
public String getFieldName() {
return bizName;
}

View File

@@ -1,5 +1,5 @@
package com.tencent.supersonic.headless.api.pojo;
public enum SchemaElementType {
DATASET, METRIC, DIMENSION, VALUE, ENTITY, ID, DATE, TAG, TERM
DATASET, METRIC, DIMENSION, VALUE, ID, DATE, TAG, TERM
}

View File

@@ -27,15 +27,17 @@ public class SemanticParseInfo implements Serializable {
private Integer id;
private String queryMode = "PLAIN_TEXT";
private SchemaElement dataSet;
private QueryConfig queryConfig;
private QueryType queryType = QueryType.DETAIL;
private SchemaElement dataSet;
private Set<SchemaElement> metrics = Sets.newTreeSet(new SchemaNameLengthComparator());
private Set<SchemaElement> dimensions = Sets.newTreeSet(new SchemaNameLengthComparator());
private SchemaElement entity;
private AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
private FilterType filterType = FilterType.AND;
private Set<QueryFilter> dimensionFilters = Sets.newHashSet();
private Set<QueryFilter> metricFilters = Sets.newHashSet();
private AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
private FilterType filterType = FilterType.AND;
private Set<Order> orders = Sets.newHashSet();
private DateConf dateInfo;
private long limit = DEFAULT_DETAIL_LIMIT;
@@ -43,8 +45,6 @@ public class SemanticParseInfo implements Serializable {
private List<SchemaElementMatch> elementMatches = Lists.newArrayList();
private SqlInfo sqlInfo = new SqlInfo();
private SqlEvaluation sqlEvaluation = new SqlEvaluation();
private QueryType queryType = QueryType.ID;
private EntityInfo entityInfo;
private String textInfo;
private Map<String, Object> properties = Maps.newHashMap();

View File

@@ -27,9 +27,6 @@ public class SemanticSchema implements Serializable {
Optional<SchemaElement> element = Optional.empty();
switch (elementType) {
case ENTITY:
element = getElementsById(elementID, getEntities());
break;
case DATASET:
element = getElementsById(elementID, getDataSets());
break;
@@ -97,17 +94,6 @@ public class SemanticSchema implements Serializable {
return getElementsByDataSetId(dataSetId, metrics);
}
public List<SchemaElement> getEntities() {
List<SchemaElement> entities = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> entities.add(d.getEntity()));
return entities;
}
public List<SchemaElement> getEntities(Long dataSetId) {
List<SchemaElement> entities = getEntities();
return getElementsByDataSetId(dataSetId, entities);
}
public List<SchemaElement> getTags() {
List<SchemaElement> tags = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> tags.addAll(d.getTags()));

View File

@@ -30,6 +30,6 @@ public class QueryDataSetReq {
private List<Filter> metricFilters = new ArrayList<>();
private DateConf dateInfo;
private Long limit = 2000L;
private QueryType queryType = QueryType.ID;
private QueryType queryType = QueryType.DETAIL;
private boolean innerLayerNative = false;
}

View File

@@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
@@ -42,6 +43,7 @@ import java.util.stream.Collectors;
@Slf4j
public class QueryStructReq extends SemanticQueryReq {
private List<SchemaElement> dimensions = new ArrayList<>();
private List<String> groups = new ArrayList<>();
private List<Aggregator> aggregators = new ArrayList<>();
private List<Order> orders = new ArrayList<>();
@@ -49,7 +51,7 @@ public class QueryStructReq extends SemanticQueryReq {
private List<Filter> metricFilters = new ArrayList<>();
private DateConf dateInfo;
private long limit = Constants.DEFAULT_DETAIL_LIMIT;
private QueryType queryType = QueryType.ID;
private QueryType queryType = QueryType.DETAIL;
private boolean convertToSql = true;
public List<String> getGroups() {

View File

@@ -19,19 +19,4 @@ public class DataSetSchemaResp extends DataSetResp {
private List<ModelResp> modelResps = Lists.newArrayList();
private List<TermResp> termResps = Lists.newArrayList();
public DimSchemaResp getPrimaryKey() {
for (ModelResp modelResp : modelResps) {
Identify identify = modelResp.getPrimaryIdentify();
if (identify == null) {
continue;
}
for (DimSchemaResp dimension : dimensions) {
if (identify.getBizName().equals(dimension.getBizName())) {
dimension.setEntityAlias(identify.getEntityNames());
return dimension;
}
}
}
return null;
}
}

View File

@@ -62,21 +62,6 @@ public class ModelResp extends SchemaItem {
return isOpen != null && isOpen == 1;
}
public Identify getPrimaryIdentify() {
if (modelDetail == null) {
return null;
}
if (CollectionUtils.isEmpty(modelDetail.getIdentifiers())) {
return null;
}
for (Identify identify : modelDetail.getIdentifiers()) {
if (!CollectionUtils.isEmpty(identify.getEntityNames())) {
return identify;
}
}
return null;
}
public List<Dim> getTimeDimension() {
if (modelDetail == null) {
return Lists.newArrayList();

View File

@@ -1,13 +1,9 @@
package com.tencent.supersonic.headless.chat.corrector;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.jsqlparser.SqlValidHelper;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import lombok.extern.slf4j.Slf4j;
@@ -18,9 +14,7 @@ import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
/** Perform SQL corrections on the "Select" section in S2SQL. */
@Slf4j
@@ -42,13 +36,11 @@ public class SelectCorrector extends BaseSemanticCorrector {
&& aggregateFields.size() == selectFields.size()) {
return;
}
correctS2SQL = addFieldsToSelect(chatQueryContext, semanticParseInfo, correctS2SQL);
correctS2SQL = addFieldsToSelect(semanticParseInfo, correctS2SQL);
semanticParseInfo.getSqlInfo().setCorrectedS2SQL(correctS2SQL);
}
protected String addFieldsToSelect(ChatQueryContext chatQueryContext,
SemanticParseInfo semanticParseInfo, String correctS2SQL) {
correctS2SQL = addTagDefaultFields(chatQueryContext, semanticParseInfo, correctS2SQL);
protected String addFieldsToSelect(SemanticParseInfo semanticParseInfo, String correctS2SQL) {
Set<String> selectFields = new HashSet<>(SqlSelectHelper.getSelectFields(correctS2SQL));
Set<String> needAddFields = new HashSet<>(SqlSelectHelper.getGroupByFields(correctS2SQL));
@@ -70,34 +62,4 @@ public class SelectCorrector extends BaseSemanticCorrector {
return addFieldsToSelectSql;
}
private String addTagDefaultFields(ChatQueryContext chatQueryContext,
SemanticParseInfo semanticParseInfo, String correctS2SQL) {
// If it is in DETAIL mode and select *, add default metrics and dimensions.
boolean hasAsterisk = SqlSelectFunctionHelper.hasAsterisk(correctS2SQL);
if (!(hasAsterisk && QueryType.DETAIL.equals(semanticParseInfo.getQueryType()))) {
return correctS2SQL;
}
Long dataSetId = semanticParseInfo.getDataSetId();
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
Set<String> needAddDefaultFields = new HashSet<>();
if (Objects.nonNull(dataSetSchema)) {
if (!CollectionUtils.isEmpty(dataSetSchema.getTagDefaultMetrics())) {
Set<String> metrics = dataSetSchema.getTagDefaultMetrics().stream()
.map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
needAddDefaultFields.addAll(metrics);
}
if (!CollectionUtils.isEmpty(dataSetSchema.getTagDefaultDimensions())) {
Set<String> dimensions = dataSetSchema.getTagDefaultDimensions().stream()
.map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
needAddDefaultFields.addAll(dimensions);
}
}
// remove * in sql and add default fields.
if (!CollectionUtils.isEmpty(needAddDefaultFields)) {
correctS2SQL =
SqlRemoveHelper.removeAsteriskAndAddFields(correctS2SQL, needAddDefaultFields);
}
return correctS2SQL;
}
}

View File

@@ -35,9 +35,6 @@ public class NatureHelper {
case DIMENSION:
result = SchemaElementType.DIMENSION;
break;
case ENTITY:
result = SchemaElementType.ENTITY;
break;
case DATASET:
result = SchemaElementType.DATASET;
break;

View File

@@ -1,77 +0,0 @@
package com.tencent.supersonic.headless.chat.mapper;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
/** A mapper capable of converting the VALUE of entity dimension values into ID types. */
@Slf4j
public class EntityMapper extends BaseMapper {
@Override
public void doMap(ChatQueryContext chatQueryContext) {
SchemaMapInfo schemaMapInfo = chatQueryContext.getMapInfo();
for (Long dataSetId : schemaMapInfo.getMatchedDataSetInfos()) {
List<SchemaElementMatch> schemaElementMatchList =
schemaMapInfo.getMatchedElements(dataSetId);
if (CollectionUtils.isEmpty(schemaElementMatchList)) {
continue;
}
SchemaElement entity = getEntity(dataSetId, chatQueryContext);
if (entity == null || entity.getId() == null) {
continue;
}
List<SchemaElementMatch> valueSchemaElements = schemaElementMatchList.stream()
.filter(schemaElementMatch -> SchemaElementType.VALUE
.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : valueSchemaElements) {
if (!entity.getId().equals(schemaElementMatch.getElement().getId())) {
continue;
}
if (!checkExistSameEntitySchemaElements(schemaElementMatch,
schemaElementMatchList)) {
SchemaElementMatch entitySchemaElementMath = new SchemaElementMatch();
BeanUtils.copyProperties(schemaElementMatch, entitySchemaElementMath);
entitySchemaElementMath.setElement(entity);
schemaElementMatchList.add(entitySchemaElementMath);
}
schemaElementMatch.getElement().setType(SchemaElementType.ID);
}
}
}
private boolean checkExistSameEntitySchemaElements(SchemaElementMatch valueSchemaElementMatch,
List<SchemaElementMatch> schemaElementMatchList) {
List<SchemaElementMatch> entitySchemaElements = schemaElementMatchList.stream()
.filter(schemaElementMatch -> SchemaElementType.ENTITY
.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : entitySchemaElements) {
if (schemaElementMatch.getElement().getId()
.equals(valueSchemaElementMatch.getElement().getId())) {
return true;
}
}
return false;
}
private SchemaElement getEntity(Long dataSetId, ChatQueryContext chatQueryContext) {
SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema();
DataSetSchema modelSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId);
if (modelSchema != null && modelSchema.getEntity() != null) {
return modelSchema.getEntity();
}
return null;
}
}

View File

@@ -94,9 +94,8 @@ public class MapFilter {
SchemaElement element = schemaElementMatch.getElement();
SchemaElementType type = element.getType();
boolean isEntityOrDatasetOrId = SchemaElementType.ENTITY.equals(type)
|| SchemaElementType.DATASET.equals(type)
|| SchemaElementType.ID.equals(type);
boolean isEntityOrDatasetOrId =
SchemaElementType.DATASET.equals(type) || SchemaElementType.ID.equals(type);
return !isEntityOrDatasetOrId && needRemovePredicate.test(element);
});

View File

@@ -1,29 +1,20 @@
package com.tencent.supersonic.headless.chat.parser;
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.SemanticQuery;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery;
import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
/** QueryTypeParser resolves query type as either AGGREGATE or DETAIL or ID. */
/** QueryTypeParser resolves query type as either AGGREGATE or DETAIL */
@Slf4j
public class QueryTypeParser implements SemanticParser {
@@ -52,22 +43,6 @@ public class QueryTypeParser implements SemanticParser {
return QueryType.DETAIL;
}
// 1. entity queryType
Long dataSetId = parseInfo.getDataSetId();
SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema();
if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) {
List<String> whereFields = SqlSelectHelper.getWhereFields(sqlInfo.getParsedS2SQL());
List<String> whereFilterByTimeFields = filterByTimeFields(whereFields);
if (CollectionUtils.isNotEmpty(whereFilterByTimeFields)) {
Set<String> ids = semanticSchema.getEntities(dataSetId).stream()
.map(SchemaElement::getName).collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(ids)
&& ids.stream().anyMatch(whereFilterByTimeFields::contains)) {
return QueryType.ID;
}
}
}
// 2. AGG queryType
if (SqlSelectFunctionHelper.hasAggregateFunction(sqlInfo.getParsedS2SQL())) {
return QueryType.AGGREGATE;
@@ -76,20 +51,4 @@ public class QueryTypeParser implements SemanticParser {
return QueryType.DETAIL;
}
private static List<String> filterByTimeFields(List<String> whereFields) {
return whereFields.stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
}
private static boolean selectContainsMetric(SqlInfo sqlInfo, Long dataSetId,
SemanticSchema semanticSchema) {
List<String> selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getParsedS2SQL());
List<SchemaElement> metrics = semanticSchema.getMetrics(dataSetId);
if (CollectionUtils.isNotEmpty(metrics)) {
Set<String> metricNameSet =
metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet());
return selectFields.stream().anyMatch(metricNameSet::contains);
}
return false;
}
}

View File

@@ -10,7 +10,6 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.BaseSemanticQuery;
@@ -124,18 +123,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
SchemaElement element = schemaMatch.getElement();
element.setOrder(1 - schemaMatch.getSimilarity());
switch (element.getType()) {
case ID:
SchemaElement entityElement =
semanticSchema.getElement(SchemaElementType.ENTITY, element.getId());
if (entityElement != null) {
if (id2Values.containsKey(element.getId())) {
id2Values.get(element.getId()).add(schemaMatch);
} else {
id2Values.put(element.getId(),
new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
break;
case VALUE:
SchemaElement dimElement =
semanticSchema.getElement(SchemaElementType.DIMENSION, element.getId());
@@ -154,13 +141,9 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
case METRIC:
parseInfo.getMetrics().add(element);
break;
case ENTITY:
parseInfo.setEntity(element);
break;
default:
}
}
addToFilters(id2Values, parseInfo, semanticSchema, SchemaElementType.ENTITY);
addToFilters(dim2Values, parseInfo, semanticSchema, SchemaElementType.DIMENSION);
}
@@ -182,8 +165,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
dimensionFilter.setElementID(schemaMatch.getElement().getId());
parseInfo.setEntity(
semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey()));
parseInfo.getDimensionFilters().add(dimensionFilter);
} else {
QueryFilter dimensionFilter = new QueryFilter();
@@ -216,11 +197,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
return convertQueryMultiStruct();
}
@Override
public void setParseInfo(SemanticParseInfo parseInfo) {
this.parseInfo = parseInfo;
}
public static List<RuleSemanticQuery> resolve(Long dataSetId,
List<SchemaElementMatch> candidateElementMatches, ChatQueryContext chatQueryContext) {
List<RuleSemanticQuery> matchedQueries = new ArrayList<>();
@@ -228,7 +204,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
List<SchemaElementMatch> matches =
semanticQuery.match(candidateElementMatches, chatQueryContext);
if (matches.size() > 0) {
if (!matches.isEmpty()) {
RuleSemanticQuery query =
QueryManager.createRuleQuery(semanticQuery.getQueryMode());
query.getParseInfo().getElementMatches().addAll(matches);
@@ -238,10 +214,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
return matchedQueries;
}
protected QueryStructReq convertQueryStruct() {
return QueryReqBuilder.buildStructReq(parseInfo);
}
protected QueryMultiStructReq convertQueryMultiStruct() {
return QueryReqBuilder.buildMultiStructReq(parseInfo);
}

View File

@@ -1,27 +0,0 @@
package com.tencent.supersonic.headless.chat.query.rule.detail;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
@Slf4j
@Component
public class DetailFilterQuery extends DetailListQuery {
public static final String QUERY_MODE = "DETAIL_LIST_FILTER";
public DetailFilterQuery() {
super();
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1);
queryMatcher.addOption(ENTITY, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
}

View File

@@ -1,23 +0,0 @@
package com.tencent.supersonic.headless.chat.query.rule.detail;
import org.springframework.stereotype.Component;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
@Component
public class DetailIdQuery extends DetailListQuery {
public static final String QUERY_MODE = "DETAIL_ID";
public DetailIdQuery() {
super();
queryMatcher.addOption(ID, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
}

View File

@@ -1,65 +0,0 @@
package com.tencent.supersonic.headless.chat.query.rule.detail;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.DetailTypeDefaultConfig;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import org.apache.commons.collections.CollectionUtils;
import java.util.LinkedHashSet;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
public abstract class DetailListQuery extends DetailSemanticQuery {
@Override
public void fillParseInfo(ChatQueryContext chatQueryContext) {
super.fillParseInfo(chatQueryContext);
this.addEntityDetailAndOrderByMetric(chatQueryContext, parseInfo);
}
private void addEntityDetailAndOrderByMetric(ChatQueryContext chatQueryContext,
SemanticParseInfo parseInfo) {
Long dataSetId = parseInfo.getDataSetId();
if (Objects.isNull(dataSetId) || dataSetId <= 0L) {
return;
}
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
if (dataSetSchema != null && Objects.nonNull(dataSetSchema.getEntity())) {
Set<SchemaElement> dimensions = new LinkedHashSet<>();
Set<SchemaElement> metrics = new LinkedHashSet<>();
Set<Order> orders = new LinkedHashSet<>();
DetailTypeDefaultConfig detailTypeDefaultConfig =
dataSetSchema.getTagTypeDefaultConfig();
if (detailTypeDefaultConfig != null
&& detailTypeDefaultConfig.getDefaultDisplayInfo() != null) {
if (CollectionUtils.isNotEmpty(
detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) {
metrics = detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds()
.stream().map(id -> {
SchemaElement metric =
dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (metric != null) {
orders.add(
new Order(metric.getBizName(), Constants.DESC_UPPER));
}
return metric;
}).filter(Objects::nonNull).collect(Collectors.toSet());
}
if (CollectionUtils.isNotEmpty(
detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) {
dimensions = detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds()
.stream()
.map(id -> dataSetSchema.getElement(SchemaElementType.DIMENSION, id))
.filter(Objects::nonNull).collect(Collectors.toSet());
}
}
parseInfo.setDimensions(dimensions);
parseInfo.setMetrics(metrics);
parseInfo.setOrders(orders);
}
}
}

View File

@@ -41,7 +41,7 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery {
Map<Long, DataSetSchema> dataSetSchemaMap =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap();
DataSetSchema dataSetSchema = dataSetSchemaMap.get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getDetailTypeTimeDefaultConfig();
if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {

View File

@@ -1,89 +0,0 @@
package com.tencent.supersonic.headless.chat.query.rule.metric;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
@Slf4j
@Component
public class MetricIdQuery extends MetricSemanticQuery {
public static final String QUERY_MODE = "METRIC_ID";
public MetricIdQuery() {
super();
queryMatcher.addOption(ID, REQUIRED, AT_LEAST, 1).addOption(ENTITY, REQUIRED, AT_LEAST, 1);
}
@Override
public String getQueryMode() {
return QUERY_MODE;
}
@Override
public SemanticQueryReq buildSemanticQueryReq() {
if (!isMultiStructQuery()) {
return super.buildSemanticQueryReq();
}
return super.multiStructExecute();
}
protected boolean isMultiStructQuery() {
Set<String> filterBizName = new HashSet<>();
parseInfo.getDimensionFilters().stream().filter(filter -> filter.getElementID() != null)
.forEach(filter -> filterBizName.add(filter.getBizName()));
return FilterType.UNION.equals(parseInfo.getFilterType()) && filterBizName.size() > 1;
}
@Override
protected QueryStructReq convertQueryStruct() {
QueryStructReq queryStructReq = super.convertQueryStruct();
addDimension(queryStructReq, true);
return queryStructReq;
}
@Override
protected QueryMultiStructReq convertQueryMultiStruct() {
QueryMultiStructReq queryMultiStructReq = super.convertQueryMultiStruct();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
addDimension(queryStructReq, false);
}
return queryMultiStructReq;
}
private void addDimension(QueryStructReq queryStructReq, boolean onlyOperateInFilter) {
if (!queryStructReq.getDimensionFilters().isEmpty()) {
List<String> dimensions = queryStructReq.getGroups();
log.info("addDimension before [{}]", queryStructReq.getGroups());
List<Filter> filters = new ArrayList<>(queryStructReq.getDimensionFilters());
if (onlyOperateInFilter) {
filters = filters.stream()
.filter(filter -> filter.getOperator().equals(FilterOperatorEnum.IN))
.collect(Collectors.toList());
}
filters.forEach(d -> {
if (!dimensions.contains(d.getBizName())) {
dimensions.add(d.getBizName());
}
});
queryStructReq.setGroups(dimensions);
log.info("addDimension after [{}]", queryStructReq.getGroups());
}
}
}

View File

@@ -1,108 +0,0 @@
package com.tencent.supersonic.headless.chat.corrector;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.DetailTypeDefaultConfig;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.core.env.Environment;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static org.mockito.Mockito.when;
class SelectCorrectorTest {
Long dataSetId = 2L;
@Test
void testDoCorrect() {
MockedStatic<ContextUtils> mocked = Mockito.mockStatic(ContextUtils.class);
Environment mockEnvironment = Mockito.mock(Environment.class);
mocked.when(() -> ContextUtils.getBean(Environment.class)).thenReturn(mockEnvironment);
when(mockEnvironment.getProperty(SelectCorrector.ADDITIONAL_INFORMATION)).thenReturn("");
BaseSemanticCorrector corrector = new SelectCorrector();
ChatQueryContext chatQueryContext = buildQueryContext(dataSetId);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
SchemaElement dataSet = new SchemaElement();
dataSet.setDataSetId(dataSetId);
semanticParseInfo.setDataSet(dataSet);
semanticParseInfo.setQueryType(QueryType.DETAIL);
SqlInfo sqlInfo = new SqlInfo();
String sql = "SELECT * FROM 艺人库 WHERE 艺人名='周杰伦'";
sqlInfo.setParsedS2SQL(sql);
sqlInfo.setCorrectedS2SQL(sql);
semanticParseInfo.setSqlInfo(sqlInfo);
corrector.correct(chatQueryContext, semanticParseInfo);
Assert.assertEquals("SELECT 粉丝数, 国籍, 艺人名, 性别 FROM 艺人库 WHERE 艺人名 = '周杰伦'",
semanticParseInfo.getSqlInfo().getCorrectedS2SQL());
}
private ChatQueryContext buildQueryContext(Long dataSetId) {
ChatQueryContext chatQueryContext = new ChatQueryContext();
List<DataSetSchema> dataSetSchemaList = new ArrayList<>();
DataSetSchema dataSetSchema = new DataSetSchema();
QueryConfig queryConfig = new QueryConfig();
DetailTypeDefaultConfig detailTypeDefaultConfig = new DetailTypeDefaultConfig();
DefaultDisplayInfo defaultDisplayInfo = new DefaultDisplayInfo();
List<Long> dimensionIds = new ArrayList<>();
dimensionIds.add(1L);
dimensionIds.add(2L);
dimensionIds.add(3L);
defaultDisplayInfo.setDimensionIds(dimensionIds);
List<Long> metricIds = new ArrayList<>();
metricIds.add(4L);
defaultDisplayInfo.setMetricIds(metricIds);
detailTypeDefaultConfig.setDefaultDisplayInfo(defaultDisplayInfo);
queryConfig.setDetailTypeDefaultConfig(detailTypeDefaultConfig);
dataSetSchema.setQueryConfig(queryConfig);
SchemaElement schemaElement = new SchemaElement();
schemaElement.setDataSetId(dataSetId);
dataSetSchema.setDataSet(schemaElement);
Set<SchemaElement> dimensions = new HashSet<>();
SchemaElement element1 = new SchemaElement();
element1.setDataSetId(dataSetId);
element1.setId(1L);
element1.setName("艺人名");
dimensions.add(element1);
SchemaElement element2 = new SchemaElement();
element2.setDataSetId(dataSetId);
element2.setId(2L);
element2.setName("性别");
dimensions.add(element2);
SchemaElement element3 = new SchemaElement();
element3.setDataSetId(dataSetId);
element3.setId(3L);
element3.setName("国籍");
dimensions.add(element3);
dataSetSchema.setDimensions(dimensions);
Set<SchemaElement> metrics = new HashSet<>();
SchemaElement metric1 = new SchemaElement();
metric1.setDataSetId(dataSetId);
metric1.setId(4L);
metric1.setName("粉丝数");
metrics.add(metric1);
dataSetSchema.setMetrics(metrics);
dataSetSchemaList.add(dataSetSchema);
SemanticSchema semanticSchema = new SemanticSchema(dataSetSchemaList);
chatQueryContext.setSemanticSchema(semanticSchema);
return chatQueryContext;
}
}

View File

@@ -1,13 +0,0 @@
package com.tencent.supersonic.headless.core.config;
import lombok.Data;
import java.util.List;
/** when query an entity, return related dimension/metric info */
@Data
public class EntityDetailData {
private List<Long> dimensionIds;
private List<Long> metricIds;
}

View File

@@ -1,14 +0,0 @@
package com.tencent.supersonic.headless.core.config;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import lombok.Data;
import java.util.List;
@Data
public class EntityInternalDetail {
List<DimSchemaResp> dimensionList;
List<MetricSchemaResp> metricList;
}

View File

@@ -2,9 +2,7 @@ package com.tencent.supersonic.headless.server.facade.service;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
@@ -18,15 +16,13 @@ import java.util.List;
/** This interface abstracts functionalities provided by a semantic layer. */
public interface SemanticLayerService {
DataSetSchema getDataSetSchema(Long id);
SemanticTranslateResp translate(SemanticQueryReq queryReq, User user) throws Exception;
SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;
SemanticQueryResp queryDimensionValue(DimensionValueReq dimensionValueReq, User user);
EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user);
DataSetSchema getDataSetSchema(Long id);
List<ItemResp> getDomainDataSetTree();

View File

@@ -2,18 +2,16 @@ package com.tencent.supersonic.headless.server.facade.service.impl;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.DetailTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
@@ -32,7 +30,6 @@ import com.tencent.supersonic.headless.chat.knowledge.MapResult;
import com.tencent.supersonic.headless.chat.knowledge.SearchService;
import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.headless.core.cache.QueryCache;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
@@ -54,13 +51,10 @@ import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -262,39 +256,6 @@ public class S2SemanticLayerService implements SemanticLayerService {
return dimensionResp;
}
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema,
User user) {
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
if (parseInfo.getDimensionFilters().size() <= 0
|| entityInfo.getDataSetInfo() == null) {
entityInfo.setMetrics(null);
entityInfo.setDimensions(null);
return entityInfo;
}
String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey();
if (StringUtils.isNotBlank(primaryKey)) {
String entityId = "";
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
if (chatFilter != null && chatFilter.getBizName() != null
&& chatFilter.getBizName().equals(primaryKey)) {
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
entityId = chatFilter.getValue().toString();
}
}
}
entityInfo.setEntityId(entityId);
try {
fillEntityInfoValue(entityInfo, dataSetSchema, user);
return entityInfo;
} catch (Exception e) {
log.error("setMainModel error", e);
}
}
}
return null;
}
@Override
public List<ItemResp> getDomainDataSetTree() {
return schemaService.getDomainDataSetTree();
@@ -305,31 +266,11 @@ public class S2SemanticLayerService implements SemanticLayerService {
return dimensionService.getDimensions(metaFilter);
}
private Set<SchemaElement> getDimensions(EntityInfo modelInfo) {
Set<SchemaElement> dimensions = new LinkedHashSet();
for (DataInfo mainEntityDimension : modelInfo.getDimensions()) {
SchemaElement dimension = new SchemaElement();
dimension.setBizName(mainEntityDimension.getBizName());
dimensions.add(dimension);
}
return dimensions;
}
@Override
public List<MetricResp> getMetrics(MetaFilter metaFilter) {
return metricService.getMetrics(metaFilter);
}
private Set<SchemaElement> getMetrics(EntityInfo modelInfo) {
Set<SchemaElement> metrics = Sets.newHashSet();
for (DataInfo metricValue : modelInfo.getMetrics()) {
SchemaElement metric = new SchemaElement();
BeanUtils.copyProperties(metricValue, metric);
metrics.add(metric);
}
return metrics;
}
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user)
throws Exception {
// If dataSetId or DataSetName is empty, parse dataSetId from the SQL
@@ -437,131 +378,4 @@ public class S2SemanticLayerService implements SemanticLayerService {
metricDrillDownChecker.checkQuery(queryStatement);
}
private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) {
EntityInfo entityInfo = new EntityInfo();
if (dataSetSchema == null) {
return entityInfo;
}
Long dataSetId = dataSetSchema.getDataSet().getDataSetId();
DataSetInfo dataSetInfo = new DataSetInfo();
dataSetInfo.setItemId(dataSetId.intValue());
dataSetInfo.setName(dataSetSchema.getDataSet().getName());
dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias());
dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName());
if (Objects.nonNull(dataSetSchema.getEntity())) {
dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName());
}
entityInfo.setDataSetInfo(dataSetInfo);
DetailTypeDefaultConfig detailTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig();
if (detailTypeDefaultConfig == null
|| detailTypeDefaultConfig.getDefaultDisplayInfo() == null) {
return entityInfo;
}
List<DataInfo> dimensions = detailTypeDefaultConfig.getDefaultDisplayInfo()
.getDimensionIds().stream().map(id -> {
SchemaElement element =
dataSetSchema.getElement(SchemaElementType.DIMENSION, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(),
element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
List<DataInfo> metrics = detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds()
.stream().map(id -> {
SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(),
element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
entityInfo.setDimensions(dimensions);
entityInfo.setMetrics(metrics);
return entityInfo;
}
private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema,
User user) {
SemanticQueryResp queryResultWithColumns =
getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user);
if (queryResultWithColumns != null) {
if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList())) {
Map<String, Object> result = queryResultWithColumns.getResultList().get(0);
for (Map.Entry<String, Object> entry : result.entrySet()) {
String entryKey = getEntryKey(entry);
if (entry.getValue() == null || entryKey == null) {
continue;
}
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
}
}
}
}
private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
DataSetSchema dataSetSchema, User user) {
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
semanticParseInfo.setQueryType(QueryType.DETAIL);
semanticParseInfo.setMetrics(getMetrics(entityInfo));
semanticParseInfo.setDimensions(getDimensions(entityInfo));
if (dataSetSchema.containsPartitionDimensions()) {
DateConf dateInfo = new DateConf();
int unit = 1;
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
if (Objects.nonNull(timeDefaultConfig)) {
unit = timeDefaultConfig.getUnit();
String date = LocalDate.now().minusDays(unit).toString();
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(date);
dateInfo.setEndDate(date);
} else {
dateInfo.setUnit(unit);
dateInfo.setDateMode(DateConf.DateMode.RECENT);
}
semanticParseInfo.setDateInfo(dateInfo);
}
// add filter
QueryFilter chatFilter = getQueryFilter(entityInfo);
Set<QueryFilter> chatFilters = Sets.newHashSet();
chatFilters.add(chatFilter);
semanticParseInfo.setDimensionFilters(chatFilters);
SemanticQueryResp queryResultWithColumns = null;
try {
QuerySqlReq querySqlReq = QueryReqBuilder.buildStructReq(semanticParseInfo).convert();
queryResultWithColumns = queryByReq(querySqlReq, user);
} catch (Exception e) {
log.warn("setMainModel queryByStruct error, e:", e);
}
return queryResultWithColumns;
}
private QueryFilter getQueryFilter(EntityInfo entityInfo) {
QueryFilter chatFilter = new QueryFilter();
chatFilter.setValue(entityInfo.getEntityId());
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
chatFilter.setBizName(getEntityPrimaryName(entityInfo));
return chatFilter;
}
private String getEntryKey(Map.Entry<String, Object> entry) {
// metric parser special handle, TODO delete
String entryKey = entry.getKey();
if (entryKey.contains("__")) {
entryKey = entryKey.split("__")[1];
}
return entryKey;
}
private String getEntityPrimaryName(EntityInfo entityInfo) {
return entityInfo.getDataSetInfo().getPrimaryKey();
}
}

View File

@@ -1,33 +0,0 @@
package com.tencent.supersonic.headless.server.processor;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.QueryManager;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
/**
* EntityInfoProcessor fills core attributes of an entity so that users get to know which entity is
* parsed out.
*/
public class EntityInfoProcessor implements ResultProcessor {
@Override
public void process(ParseResp parseResp, ChatQueryContext chatQueryContext) {
parseResp.getSelectedParses().forEach(parseInfo -> {
String queryMode = parseInfo.getQueryMode();
if (!QueryManager.isDetailQuery(queryMode) && !QueryManager.isMetricQuery(queryMode)) {
return;
}
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dataSetSchema =
semanticService.getDataSetSchema(parseInfo.getDataSetId());
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema,
chatQueryContext.getRequest().getUser());
parseInfo.setEntityInfo(entityInfo);
});
}
}

View File

@@ -58,7 +58,6 @@ public class DictWordService {
addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words);
addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words);
addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words);
addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words);
addWordsByType(DictWordType.TERM, semanticSchema.getTerms(), words);
return words;

View File

@@ -15,10 +15,7 @@ import com.tencent.supersonic.common.pojo.DataItem;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.enums.*;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Measure;
@@ -810,6 +807,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
queryStructReq.setDimensionFilters(filters);
// 7. set dateInfo
queryStructReq.setDateInfo(dateInfo);
queryStructReq.setQueryType(QueryType.AGGREGATE);
return queryStructReq;
}

View File

@@ -148,11 +148,6 @@ public class RetrieveServiceImpl implements RetrieveService {
Long dataSetId = NatureHelper.getDataSetId(nature);
SchemaElementType schemaElementType = NatureHelper.convertToElementType(nature);
// Skip if the schema element type is ENTITY
if (SchemaElementType.ENTITY.equals(schemaElementType)) {
return searchResults;
}
// Create a base search result
SearchResult baseSearchResult = createBaseSearchResult(dataSetId, dataSetIdToName,
matchText, wordName, schemaElementType);

View File

@@ -4,7 +4,15 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DimensionConstants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
@@ -17,7 +25,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -50,10 +57,6 @@ public class DataSetSchemaBuilder {
Set<SchemaElement> terms = getTerms(resp);
dataSetSchema.getTerms().addAll(terms);
SchemaElement entity = getEntity(resp);
if (Objects.nonNull(entity)) {
dataSetSchema.setEntity(entity);
}
return dataSetSchema;
}
@@ -99,17 +102,6 @@ public class DataSetSchemaBuilder {
return tags;
}
private static SchemaElement getEntity(DataSetSchemaResp resp) {
DimSchemaResp dim = resp.getPrimaryKey();
if (Objects.isNull(dim)) {
return null;
}
return SchemaElement.builder().dataSetId(resp.getId()).model(dim.getModelId())
.id(dim.getId()).name(dim.getName()).bizName(dim.getBizName())
.type(SchemaElementType.ENTITY).useCnt(dim.getUseCnt()).alias(dim.getEntityAlias())
.build();
}
private static Set<SchemaElement> getDimensions(DataSetSchemaResp resp) {
Set<SchemaElement> dimensions = new HashSet<>();
for (DimSchemaResp dim : resp.getDimensions()) {

View File

@@ -4,7 +4,6 @@ com.tencent.supersonic.headless.chat.mapper.SchemaMapper=\
com.tencent.supersonic.headless.chat.mapper.EmbeddingMapper, \
com.tencent.supersonic.headless.chat.mapper.KeywordMapper, \
com.tencent.supersonic.headless.chat.mapper.QueryFilterMapper, \
com.tencent.supersonic.headless.chat.mapper.EntityMapper, \
com.tencent.supersonic.headless.chat.mapper.TermDescMapper
com.tencent.supersonic.headless.chat.parser.SemanticParser=\
@@ -13,7 +12,8 @@ com.tencent.supersonic.headless.chat.parser.SemanticParser=\
com.tencent.supersonic.headless.chat.parser.QueryTypeParser
com.tencent.supersonic.headless.chat.corrector.SemanticCorrector=\
com.tencent.supersonic.headless.chat.corrector.RuleSqlCorrector
com.tencent.supersonic.headless.chat.corrector.RuleSqlCorrector,\
com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector
com.tencent.supersonic.headless.chat.knowledge.file.FileHandler=\
com.tencent.supersonic.headless.chat.knowledge.file.FileHandlerImpl
@@ -46,13 +46,7 @@ com.tencent.supersonic.headless.core.cache.QueryCache=\
### headless-server SPIs
com.tencent.supersonic.headless.server.processor.ResultProcessor=\
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor,\
com.tencent.supersonic.headless.server.processor.EntityInfoProcessor
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor
### auth-authentication SPIs
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor
com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor
com.tencent.supersonic.headless.server.modeller.SemanticModeller=\
com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller

View File

@@ -211,10 +211,6 @@ public class CspiderDemo extends S2BaseDemo {
tagTimeDefaultConfig.setTimeMode(TimeMode.LAST);
tagTimeDefaultConfig.setUnit(7);
detailTypeDefaultConfig.setTimeDefaultConfig(tagTimeDefaultConfig);
DefaultDisplayInfo defaultDisplayInfo = new DefaultDisplayInfo();
defaultDisplayInfo.setDimensionIds(Lists.newArrayList());
defaultDisplayInfo.setMetricIds(Lists.newArrayList());
detailTypeDefaultConfig.setDefaultDisplayInfo(defaultDisplayInfo);
AggregateTypeDefaultConfig aggregateTypeDefaultConfig = new AggregateTypeDefaultConfig();
TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig();
timeDefaultConfig.setTimeMode(TimeMode.RECENT);

View File

@@ -12,8 +12,15 @@ import com.tencent.supersonic.common.pojo.enums.AppModule;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.ChatAppManager;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.AggregateTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig;
import com.tencent.supersonic.headless.api.pojo.DetailTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
@@ -24,7 +31,6 @@ import com.tencent.supersonic.headless.api.pojo.request.TagObjectReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.TagObjectResp;
import lombok.extern.slf4j.Slf4j;
@@ -105,7 +111,6 @@ public class S2ArtistDemo extends S2BaseDemo {
ModelDetail modelDetail = new ModelDetail();
List<Identify> identifiers = new ArrayList<>();
Identify identify = new Identify("歌手名", IdentifyType.primary.name(), "singer_name", 1);
identify.setEntityNames(Lists.newArrayList("歌手"));
identifiers.add(identify);
modelDetail.setIdentifiers(identifiers);
@@ -152,11 +157,6 @@ public class S2ArtistDemo extends S2BaseDemo {
dataSetReq.setTypeEnum(TypeEnums.DATASET);
QueryConfig queryConfig = new QueryConfig();
DetailTypeDefaultConfig detailTypeDefaultConfig = new DetailTypeDefaultConfig();
DefaultDisplayInfo defaultDisplayInfo = new DefaultDisplayInfo();
defaultDisplayInfo.setDimensionIds(dataSetModelConfigs.get(0).getDimensions());
MetricResp jsPlayCntMetric = getMetric("js_play_cnt", singerModel);
defaultDisplayInfo.setMetricIds(Lists.newArrayList(jsPlayCntMetric.getId()));
detailTypeDefaultConfig.setDefaultDisplayInfo(defaultDisplayInfo);
AggregateTypeDefaultConfig aggregateTypeDefaultConfig = new AggregateTypeDefaultConfig();
queryConfig.setDetailTypeDefaultConfig(detailTypeDefaultConfig);
queryConfig.setAggregateTypeDefaultConfig(aggregateTypeDefaultConfig);

View File

@@ -4,7 +4,6 @@ com.tencent.supersonic.headless.chat.mapper.SchemaMapper=\
com.tencent.supersonic.headless.chat.mapper.EmbeddingMapper, \
com.tencent.supersonic.headless.chat.mapper.KeywordMapper, \
com.tencent.supersonic.headless.chat.mapper.QueryFilterMapper, \
com.tencent.supersonic.headless.chat.mapper.EntityMapper, \
com.tencent.supersonic.headless.chat.mapper.TermDescMapper
com.tencent.supersonic.headless.chat.parser.SemanticParser=\
@@ -47,8 +46,7 @@ com.tencent.supersonic.headless.core.cache.QueryCache=\
### headless-server SPIs
com.tencent.supersonic.headless.server.processor.ResultProcessor=\
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor,\
com.tencent.supersonic.headless.server.processor.EntityInfoProcessor
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor
com.tencent.supersonic.headless.server.modeller.SemanticModeller=\
com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller

View File

@@ -9,8 +9,6 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.chat.query.rule.detail.DetailDimensionQuery;
import com.tencent.supersonic.headless.chat.query.rule.detail.DetailFilterQuery;
import com.tencent.supersonic.headless.chat.query.rule.detail.DetailIdQuery;
import com.tencent.supersonic.util.DataUtils;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
@@ -44,53 +42,22 @@ public class DetailTest extends BaseTest {
}
@Test
public void test_detail_id() throws Exception {
QueryResult actualResult = submitNewChat("周杰伦", DataUtils.tagAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
expectedResult.setChatContext(expectedParseInfo);
expectedResult.setQueryMode(DetailIdQuery.QUERY_MODE);
expectedParseInfo.setQueryType(QueryType.DETAIL);
expectedParseInfo.setAggType(AggregateTypeEnum.NONE);
QueryFilter dimensionFilter =
DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 8L);
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
expectedParseInfo.getMetrics().add(SchemaElement.builder().name("播放量").build());
expectedParseInfo.getDimensions()
.addAll(Lists.newArrayList(SchemaElement.builder().name("歌手名").build(),
SchemaElement.builder().name("活跃区域").build(),
SchemaElement.builder().name("流派").build(),
SchemaElement.builder().name("代表作").build()));
assertQueryResult(expectedResult, actualResult);
}
@Test
public void test_detail_list_filter() throws Exception {
public void test_detail_filter() throws Exception {
QueryResult actualResult = submitNewChat("国风歌手", DataUtils.tagAgentId);
QueryResult expectedResult = new QueryResult();
SemanticParseInfo expectedParseInfo = new SemanticParseInfo();
expectedResult.setChatContext(expectedParseInfo);
expectedResult.setQueryMode(DetailFilterQuery.QUERY_MODE);
expectedResult.setQueryMode(DetailDimensionQuery.QUERY_MODE);
expectedParseInfo.setQueryType(QueryType.DETAIL);
expectedParseInfo.setAggType(AggregateTypeEnum.NONE);
QueryFilter dimensionFilter =
DataUtils.getFilter("genre", FilterOperatorEnum.EQUALS, "国风", "流派", 7L);
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
expectedParseInfo.getMetrics().add(SchemaElement.builder().name("播放量").build());
expectedParseInfo.getDimensions()
.addAll(Lists.newArrayList(SchemaElement.builder().name("歌手名").build(),
SchemaElement.builder().name("活跃区域").build(),
SchemaElement.builder().name("流派").build(),
SchemaElement.builder().name("代表作").build()));
.addAll(Lists.newArrayList(SchemaElement.builder().name("歌手名").build()));
assertQueryResult(expectedResult, actualResult);
}