diff --git a/README.md b/README.md index 1d1560e81..ce934e3b0 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # SuperSonic -SuperSonic is the next-generation BI platform that unifies **Chat BI** (powered by LLM) and **Headless BI** (powered by semantic layer) paradigms. This unification ensures that Chat BI has access to the same curated and governed semantic data models as traditional BI. Furthermore, the implementation of both paradigms benefit from each other: +SuperSonic is the next-generation AI+BI platform that unifies **Chat BI** (powered by LLM) and **Headless BI** (powered by semantic layer) paradigms. This unification ensures that Chat BI has access to the same curated and governed semantic data models as traditional BI. Furthermore, the implementation of both paradigms benefit from each other: - Chat BI's Text2SQL gets augmented with context-retrieval from semantic models. - Headless BI's query interface gets extended with natural language API. diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java deleted file mode 100644 index 30005d8fd..000000000 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.tencent.supersonic.chat.api.pojo.request; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.ToString; - -import java.util.List; - -/** the entity info about the model */ -@Data -@AllArgsConstructor -@ToString -@NoArgsConstructor -public class Entity { - - /** uniquely identifies an entity */ - private Long entityId; - - /** entity name list */ - private List names; -} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java deleted file mode 100644 index 3ed13d59b..000000000 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.tencent.supersonic.chat.api.pojo.response; - -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import lombok.Data; - -import java.util.List; - -@Data -public class EntityRichInfoResp { - /** entity alias */ - private List names; - - private SchemaElement dimItem; -} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java index 32dd0f4d4..7576d1f64 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.api.pojo.response; import com.tencent.supersonic.common.pojo.QueryAuthorization; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.headless.api.pojo.AggregateInfo; -import com.tencent.supersonic.headless.api.pojo.EntityInfo; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.response.QueryState; @@ -26,7 +25,6 @@ public class QueryResult { private String textResult; private String textSummary; private Long queryTimeCost; - private EntityInfo entityInfo; private List recommendedDimensions; private AggregateInfo aggregateInfo; private String errorMsg; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java index 631fd9595..b59853ba5 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java @@ -100,6 +100,9 @@ public class NL2SQLParser implements ChatQueryParser { queryNLReq.setMapModeEnum(MapModeEnum.LOOSE); doParse(queryNLReq, parseResp); } + if (parseResp.getSelectedParses().isEmpty()) { + return; + } // for one dataset select the top 1 parse after sorting SemanticParseInfo.sort(parseResp.getSelectedParses()); candidateParses.add(parseResp.getSelectedParses().get(0)); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/QueryDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/QueryDO.java index 46dc0a393..386a2783e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/QueryDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/QueryDO.java @@ -5,9 +5,6 @@ import lombok.Data; @Data public class QueryDO { - public String aggregator = "trend"; - public String startTime; - public String endTime; private long id; private long questionId; private String createTime; @@ -25,7 +22,6 @@ public class QueryDO { private int topNum; private String querySql; private Object queryColumn; - private Object entityInfo; private int score; private String feedback; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java index 78571dc68..113cf1140 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java @@ -25,11 +25,9 @@ import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.EntityInfo; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SqlInfo; @@ -198,7 +196,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { handleRuleQueryMode(semanticQuery, dataSetSchema, user); } - return executeQuery(semanticQuery, user, dataSetSchema); + return executeQuery(semanticQuery, user); } private List getFieldsFromSql(SemanticParseInfo parseInfo) { @@ -236,15 +234,11 @@ public class ChatQueryServiceImpl implements ChatQueryService { semanticQuery.initS2Sql(dataSetSchema, user); } - private QueryResult executeQuery(SemanticQuery semanticQuery, User user, - DataSetSchema dataSetSchema) throws Exception { + private QueryResult executeQuery(SemanticQuery semanticQuery, User user) throws Exception { SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq(); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); QueryResult queryResult = doExecution(semanticQueryReq, parseInfo.getQueryMode(), user); queryResult.setChatContext(semanticQuery.getParseInfo()); - SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class); - EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user); - queryResult.setEntityInfo(entityInfo); parseInfo.getSqlInfo().setQuerySQL(queryResult.getQuerySql()); return queryResult; } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java index 87ca561ba..a39a1849a 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java @@ -7,7 +7,6 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; -import com.tencent.supersonic.chat.api.pojo.request.Entity; import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo; import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; @@ -16,7 +15,6 @@ import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatDetailRichConfigResp; -import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp; import com.tencent.supersonic.chat.api.pojo.response.ItemVisibilityInfo; import com.tencent.supersonic.chat.server.config.ChatConfig; import com.tencent.supersonic.chat.server.persistence.repository.ChatConfigRepository; @@ -240,19 +238,6 @@ public class ConfigServiceImpl implements ConfigService { return detailRichConfig; } - private EntityRichInfoResp generateRichEntity(Entity entity, DataSetSchema modelSchema) { - EntityRichInfoResp entityRichInfo = new EntityRichInfoResp(); - if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) { - return entityRichInfo; - } - BeanUtils.copyProperties(entity, entityRichInfo); - Map dimIdAndRespPair = modelSchema.getDimensions().stream().collect( - Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1)); - - entityRichInfo.setDimItem(dimIdAndRespPair.get(entity.getEntityId())); - return entityRichInfo; - } - private ChatAggRichConfigResp fillChatAggRichConfig(DataSetSchema modelSchema, ChatConfigResp chatConfigResp) { if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java index 8a2d80a66..d98bd6d10 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/QueryType.java @@ -5,9 +5,7 @@ public enum QueryType { /** queries with aggregation (optionally slice and dice by dimensions) */ AGGREGATE, /** queries with field selection */ - DETAIL, - /** queries with ID-based entity selection */ - ID; + DETAIL; public boolean isNativeAggQuery() { return DETAIL.equals(this); diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java index 4d8b12e1e..a1fc12741 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java @@ -1,5 +1,5 @@ package com.tencent.supersonic.common.pojo.enums; public enum TypeEnums { - METRIC, DIMENSION, TAG_OBJECT, TAG, DOMAIN, ENTITY, DATASET, MODEL, UNKNOWN + METRIC, DIMENSION, TAG_OBJECT, TAG, DOMAIN, DATASET, MODEL, UNKNOWN } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java index cc26aa74c..730dc00d1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.api.pojo; import lombok.Data; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.io.Serializable; @@ -9,7 +8,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -31,9 +29,6 @@ public class DataSetSchema implements Serializable { Optional element = Optional.empty(); switch (elementType) { - case ENTITY: - element = Optional.ofNullable(entity); - break; case DATASET: element = Optional.of(dataSet); break; @@ -55,11 +50,7 @@ public class DataSetSchema implements Serializable { default: } - if (element.isPresent()) { - return element.get(); - } else { - return null; - } + return element.orElse(null); } public Map getBizNameToName() { @@ -70,7 +61,7 @@ public class DataSetSchema implements Serializable { SchemaElement::getName, (k1, k2) -> k1)); } - public TimeDefaultConfig getTagTypeTimeDefaultConfig() { + public TimeDefaultConfig getDetailTypeTimeDefaultConfig() { if (queryConfig == null) { return null; } @@ -97,38 +88,6 @@ public class DataSetSchema implements Serializable { return queryConfig.getDetailTypeDefaultConfig(); } - public List getTagDefaultDimensions() { - DetailTypeDefaultConfig detailTypeDefaultConfig = getTagTypeDefaultConfig(); - if (Objects.isNull(detailTypeDefaultConfig) - || Objects.isNull(detailTypeDefaultConfig.getDefaultDisplayInfo())) { - return new ArrayList<>(); - } - if (CollectionUtils - .isNotEmpty(detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) { - return detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds().stream() - .map(id -> { - SchemaElement metric = getElement(SchemaElementType.METRIC, id); - return metric; - }).filter(Objects::nonNull).collect(Collectors.toList()); - } - return new ArrayList<>(); - } - - public List getTagDefaultMetrics() { - DetailTypeDefaultConfig detailTypeDefaultConfig = getTagTypeDefaultConfig(); - if (Objects.isNull(detailTypeDefaultConfig) - || Objects.isNull(detailTypeDefaultConfig.getDefaultDisplayInfo())) { - return new ArrayList<>(); - } - if (CollectionUtils - .isNotEmpty(detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) { - return detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream() - .map(id -> getElement(SchemaElementType.DIMENSION, id)).filter(Objects::nonNull) - .collect(Collectors.toList()); - } - return new ArrayList<>(); - } - public boolean containsPartitionDimensions() { return dimensions.stream().anyMatch(SchemaElement::isPartitionTime); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java deleted file mode 100644 index ef2d8e2e1..000000000 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DefaultDisplayInfo.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.tencent.supersonic.headless.api.pojo; - -import lombok.Data; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -@Data -public class DefaultDisplayInfo implements Serializable { - - // When displaying tag selection results, the information displayed by default - private List dimensionIds = new ArrayList<>(); - - private List metricIds = new ArrayList<>(); -} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DetailTypeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DetailTypeDefaultConfig.java index 7977db800..eccc7b742 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DetailTypeDefaultConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DetailTypeDefaultConfig.java @@ -8,8 +8,6 @@ import java.io.Serializable; @Data public class DetailTypeDefaultConfig implements Serializable { - private DefaultDisplayInfo defaultDisplayInfo; - // default time to filter tag selection results private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Entity.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Entity.java deleted file mode 100644 index c5fc5b333..000000000 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Entity.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.tencent.supersonic.headless.api.pojo; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.ToString; - -import java.util.List; - -@Data -@ToString -@AllArgsConstructor -@NoArgsConstructor -public class Entity { - - /** uniquely identifies an entity */ - private Long entityId; - - /** entity name list */ - private List names; -} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java deleted file mode 100644 index 3144752c6..000000000 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/EntityInfo.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.tencent.supersonic.headless.api.pojo; - -import lombok.Data; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -@Data -public class EntityInfo implements Serializable { - - private DataSetInfo dataSetInfo = new DataSetInfo(); - private List dimensions = new ArrayList<>(); - private List metrics = new ArrayList<>(); - private String entityId; -} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java index ed3667af4..2b592e334 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Identify.java @@ -18,8 +18,6 @@ public class Identify { private String bizName; - private List entityNames; - private Integer isCreateDimension = 0; public Identify(String name, String type, String bizName) { @@ -28,13 +26,6 @@ public class Identify { this.bizName = bizName; } - public Identify(String name, String type, String bizName, Integer isCreateDimension) { - this.name = name; - this.type = type; - this.bizName = bizName; - this.isCreateDimension = isCreateDimension; - } - public String getFieldName() { return bizName; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java index 1c005d39c..6cd988f2f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementType.java @@ -1,5 +1,5 @@ package com.tencent.supersonic.headless.api.pojo; public enum SchemaElementType { - DATASET, METRIC, DIMENSION, VALUE, ENTITY, ID, DATE, TAG, TERM + DATASET, METRIC, DIMENSION, VALUE, ID, DATE, TAG, TERM } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java index 92e33d5c7..ba82ae54a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java @@ -27,15 +27,17 @@ public class SemanticParseInfo implements Serializable { private Integer id; private String queryMode = "PLAIN_TEXT"; - private SchemaElement dataSet; private QueryConfig queryConfig; + private QueryType queryType = QueryType.DETAIL; + + private SchemaElement dataSet; private Set metrics = Sets.newTreeSet(new SchemaNameLengthComparator()); private Set dimensions = Sets.newTreeSet(new SchemaNameLengthComparator()); - private SchemaElement entity; - private AggregateTypeEnum aggType = AggregateTypeEnum.NONE; - private FilterType filterType = FilterType.AND; private Set dimensionFilters = Sets.newHashSet(); private Set metricFilters = Sets.newHashSet(); + + private AggregateTypeEnum aggType = AggregateTypeEnum.NONE; + private FilterType filterType = FilterType.AND; private Set orders = Sets.newHashSet(); private DateConf dateInfo; private long limit = DEFAULT_DETAIL_LIMIT; @@ -43,8 +45,6 @@ public class SemanticParseInfo implements Serializable { private List elementMatches = Lists.newArrayList(); private SqlInfo sqlInfo = new SqlInfo(); private SqlEvaluation sqlEvaluation = new SqlEvaluation(); - private QueryType queryType = QueryType.ID; - private EntityInfo entityInfo; private String textInfo; private Map properties = Maps.newHashMap(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java index 00f7aef8d..6e9aeb1da 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java @@ -27,9 +27,6 @@ public class SemanticSchema implements Serializable { Optional element = Optional.empty(); switch (elementType) { - case ENTITY: - element = getElementsById(elementID, getEntities()); - break; case DATASET: element = getElementsById(elementID, getDataSets()); break; @@ -97,17 +94,6 @@ public class SemanticSchema implements Serializable { return getElementsByDataSetId(dataSetId, metrics); } - public List getEntities() { - List entities = new ArrayList<>(); - dataSetSchemaList.stream().forEach(d -> entities.add(d.getEntity())); - return entities; - } - - public List getEntities(Long dataSetId) { - List entities = getEntities(); - return getElementsByDataSetId(dataSetId, entities); - } - public List getTags() { List tags = new ArrayList<>(); dataSetSchemaList.stream().forEach(d -> tags.addAll(d.getTags())); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java index bda9f7a83..f385c73fc 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryDataSetReq.java @@ -30,6 +30,6 @@ public class QueryDataSetReq { private List metricFilters = new ArrayList<>(); private DateConf dateInfo; private Long limit = 2000L; - private QueryType queryType = QueryType.ID; + private QueryType queryType = QueryType.DETAIL; private boolean innerLayerNative = false; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 35cddca71..9ddc4e0d7 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import lombok.Data; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; @@ -42,6 +43,7 @@ import java.util.stream.Collectors; @Slf4j public class QueryStructReq extends SemanticQueryReq { + private List dimensions = new ArrayList<>(); private List groups = new ArrayList<>(); private List aggregators = new ArrayList<>(); private List orders = new ArrayList<>(); @@ -49,7 +51,7 @@ public class QueryStructReq extends SemanticQueryReq { private List metricFilters = new ArrayList<>(); private DateConf dateInfo; private long limit = Constants.DEFAULT_DETAIL_LIMIT; - private QueryType queryType = QueryType.ID; + private QueryType queryType = QueryType.DETAIL; private boolean convertToSql = true; public List getGroups() { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java index 322dc9ede..e1f373224 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetSchemaResp.java @@ -19,19 +19,4 @@ public class DataSetSchemaResp extends DataSetResp { private List modelResps = Lists.newArrayList(); private List termResps = Lists.newArrayList(); - public DimSchemaResp getPrimaryKey() { - for (ModelResp modelResp : modelResps) { - Identify identify = modelResp.getPrimaryIdentify(); - if (identify == null) { - continue; - } - for (DimSchemaResp dimension : dimensions) { - if (identify.getBizName().equals(dimension.getBizName())) { - dimension.setEntityAlias(identify.getEntityNames()); - return dimension; - } - } - } - return null; - } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java index 3677f5ceb..d4efdcb29 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java @@ -62,21 +62,6 @@ public class ModelResp extends SchemaItem { return isOpen != null && isOpen == 1; } - public Identify getPrimaryIdentify() { - if (modelDetail == null) { - return null; - } - if (CollectionUtils.isEmpty(modelDetail.getIdentifiers())) { - return null; - } - for (Identify identify : modelDetail.getIdentifiers()) { - if (!CollectionUtils.isEmpty(identify.getEntityNames())) { - return identify; - } - } - return null; - } - public List getTimeDimension() { if (modelDetail == null) { return Lists.newArrayList(); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java index 9ffa308d9..eeb1d48f2 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrector.java @@ -1,13 +1,9 @@ package com.tencent.supersonic.headless.chat.corrector; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; -import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.jsqlparser.SqlValidHelper; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.chat.ChatQueryContext; import lombok.extern.slf4j.Slf4j; @@ -18,9 +14,7 @@ import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; /** Perform SQL corrections on the "Select" section in S2SQL. */ @Slf4j @@ -42,13 +36,11 @@ public class SelectCorrector extends BaseSemanticCorrector { && aggregateFields.size() == selectFields.size()) { return; } - correctS2SQL = addFieldsToSelect(chatQueryContext, semanticParseInfo, correctS2SQL); + correctS2SQL = addFieldsToSelect(semanticParseInfo, correctS2SQL); semanticParseInfo.getSqlInfo().setCorrectedS2SQL(correctS2SQL); } - protected String addFieldsToSelect(ChatQueryContext chatQueryContext, - SemanticParseInfo semanticParseInfo, String correctS2SQL) { - correctS2SQL = addTagDefaultFields(chatQueryContext, semanticParseInfo, correctS2SQL); + protected String addFieldsToSelect(SemanticParseInfo semanticParseInfo, String correctS2SQL) { Set selectFields = new HashSet<>(SqlSelectHelper.getSelectFields(correctS2SQL)); Set needAddFields = new HashSet<>(SqlSelectHelper.getGroupByFields(correctS2SQL)); @@ -70,34 +62,4 @@ public class SelectCorrector extends BaseSemanticCorrector { return addFieldsToSelectSql; } - private String addTagDefaultFields(ChatQueryContext chatQueryContext, - SemanticParseInfo semanticParseInfo, String correctS2SQL) { - // If it is in DETAIL mode and select *, add default metrics and dimensions. - boolean hasAsterisk = SqlSelectFunctionHelper.hasAsterisk(correctS2SQL); - if (!(hasAsterisk && QueryType.DETAIL.equals(semanticParseInfo.getQueryType()))) { - return correctS2SQL; - } - Long dataSetId = semanticParseInfo.getDataSetId(); - DataSetSchema dataSetSchema = - chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); - Set needAddDefaultFields = new HashSet<>(); - if (Objects.nonNull(dataSetSchema)) { - if (!CollectionUtils.isEmpty(dataSetSchema.getTagDefaultMetrics())) { - Set metrics = dataSetSchema.getTagDefaultMetrics().stream() - .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet()); - needAddDefaultFields.addAll(metrics); - } - if (!CollectionUtils.isEmpty(dataSetSchema.getTagDefaultDimensions())) { - Set dimensions = dataSetSchema.getTagDefaultDimensions().stream() - .map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet()); - needAddDefaultFields.addAll(dimensions); - } - } - // remove * in sql and add default fields. - if (!CollectionUtils.isEmpty(needAddDefaultFields)) { - correctS2SQL = - SqlRemoveHelper.removeAsteriskAndAddFields(correctS2SQL, needAddDefaultFields); - } - return correctS2SQL; - } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/NatureHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/NatureHelper.java index 46a91d5bf..013f01851 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/NatureHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/NatureHelper.java @@ -35,9 +35,6 @@ public class NatureHelper { case DIMENSION: result = SchemaElementType.DIMENSION; break; - case ENTITY: - result = SchemaElementType.ENTITY; - break; case DATASET: result = SchemaElementType.DATASET; break; diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EntityMapper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EntityMapper.java deleted file mode 100644 index 138a45ea4..000000000 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EntityMapper.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.tencent.supersonic.headless.chat.mapper; - -import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.headless.api.pojo.SemanticSchema; -import com.tencent.supersonic.headless.chat.ChatQueryContext; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.util.CollectionUtils; - -import java.util.List; -import java.util.stream.Collectors; - -/** A mapper capable of converting the VALUE of entity dimension values into ID types. */ -@Slf4j -public class EntityMapper extends BaseMapper { - - @Override - public void doMap(ChatQueryContext chatQueryContext) { - SchemaMapInfo schemaMapInfo = chatQueryContext.getMapInfo(); - for (Long dataSetId : schemaMapInfo.getMatchedDataSetInfos()) { - List schemaElementMatchList = - schemaMapInfo.getMatchedElements(dataSetId); - if (CollectionUtils.isEmpty(schemaElementMatchList)) { - continue; - } - SchemaElement entity = getEntity(dataSetId, chatQueryContext); - if (entity == null || entity.getId() == null) { - continue; - } - List valueSchemaElements = schemaElementMatchList.stream() - .filter(schemaElementMatch -> SchemaElementType.VALUE - .equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); - for (SchemaElementMatch schemaElementMatch : valueSchemaElements) { - if (!entity.getId().equals(schemaElementMatch.getElement().getId())) { - continue; - } - if (!checkExistSameEntitySchemaElements(schemaElementMatch, - schemaElementMatchList)) { - SchemaElementMatch entitySchemaElementMath = new SchemaElementMatch(); - BeanUtils.copyProperties(schemaElementMatch, entitySchemaElementMath); - entitySchemaElementMath.setElement(entity); - schemaElementMatchList.add(entitySchemaElementMath); - } - schemaElementMatch.getElement().setType(SchemaElementType.ID); - } - } - } - - private boolean checkExistSameEntitySchemaElements(SchemaElementMatch valueSchemaElementMatch, - List schemaElementMatchList) { - List entitySchemaElements = schemaElementMatchList.stream() - .filter(schemaElementMatch -> SchemaElementType.ENTITY - .equals(schemaElementMatch.getElement().getType())) - .collect(Collectors.toList()); - for (SchemaElementMatch schemaElementMatch : entitySchemaElements) { - if (schemaElementMatch.getElement().getId() - .equals(valueSchemaElementMatch.getElement().getId())) { - return true; - } - } - return false; - } - - private SchemaElement getEntity(Long dataSetId, ChatQueryContext chatQueryContext) { - SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); - DataSetSchema modelSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId); - if (modelSchema != null && modelSchema.getEntity() != null) { - return modelSchema.getEntity(); - } - return null; - } -} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java index 02fa9691b..5fa6df097 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java @@ -94,9 +94,8 @@ public class MapFilter { SchemaElement element = schemaElementMatch.getElement(); SchemaElementType type = element.getType(); - boolean isEntityOrDatasetOrId = SchemaElementType.ENTITY.equals(type) - || SchemaElementType.DATASET.equals(type) - || SchemaElementType.ID.equals(type); + boolean isEntityOrDatasetOrId = + SchemaElementType.DATASET.equals(type) || SchemaElementType.ID.equals(type); return !isEntityOrDatasetOrId && needRemovePredicate.test(element); }); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/QueryTypeParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/QueryTypeParser.java index 11ad82f97..733e4df5e 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/QueryTypeParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/QueryTypeParser.java @@ -1,29 +1,20 @@ package com.tencent.supersonic.headless.chat.parser; import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.SqlInfo; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.query.SemanticQuery; -import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMSqlQuery; -import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -/** QueryTypeParser resolves query type as either AGGREGATE or DETAIL or ID. */ +/** QueryTypeParser resolves query type as either AGGREGATE or DETAIL */ @Slf4j public class QueryTypeParser implements SemanticParser { @@ -52,22 +43,6 @@ public class QueryTypeParser implements SemanticParser { return QueryType.DETAIL; } - // 1. entity queryType - Long dataSetId = parseInfo.getDataSetId(); - SemanticSchema semanticSchema = chatQueryContext.getSemanticSchema(); - if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) { - List whereFields = SqlSelectHelper.getWhereFields(sqlInfo.getParsedS2SQL()); - List whereFilterByTimeFields = filterByTimeFields(whereFields); - if (CollectionUtils.isNotEmpty(whereFilterByTimeFields)) { - Set ids = semanticSchema.getEntities(dataSetId).stream() - .map(SchemaElement::getName).collect(Collectors.toSet()); - if (CollectionUtils.isNotEmpty(ids) - && ids.stream().anyMatch(whereFilterByTimeFields::contains)) { - return QueryType.ID; - } - } - } - // 2. AGG queryType if (SqlSelectFunctionHelper.hasAggregateFunction(sqlInfo.getParsedS2SQL())) { return QueryType.AGGREGATE; @@ -76,20 +51,4 @@ public class QueryTypeParser implements SemanticParser { return QueryType.DETAIL; } - private static List filterByTimeFields(List whereFields) { - return whereFields.stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) - .collect(Collectors.toList()); - } - - private static boolean selectContainsMetric(SqlInfo sqlInfo, Long dataSetId, - SemanticSchema semanticSchema) { - List selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getParsedS2SQL()); - List metrics = semanticSchema.getMetrics(dataSetId); - if (CollectionUtils.isNotEmpty(metrics)) { - Set metricNameSet = - metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet()); - return selectFields.stream().anyMatch(metricNameSet::contains); - } - return false; - } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java index 116a6f057..4a915f0e5 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java @@ -10,7 +10,6 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.query.BaseSemanticQuery; @@ -124,18 +123,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { SchemaElement element = schemaMatch.getElement(); element.setOrder(1 - schemaMatch.getSimilarity()); switch (element.getType()) { - case ID: - SchemaElement entityElement = - semanticSchema.getElement(SchemaElementType.ENTITY, element.getId()); - if (entityElement != null) { - if (id2Values.containsKey(element.getId())) { - id2Values.get(element.getId()).add(schemaMatch); - } else { - id2Values.put(element.getId(), - new ArrayList<>(Arrays.asList(schemaMatch))); - } - } - break; case VALUE: SchemaElement dimElement = semanticSchema.getElement(SchemaElementType.DIMENSION, element.getId()); @@ -154,13 +141,9 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { case METRIC: parseInfo.getMetrics().add(element); break; - case ENTITY: - parseInfo.setEntity(element); - break; default: } } - addToFilters(id2Values, parseInfo, semanticSchema, SchemaElementType.ENTITY); addToFilters(dim2Values, parseInfo, semanticSchema, SchemaElementType.DIMENSION); } @@ -182,8 +165,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { dimensionFilter.setName(dimension.getName()); dimensionFilter.setOperator(FilterOperatorEnum.EQUALS); dimensionFilter.setElementID(schemaMatch.getElement().getId()); - parseInfo.setEntity( - semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey())); parseInfo.getDimensionFilters().add(dimensionFilter); } else { QueryFilter dimensionFilter = new QueryFilter(); @@ -216,11 +197,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { return convertQueryMultiStruct(); } - @Override - public void setParseInfo(SemanticParseInfo parseInfo) { - this.parseInfo = parseInfo; - } - public static List resolve(Long dataSetId, List candidateElementMatches, ChatQueryContext chatQueryContext) { List matchedQueries = new ArrayList<>(); @@ -228,7 +204,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { List matches = semanticQuery.match(candidateElementMatches, chatQueryContext); - if (matches.size() > 0) { + if (!matches.isEmpty()) { RuleSemanticQuery query = QueryManager.createRuleQuery(semanticQuery.getQueryMode()); query.getParseInfo().getElementMatches().addAll(matches); @@ -238,10 +214,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { return matchedQueries; } - protected QueryStructReq convertQueryStruct() { - return QueryReqBuilder.buildStructReq(parseInfo); - } - protected QueryMultiStructReq convertQueryMultiStruct() { return QueryReqBuilder.buildMultiStructReq(parseInfo); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailFilterQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailFilterQuery.java deleted file mode 100644 index 83150295a..000000000 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailFilterQuery.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.supersonic.headless.chat.query.rule.detail; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE; -import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - -@Slf4j -@Component -public class DetailFilterQuery extends DetailListQuery { - - public static final String QUERY_MODE = "DETAIL_LIST_FILTER"; - - public DetailFilterQuery() { - super(); - queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1); - queryMatcher.addOption(ENTITY, REQUIRED, AT_LEAST, 1); - } - - @Override - public String getQueryMode() { - return QUERY_MODE; - } -} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailIdQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailIdQuery.java deleted file mode 100644 index b96d5971a..000000000 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailIdQuery.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.tencent.supersonic.headless.chat.query.rule.detail; - -import org.springframework.stereotype.Component; - -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; -import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - -@Component -public class DetailIdQuery extends DetailListQuery { - - public static final String QUERY_MODE = "DETAIL_ID"; - - public DetailIdQuery() { - super(); - queryMatcher.addOption(ID, REQUIRED, AT_LEAST, 1); - } - - @Override - public String getQueryMode() { - return QUERY_MODE; - } -} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailListQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailListQuery.java deleted file mode 100644 index f127d40d8..000000000 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailListQuery.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.tencent.supersonic.headless.chat.query.rule.detail; - -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.Order; -import com.tencent.supersonic.headless.api.pojo.*; -import com.tencent.supersonic.headless.api.pojo.DetailTypeDefaultConfig; -import com.tencent.supersonic.headless.chat.ChatQueryContext; -import org.apache.commons.collections.CollectionUtils; - -import java.util.LinkedHashSet; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -public abstract class DetailListQuery extends DetailSemanticQuery { - - @Override - public void fillParseInfo(ChatQueryContext chatQueryContext) { - super.fillParseInfo(chatQueryContext); - this.addEntityDetailAndOrderByMetric(chatQueryContext, parseInfo); - } - - private void addEntityDetailAndOrderByMetric(ChatQueryContext chatQueryContext, - SemanticParseInfo parseInfo) { - Long dataSetId = parseInfo.getDataSetId(); - if (Objects.isNull(dataSetId) || dataSetId <= 0L) { - return; - } - DataSetSchema dataSetSchema = - chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); - if (dataSetSchema != null && Objects.nonNull(dataSetSchema.getEntity())) { - Set dimensions = new LinkedHashSet<>(); - Set metrics = new LinkedHashSet<>(); - Set orders = new LinkedHashSet<>(); - DetailTypeDefaultConfig detailTypeDefaultConfig = - dataSetSchema.getTagTypeDefaultConfig(); - if (detailTypeDefaultConfig != null - && detailTypeDefaultConfig.getDefaultDisplayInfo() != null) { - if (CollectionUtils.isNotEmpty( - detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) { - metrics = detailTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds() - .stream().map(id -> { - SchemaElement metric = - dataSetSchema.getElement(SchemaElementType.METRIC, id); - if (metric != null) { - orders.add( - new Order(metric.getBizName(), Constants.DESC_UPPER)); - } - return metric; - }).filter(Objects::nonNull).collect(Collectors.toSet()); - } - if (CollectionUtils.isNotEmpty( - detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) { - dimensions = detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds() - .stream() - .map(id -> dataSetSchema.getElement(SchemaElementType.DIMENSION, id)) - .filter(Objects::nonNull).collect(Collectors.toSet()); - } - } - parseInfo.setDimensions(dimensions); - parseInfo.setMetrics(metrics); - parseInfo.setOrders(orders); - } - } -} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java index 922ad43a1..0bb9acb61 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java @@ -41,7 +41,7 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery { Map dataSetSchemaMap = chatQueryContext.getSemanticSchema().getDataSetSchemaMap(); DataSetSchema dataSetSchema = dataSetSchemaMap.get(parseInfo.getDataSetId()); - TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); + TimeDefaultConfig timeDefaultConfig = dataSetSchema.getDetailTypeTimeDefaultConfig(); if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit()) && timeDefaultConfig.getUnit() != -1) { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricIdQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricIdQuery.java deleted file mode 100644 index a42b14b0b..000000000 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricIdQuery.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.tencent.supersonic.headless.chat.query.rule.metric; - -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.FilterType; -import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID; -import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; - -@Slf4j -@Component -public class MetricIdQuery extends MetricSemanticQuery { - - public static final String QUERY_MODE = "METRIC_ID"; - - public MetricIdQuery() { - super(); - queryMatcher.addOption(ID, REQUIRED, AT_LEAST, 1).addOption(ENTITY, REQUIRED, AT_LEAST, 1); - } - - @Override - public String getQueryMode() { - return QUERY_MODE; - } - - @Override - public SemanticQueryReq buildSemanticQueryReq() { - if (!isMultiStructQuery()) { - return super.buildSemanticQueryReq(); - } - return super.multiStructExecute(); - } - - protected boolean isMultiStructQuery() { - Set filterBizName = new HashSet<>(); - parseInfo.getDimensionFilters().stream().filter(filter -> filter.getElementID() != null) - .forEach(filter -> filterBizName.add(filter.getBizName())); - return FilterType.UNION.equals(parseInfo.getFilterType()) && filterBizName.size() > 1; - } - - @Override - protected QueryStructReq convertQueryStruct() { - QueryStructReq queryStructReq = super.convertQueryStruct(); - addDimension(queryStructReq, true); - return queryStructReq; - } - - @Override - protected QueryMultiStructReq convertQueryMultiStruct() { - QueryMultiStructReq queryMultiStructReq = super.convertQueryMultiStruct(); - for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { - addDimension(queryStructReq, false); - } - return queryMultiStructReq; - } - - private void addDimension(QueryStructReq queryStructReq, boolean onlyOperateInFilter) { - if (!queryStructReq.getDimensionFilters().isEmpty()) { - List dimensions = queryStructReq.getGroups(); - log.info("addDimension before [{}]", queryStructReq.getGroups()); - List filters = new ArrayList<>(queryStructReq.getDimensionFilters()); - if (onlyOperateInFilter) { - filters = filters.stream() - .filter(filter -> filter.getOperator().equals(FilterOperatorEnum.IN)) - .collect(Collectors.toList()); - } - filters.forEach(d -> { - if (!dimensions.contains(d.getBizName())) { - dimensions.add(d.getBizName()); - } - }); - queryStructReq.setGroups(dimensions); - log.info("addDimension after [{}]", queryStructReq.getGroups()); - } - } -} diff --git a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrectorTest.java b/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrectorTest.java deleted file mode 100644 index 7bdc961ee..000000000 --- a/headless/chat/src/test/java/com/tencent/supersonic/headless/chat/corrector/SelectCorrectorTest.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.tencent.supersonic.headless.chat.corrector; - -import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.pojo.*; -import com.tencent.supersonic.headless.api.pojo.DetailTypeDefaultConfig; -import com.tencent.supersonic.headless.chat.ChatQueryContext; -import org.junit.Assert; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.springframework.core.env.Environment; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.mockito.Mockito.when; - -class SelectCorrectorTest { - - Long dataSetId = 2L; - - @Test - void testDoCorrect() { - MockedStatic mocked = Mockito.mockStatic(ContextUtils.class); - Environment mockEnvironment = Mockito.mock(Environment.class); - mocked.when(() -> ContextUtils.getBean(Environment.class)).thenReturn(mockEnvironment); - when(mockEnvironment.getProperty(SelectCorrector.ADDITIONAL_INFORMATION)).thenReturn(""); - - BaseSemanticCorrector corrector = new SelectCorrector(); - ChatQueryContext chatQueryContext = buildQueryContext(dataSetId); - SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); - SchemaElement dataSet = new SchemaElement(); - dataSet.setDataSetId(dataSetId); - semanticParseInfo.setDataSet(dataSet); - semanticParseInfo.setQueryType(QueryType.DETAIL); - SqlInfo sqlInfo = new SqlInfo(); - String sql = "SELECT * FROM 艺人库 WHERE 艺人名='周杰伦'"; - sqlInfo.setParsedS2SQL(sql); - sqlInfo.setCorrectedS2SQL(sql); - semanticParseInfo.setSqlInfo(sqlInfo); - corrector.correct(chatQueryContext, semanticParseInfo); - Assert.assertEquals("SELECT 粉丝数, 国籍, 艺人名, 性别 FROM 艺人库 WHERE 艺人名 = '周杰伦'", - semanticParseInfo.getSqlInfo().getCorrectedS2SQL()); - } - - private ChatQueryContext buildQueryContext(Long dataSetId) { - ChatQueryContext chatQueryContext = new ChatQueryContext(); - List dataSetSchemaList = new ArrayList<>(); - DataSetSchema dataSetSchema = new DataSetSchema(); - QueryConfig queryConfig = new QueryConfig(); - DetailTypeDefaultConfig detailTypeDefaultConfig = new DetailTypeDefaultConfig(); - DefaultDisplayInfo defaultDisplayInfo = new DefaultDisplayInfo(); - List dimensionIds = new ArrayList<>(); - dimensionIds.add(1L); - dimensionIds.add(2L); - dimensionIds.add(3L); - defaultDisplayInfo.setDimensionIds(dimensionIds); - - List metricIds = new ArrayList<>(); - metricIds.add(4L); - defaultDisplayInfo.setMetricIds(metricIds); - - detailTypeDefaultConfig.setDefaultDisplayInfo(defaultDisplayInfo); - queryConfig.setDetailTypeDefaultConfig(detailTypeDefaultConfig); - - dataSetSchema.setQueryConfig(queryConfig); - SchemaElement schemaElement = new SchemaElement(); - schemaElement.setDataSetId(dataSetId); - dataSetSchema.setDataSet(schemaElement); - Set dimensions = new HashSet<>(); - SchemaElement element1 = new SchemaElement(); - element1.setDataSetId(dataSetId); - element1.setId(1L); - element1.setName("艺人名"); - dimensions.add(element1); - - SchemaElement element2 = new SchemaElement(); - element2.setDataSetId(dataSetId); - element2.setId(2L); - element2.setName("性别"); - dimensions.add(element2); - - SchemaElement element3 = new SchemaElement(); - element3.setDataSetId(dataSetId); - element3.setId(3L); - element3.setName("国籍"); - dimensions.add(element3); - - dataSetSchema.setDimensions(dimensions); - - Set metrics = new HashSet<>(); - SchemaElement metric1 = new SchemaElement(); - metric1.setDataSetId(dataSetId); - metric1.setId(4L); - metric1.setName("粉丝数"); - metrics.add(metric1); - - dataSetSchema.setMetrics(metrics); - dataSetSchemaList.add(dataSetSchema); - - SemanticSchema semanticSchema = new SemanticSchema(dataSetSchemaList); - chatQueryContext.setSemanticSchema(semanticSchema); - return chatQueryContext; - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java deleted file mode 100644 index a89c64f0c..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityDetailData.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.tencent.supersonic.headless.core.config; - -import lombok.Data; - -import java.util.List; - -/** when query an entity, return related dimension/metric info */ -@Data -public class EntityDetailData { - - private List dimensionIds; - private List metricIds; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java deleted file mode 100644 index d741551f4..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/EntityInternalDetail.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.tencent.supersonic.headless.core.config; - -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import lombok.Data; - -import java.util.List; - -@Data -public class EntityInternalDetail { - - List dimensionList; - List metricList; -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java index 977545fbd..9dfff837c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java @@ -2,9 +2,7 @@ package com.tencent.supersonic.headless.server.facade.service; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.EntityInfo; import com.tencent.supersonic.headless.api.pojo.MetaFilter; -import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; @@ -18,15 +16,13 @@ import java.util.List; /** This interface abstracts functionalities provided by a semantic layer. */ public interface SemanticLayerService { - DataSetSchema getDataSetSchema(Long id); - SemanticTranslateResp translate(SemanticQueryReq queryReq, User user) throws Exception; SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception; SemanticQueryResp queryDimensionValue(DimensionValueReq dimensionValueReq, User user); - EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user); + DataSetSchema getDataSetSchema(Long id); List getDomainDataSetTree(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index b9f752cdd..f1b43a77d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -2,18 +2,16 @@ package com.tencent.supersonic.headless.server.facade.service.impl; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.User; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.*; -import com.tencent.supersonic.headless.api.pojo.DetailTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; +import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; @@ -32,7 +30,6 @@ import com.tencent.supersonic.headless.chat.knowledge.MapResult; import com.tencent.supersonic.headless.chat.knowledge.SearchService; import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper; -import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder; import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.pojo.QueryStatement; @@ -54,13 +51,10 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; -import java.time.LocalDate; import java.util.ArrayList; import java.util.HashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -262,39 +256,6 @@ public class S2SemanticLayerService implements SemanticLayerService { return dimensionResp; } - public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, - User user) { - if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) { - EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema); - if (parseInfo.getDimensionFilters().size() <= 0 - || entityInfo.getDataSetInfo() == null) { - entityInfo.setMetrics(null); - entityInfo.setDimensions(null); - return entityInfo; - } - String primaryKey = entityInfo.getDataSetInfo().getPrimaryKey(); - if (StringUtils.isNotBlank(primaryKey)) { - String entityId = ""; - for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) { - if (chatFilter != null && chatFilter.getBizName() != null - && chatFilter.getBizName().equals(primaryKey)) { - if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) { - entityId = chatFilter.getValue().toString(); - } - } - } - entityInfo.setEntityId(entityId); - try { - fillEntityInfoValue(entityInfo, dataSetSchema, user); - return entityInfo; - } catch (Exception e) { - log.error("setMainModel error", e); - } - } - } - return null; - } - @Override public List getDomainDataSetTree() { return schemaService.getDomainDataSetTree(); @@ -305,31 +266,11 @@ public class S2SemanticLayerService implements SemanticLayerService { return dimensionService.getDimensions(metaFilter); } - private Set getDimensions(EntityInfo modelInfo) { - Set dimensions = new LinkedHashSet(); - for (DataInfo mainEntityDimension : modelInfo.getDimensions()) { - SchemaElement dimension = new SchemaElement(); - dimension.setBizName(mainEntityDimension.getBizName()); - dimensions.add(dimension); - } - return dimensions; - } - @Override public List getMetrics(MetaFilter metaFilter) { return metricService.getMetrics(metaFilter); } - private Set getMetrics(EntityInfo modelInfo) { - Set metrics = Sets.newHashSet(); - for (DataInfo metricValue : modelInfo.getMetrics()) { - SchemaElement metric = new SchemaElement(); - BeanUtils.copyProperties(metricValue, metric); - metrics.add(metric); - } - return metrics; - } - private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception { // If dataSetId or DataSetName is empty, parse dataSetId from the SQL @@ -437,131 +378,4 @@ public class S2SemanticLayerService implements SemanticLayerService { metricDrillDownChecker.checkQuery(queryStatement); } - private EntityInfo getEntityBasicInfo(DataSetSchema dataSetSchema) { - - EntityInfo entityInfo = new EntityInfo(); - if (dataSetSchema == null) { - return entityInfo; - } - Long dataSetId = dataSetSchema.getDataSet().getDataSetId(); - DataSetInfo dataSetInfo = new DataSetInfo(); - dataSetInfo.setItemId(dataSetId.intValue()); - dataSetInfo.setName(dataSetSchema.getDataSet().getName()); - dataSetInfo.setWords(dataSetSchema.getDataSet().getAlias()); - dataSetInfo.setBizName(dataSetSchema.getDataSet().getBizName()); - if (Objects.nonNull(dataSetSchema.getEntity())) { - dataSetInfo.setPrimaryKey(dataSetSchema.getEntity().getBizName()); - } - entityInfo.setDataSetInfo(dataSetInfo); - DetailTypeDefaultConfig detailTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig(); - if (detailTypeDefaultConfig == null - || detailTypeDefaultConfig.getDefaultDisplayInfo() == null) { - return entityInfo; - } - List dimensions = detailTypeDefaultConfig.getDefaultDisplayInfo() - .getDimensionIds().stream().map(id -> { - SchemaElement element = - dataSetSchema.getElement(SchemaElementType.DIMENSION, id); - if (element == null) { - return null; - } - return new DataInfo(element.getId().intValue(), element.getName(), - element.getBizName(), null); - }).filter(Objects::nonNull).collect(Collectors.toList()); - List metrics = detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds() - .stream().map(id -> { - SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id); - if (element == null) { - return null; - } - return new DataInfo(element.getId().intValue(), element.getName(), - element.getBizName(), null); - }).filter(Objects::nonNull).collect(Collectors.toList()); - entityInfo.setDimensions(dimensions); - entityInfo.setMetrics(metrics); - return entityInfo; - } - - private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, - User user) { - SemanticQueryResp queryResultWithColumns = - getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user); - if (queryResultWithColumns != null) { - if (!CollectionUtils.isEmpty(queryResultWithColumns.getResultList())) { - Map result = queryResultWithColumns.getResultList().get(0); - for (Map.Entry entry : result.entrySet()) { - String entryKey = getEntryKey(entry); - if (entry.getValue() == null || entryKey == null) { - continue; - } - entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName())) - .forEach(i -> i.setValue(entry.getValue().toString())); - entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName())) - .forEach(i -> i.setValue(entry.getValue().toString())); - } - } - } - } - - private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, - DataSetSchema dataSetSchema, User user) { - SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); - semanticParseInfo.setDataSet(dataSetSchema.getDataSet()); - semanticParseInfo.setQueryType(QueryType.DETAIL); - semanticParseInfo.setMetrics(getMetrics(entityInfo)); - semanticParseInfo.setDimensions(getDimensions(entityInfo)); - - if (dataSetSchema.containsPartitionDimensions()) { - DateConf dateInfo = new DateConf(); - int unit = 1; - TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig(); - if (Objects.nonNull(timeDefaultConfig)) { - unit = timeDefaultConfig.getUnit(); - String date = LocalDate.now().minusDays(unit).toString(); - dateInfo.setDateMode(DateConf.DateMode.BETWEEN); - dateInfo.setStartDate(date); - dateInfo.setEndDate(date); - } else { - dateInfo.setUnit(unit); - dateInfo.setDateMode(DateConf.DateMode.RECENT); - } - semanticParseInfo.setDateInfo(dateInfo); - } - - // add filter - QueryFilter chatFilter = getQueryFilter(entityInfo); - Set chatFilters = Sets.newHashSet(); - chatFilters.add(chatFilter); - semanticParseInfo.setDimensionFilters(chatFilters); - - SemanticQueryResp queryResultWithColumns = null; - try { - QuerySqlReq querySqlReq = QueryReqBuilder.buildStructReq(semanticParseInfo).convert(); - queryResultWithColumns = queryByReq(querySqlReq, user); - } catch (Exception e) { - log.warn("setMainModel queryByStruct error, e:", e); - } - return queryResultWithColumns; - } - - private QueryFilter getQueryFilter(EntityInfo entityInfo) { - QueryFilter chatFilter = new QueryFilter(); - chatFilter.setValue(entityInfo.getEntityId()); - chatFilter.setOperator(FilterOperatorEnum.EQUALS); - chatFilter.setBizName(getEntityPrimaryName(entityInfo)); - return chatFilter; - } - - private String getEntryKey(Map.Entry entry) { - // metric parser special handle, TODO delete - String entryKey = entry.getKey(); - if (entryKey.contains("__")) { - entryKey = entryKey.split("__")[1]; - } - return entryKey; - } - - private String getEntityPrimaryName(EntityInfo entityInfo) { - return entityInfo.getDataSetInfo().getPrimaryKey(); - } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java deleted file mode 100644 index 0bbb73e5d..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/EntityInfoProcessor.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.tencent.supersonic.headless.server.processor; - -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.EntityInfo; -import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.headless.chat.ChatQueryContext; -import com.tencent.supersonic.headless.chat.query.QueryManager; -import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; - -/** - * EntityInfoProcessor fills core attributes of an entity so that users get to know which entity is - * parsed out. - */ -public class EntityInfoProcessor implements ResultProcessor { - - @Override - public void process(ParseResp parseResp, ChatQueryContext chatQueryContext) { - parseResp.getSelectedParses().forEach(parseInfo -> { - String queryMode = parseInfo.getQueryMode(); - if (!QueryManager.isDetailQuery(queryMode) && !QueryManager.isMetricQuery(queryMode)) { - return; - } - - SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class); - DataSetSchema dataSetSchema = - semanticService.getDataSetSchema(parseInfo.getDataSetId()); - EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, - chatQueryContext.getRequest().getUser()); - parseInfo.setEntityInfo(entityInfo); - }); - } -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java index a7d73fade..501656c3e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java @@ -58,7 +58,6 @@ public class DictWordService { addWordsByType(DictWordType.DIMENSION, semanticSchema.getDimensions(), words); addWordsByType(DictWordType.METRIC, semanticSchema.getMetrics(), words); - addWordsByType(DictWordType.ENTITY, semanticSchema.getEntities(), words); addWordsByType(DictWordType.VALUE, semanticSchema.getDimensionValues(), words); addWordsByType(DictWordType.TERM, semanticSchema.getTerms(), words); return words; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index 3442c6b1f..0b2548485 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -15,10 +15,7 @@ import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.User; -import com.tencent.supersonic.common.pojo.enums.AuthType; -import com.tencent.supersonic.common.pojo.enums.EventType; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.pojo.enums.*; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.Measure; @@ -810,6 +807,7 @@ public class MetricServiceImpl extends ServiceImpl queryStructReq.setDimensionFilters(filters); // 7. set dateInfo queryStructReq.setDateInfo(dateInfo); + queryStructReq.setQueryType(QueryType.AGGREGATE); return queryStructReq; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java index ae8a63c0b..d4b88a658 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/RetrieveServiceImpl.java @@ -148,11 +148,6 @@ public class RetrieveServiceImpl implements RetrieveService { Long dataSetId = NatureHelper.getDataSetId(nature); SchemaElementType schemaElementType = NatureHelper.convertToElementType(nature); - // Skip if the schema element type is ENTITY - if (SchemaElementType.ENTITY.equals(schemaElementType)) { - return searchResults; - } - // Create a base search result SearchResult baseSearchResult = createBaseSearchResult(dataSetId, dataSetIdToName, matchText, wordName, schemaElementType); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java index 94b71979a..4263abe40 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java @@ -4,7 +4,15 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DimensionConstants; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; -import com.tencent.supersonic.headless.api.pojo.*; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.DimValueMap; +import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; +import com.tencent.supersonic.headless.api.pojo.RelateDimension; +import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; +import com.tencent.supersonic.headless.api.pojo.SchemaElementType; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.SchemaValueMap; import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; @@ -17,7 +25,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -50,10 +57,6 @@ public class DataSetSchemaBuilder { Set terms = getTerms(resp); dataSetSchema.getTerms().addAll(terms); - SchemaElement entity = getEntity(resp); - if (Objects.nonNull(entity)) { - dataSetSchema.setEntity(entity); - } return dataSetSchema; } @@ -99,17 +102,6 @@ public class DataSetSchemaBuilder { return tags; } - private static SchemaElement getEntity(DataSetSchemaResp resp) { - DimSchemaResp dim = resp.getPrimaryKey(); - if (Objects.isNull(dim)) { - return null; - } - return SchemaElement.builder().dataSetId(resp.getId()).model(dim.getModelId()) - .id(dim.getId()).name(dim.getName()).bizName(dim.getBizName()) - .type(SchemaElementType.ENTITY).useCnt(dim.getUseCnt()).alias(dim.getEntityAlias()) - .build(); - } - private static Set getDimensions(DataSetSchemaResp resp) { Set dimensions = new HashSet<>(); for (DimSchemaResp dim : resp.getDimensions()) { diff --git a/launchers/headless/src/main/resources/META-INF/spring.factories b/launchers/headless/src/main/resources/META-INF/spring.factories index 5309f5eab..26de67568 100644 --- a/launchers/headless/src/main/resources/META-INF/spring.factories +++ b/launchers/headless/src/main/resources/META-INF/spring.factories @@ -4,7 +4,6 @@ com.tencent.supersonic.headless.chat.mapper.SchemaMapper=\ com.tencent.supersonic.headless.chat.mapper.EmbeddingMapper, \ com.tencent.supersonic.headless.chat.mapper.KeywordMapper, \ com.tencent.supersonic.headless.chat.mapper.QueryFilterMapper, \ - com.tencent.supersonic.headless.chat.mapper.EntityMapper, \ com.tencent.supersonic.headless.chat.mapper.TermDescMapper com.tencent.supersonic.headless.chat.parser.SemanticParser=\ @@ -13,7 +12,8 @@ com.tencent.supersonic.headless.chat.parser.SemanticParser=\ com.tencent.supersonic.headless.chat.parser.QueryTypeParser com.tencent.supersonic.headless.chat.corrector.SemanticCorrector=\ - com.tencent.supersonic.headless.chat.corrector.RuleSqlCorrector + com.tencent.supersonic.headless.chat.corrector.RuleSqlCorrector,\ + com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector com.tencent.supersonic.headless.chat.knowledge.file.FileHandler=\ com.tencent.supersonic.headless.chat.knowledge.file.FileHandlerImpl @@ -46,13 +46,7 @@ com.tencent.supersonic.headless.core.cache.QueryCache=\ ### headless-server SPIs com.tencent.supersonic.headless.server.processor.ResultProcessor=\ - com.tencent.supersonic.headless.server.processor.ParseInfoProcessor,\ - com.tencent.supersonic.headless.server.processor.EntityInfoProcessor + com.tencent.supersonic.headless.server.processor.ParseInfoProcessor -### auth-authentication SPIs - -com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ - com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor - -com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ - com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor \ No newline at end of file +com.tencent.supersonic.headless.server.modeller.SemanticModeller=\ + com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller \ No newline at end of file diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/CspiderDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/CspiderDemo.java index 682043fd5..ffb039497 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/CspiderDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/CspiderDemo.java @@ -211,10 +211,6 @@ public class CspiderDemo extends S2BaseDemo { tagTimeDefaultConfig.setTimeMode(TimeMode.LAST); tagTimeDefaultConfig.setUnit(7); detailTypeDefaultConfig.setTimeDefaultConfig(tagTimeDefaultConfig); - DefaultDisplayInfo defaultDisplayInfo = new DefaultDisplayInfo(); - defaultDisplayInfo.setDimensionIds(Lists.newArrayList()); - defaultDisplayInfo.setMetricIds(Lists.newArrayList()); - detailTypeDefaultConfig.setDefaultDisplayInfo(defaultDisplayInfo); AggregateTypeDefaultConfig aggregateTypeDefaultConfig = new AggregateTypeDefaultConfig(); TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); timeDefaultConfig.setTimeMode(TimeMode.RECENT); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2ArtistDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2ArtistDemo.java index a09e79ab4..513c9a323 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2ArtistDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2ArtistDemo.java @@ -12,8 +12,15 @@ import com.tencent.supersonic.common.pojo.enums.AppModule; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.ChatAppManager; -import com.tencent.supersonic.headless.api.pojo.*; +import com.tencent.supersonic.headless.api.pojo.AggregateTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.DataSetDetail; +import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; import com.tencent.supersonic.headless.api.pojo.DetailTypeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.Dim; +import com.tencent.supersonic.headless.api.pojo.Identify; +import com.tencent.supersonic.headless.api.pojo.Measure; +import com.tencent.supersonic.headless.api.pojo.ModelDetail; +import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; @@ -24,7 +31,6 @@ import com.tencent.supersonic.headless.api.pojo.request.TagObjectReq; import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DomainResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.TagObjectResp; import lombok.extern.slf4j.Slf4j; @@ -105,7 +111,6 @@ public class S2ArtistDemo extends S2BaseDemo { ModelDetail modelDetail = new ModelDetail(); List identifiers = new ArrayList<>(); Identify identify = new Identify("歌手名", IdentifyType.primary.name(), "singer_name", 1); - identify.setEntityNames(Lists.newArrayList("歌手")); identifiers.add(identify); modelDetail.setIdentifiers(identifiers); @@ -152,11 +157,6 @@ public class S2ArtistDemo extends S2BaseDemo { dataSetReq.setTypeEnum(TypeEnums.DATASET); QueryConfig queryConfig = new QueryConfig(); DetailTypeDefaultConfig detailTypeDefaultConfig = new DetailTypeDefaultConfig(); - DefaultDisplayInfo defaultDisplayInfo = new DefaultDisplayInfo(); - defaultDisplayInfo.setDimensionIds(dataSetModelConfigs.get(0).getDimensions()); - MetricResp jsPlayCntMetric = getMetric("js_play_cnt", singerModel); - defaultDisplayInfo.setMetricIds(Lists.newArrayList(jsPlayCntMetric.getId())); - detailTypeDefaultConfig.setDefaultDisplayInfo(defaultDisplayInfo); AggregateTypeDefaultConfig aggregateTypeDefaultConfig = new AggregateTypeDefaultConfig(); queryConfig.setDetailTypeDefaultConfig(detailTypeDefaultConfig); queryConfig.setAggregateTypeDefaultConfig(aggregateTypeDefaultConfig); diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index cdecc8954..51484b432 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -4,7 +4,6 @@ com.tencent.supersonic.headless.chat.mapper.SchemaMapper=\ com.tencent.supersonic.headless.chat.mapper.EmbeddingMapper, \ com.tencent.supersonic.headless.chat.mapper.KeywordMapper, \ com.tencent.supersonic.headless.chat.mapper.QueryFilterMapper, \ - com.tencent.supersonic.headless.chat.mapper.EntityMapper, \ com.tencent.supersonic.headless.chat.mapper.TermDescMapper com.tencent.supersonic.headless.chat.parser.SemanticParser=\ @@ -47,8 +46,7 @@ com.tencent.supersonic.headless.core.cache.QueryCache=\ ### headless-server SPIs com.tencent.supersonic.headless.server.processor.ResultProcessor=\ - com.tencent.supersonic.headless.server.processor.ParseInfoProcessor,\ - com.tencent.supersonic.headless.server.processor.EntityInfoProcessor + com.tencent.supersonic.headless.server.processor.ParseInfoProcessor com.tencent.supersonic.headless.server.modeller.SemanticModeller=\ com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java index 591074c66..1cf356a6f 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java @@ -9,8 +9,6 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.chat.query.rule.detail.DetailDimensionQuery; -import com.tencent.supersonic.headless.chat.query.rule.detail.DetailFilterQuery; -import com.tencent.supersonic.headless.chat.query.rule.detail.DetailIdQuery; import com.tencent.supersonic.util.DataUtils; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; @@ -44,53 +42,22 @@ public class DetailTest extends BaseTest { } @Test - public void test_detail_id() throws Exception { - QueryResult actualResult = submitNewChat("周杰伦", DataUtils.tagAgentId); - - QueryResult expectedResult = new QueryResult(); - SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); - expectedResult.setChatContext(expectedParseInfo); - - expectedResult.setQueryMode(DetailIdQuery.QUERY_MODE); - expectedParseInfo.setQueryType(QueryType.DETAIL); - expectedParseInfo.setAggType(AggregateTypeEnum.NONE); - - QueryFilter dimensionFilter = - DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 8L); - expectedParseInfo.getDimensionFilters().add(dimensionFilter); - - expectedParseInfo.getMetrics().add(SchemaElement.builder().name("播放量").build()); - expectedParseInfo.getDimensions() - .addAll(Lists.newArrayList(SchemaElement.builder().name("歌手名").build(), - SchemaElement.builder().name("活跃区域").build(), - SchemaElement.builder().name("流派").build(), - SchemaElement.builder().name("代表作").build())); - - assertQueryResult(expectedResult, actualResult); - } - - @Test - public void test_detail_list_filter() throws Exception { + public void test_detail_filter() throws Exception { QueryResult actualResult = submitNewChat("国风歌手", DataUtils.tagAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); expectedResult.setChatContext(expectedParseInfo); - expectedResult.setQueryMode(DetailFilterQuery.QUERY_MODE); + expectedResult.setQueryMode(DetailDimensionQuery.QUERY_MODE); expectedParseInfo.setQueryType(QueryType.DETAIL); expectedParseInfo.setAggType(AggregateTypeEnum.NONE); QueryFilter dimensionFilter = DataUtils.getFilter("genre", FilterOperatorEnum.EQUALS, "国风", "流派", 7L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); - - expectedParseInfo.getMetrics().add(SchemaElement.builder().name("播放量").build()); expectedParseInfo.getDimensions() - .addAll(Lists.newArrayList(SchemaElement.builder().name("歌手名").build(), - SchemaElement.builder().name("活跃区域").build(), - SchemaElement.builder().name("流派").build(), - SchemaElement.builder().name("代表作").build())); + .addAll(Lists.newArrayList(SchemaElement.builder().name("歌手名").build())); assertQueryResult(expectedResult, actualResult); }