topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Dataset supports query mode settings, and the chat layer supports tag mode (#802)

Browse Source
This commit is contained in:
lexluo09
2024-03-12 12:15:21 +08:00
committed by GitHub
parent 1d91a972da
commit ae7acb4817
26 changed files with 171 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DataSetSchema.java
View File

@@ -1,15 +1,15 @@
package com.tencent.supersonic.chat.api.pojo;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import lombok.Data;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import lombok.Data;
@Data
public class DataSetSchema {
@@ -22,6 +22,7 @@ public class DataSetSchema {
private Set<SchemaElement> tagValues = new HashSet<>();
private SchemaElement entity = new SchemaElement();
private QueryConfig queryConfig;
private QueryType queryType;
public SchemaElement getElement(SchemaElementType elementType, long elementID) {
Optional<SchemaElement> element = Optional.empty();

2
chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java
View File

@@ -66,8 +66,10 @@ public class WhereCorrector extends BaseSemanticCorrector {
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
List<String> whereFields = SqlSelectHelper.getWhereFields(correctS2SQL);
if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
Pair<String, String> startEndDate = S2SqlDateHelper.getStartEndDate(queryContext,
semanticParseInfo.getDataSetId(), semanticParseInfo.getQueryType());
if (StringUtils.isNotBlank(startEndDate.getLeft())
&& StringUtils.isNotBlank(startEndDate.getRight())) {
correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL);

64
chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java
View File

@@ -1,26 +1,14 @@
package com.tencent.supersonic.chat.core.parser;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.DataSetSchema;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.chat.core.query.SemanticQuery;
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery;
import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
/**
* QueryTypeParser resolves query type as either METRIC or TAG, or ID.
@@ -38,50 +26,12 @@ public class QueryTypeParser implements SemanticParser {
// 1.init S2SQL
semanticQuery.initS2Sql(queryContext.getSemanticSchema(), user);
// 2.set queryType
QueryType queryType = getQueryType(queryContext, semanticQuery);
semanticQuery.getParseInfo().setQueryType(queryType);
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
Long dataSetId = parseInfo.getDataSetId();
DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId);
parseInfo.setQueryType(dataSetSchema.getQueryType());
}
}
private QueryType getQueryType(QueryContext queryContext, SemanticQuery semanticQuery) {
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
SqlInfo sqlInfo = parseInfo.getSqlInfo();
if (Objects.isNull(sqlInfo) || StringUtils.isBlank(sqlInfo.getS2SQL())) {
return QueryType.ID;
}
//1. entity queryType
Long dataSetId = parseInfo.getDataSetId();
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) {
//If all the fields in the SELECT statement are of tag type.
List<String> whereFields = SqlSelectHelper.getWhereFields(sqlInfo.getS2SQL())
.stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(whereFields)) {
Set<String> ids = semanticSchema.getEntities(dataSetId).stream().map(SchemaElement::getName)
.collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(ids) && ids.stream().anyMatch(whereFields::contains)) {
return QueryType.ID;
}
Set<String> tags = semanticSchema.getTags(dataSetId).stream().map(SchemaElement::getName)
.collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(tags) && tags.containsAll(whereFields)) {
return QueryType.TAG;
}
}
}
//2. metric queryType
List<String> selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getS2SQL());
List<SchemaElement> metrics = semanticSchema.getMetrics(dataSetId);
if (CollectionUtils.isNotEmpty(metrics)) {
Set<String> metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet());
boolean containMetric = selectFields.stream().anyMatch(metricNameSet::contains);
if (containMetric) {
return QueryType.METRIC;
}
}
return QueryType.ID;
}
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java
View File

@@ -2,23 +2,21 @@ package com.tencent.supersonic.chat.core.query.llm.analytics;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.query.QueryManager;
import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.util.HashMap;
@@ -53,12 +51,6 @@ public class MetricAnalyzeQuery extends LLMSemanticQuery {
QueryStructReq queryStructReq = convertQueryStruct();
SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (optimizationConfig.isUseS2SqlSwitch()) {
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getQuerySQL());
}
SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
String text = generateTableText(semanticQueryResp);
Map<String, Object> properties = parseInfo.getProperties();

117
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java
View File

@@ -2,40 +2,38 @@
package com.tencent.supersonic.chat.core.query.rule;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.chat.core.query.BaseSemanticQuery;
import com.tencent.supersonic.chat.core.query.QueryManager;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@Slf4j
@ToString
@@ -107,30 +105,20 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
parseInfo.setDataSet(semanticSchema.getDataSet(dataSetIds.iterator().next()));
Map<Long, List<SchemaElementMatch>> dim2Values = new HashMap<>();
Map<Long, List<SchemaElementMatch>> id2Values = new HashMap<>();
Map<Long, List<SchemaElementMatch>> tag2Values = new HashMap<>();
for (SchemaElementMatch schemaMatch : parseInfo.getElementMatches()) {
SchemaElement element = schemaMatch.getElement();
element.setOrder(1 - schemaMatch.getSimilarity());
switch (element.getType()) {
case ID:
SchemaElement entityElement = semanticSchema.getElement(SchemaElementType.ENTITY, element.getId());
if (entityElement != null) {
if (id2Values.containsKey(element.getId())) {
id2Values.get(element.getId()).add(schemaMatch);
} else {
id2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
addToValues(semanticSchema, SchemaElementType.ENTITY, id2Values, schemaMatch);
break;
case TAG_VALUE:
addToValues(semanticSchema, SchemaElementType.TAG, tag2Values, schemaMatch);
break;
case VALUE:
SchemaElement dimElement = semanticSchema.getElement(SchemaElementType.DIMENSION, element.getId());
if (dimElement != null) {
if (dim2Values.containsKey(element.getId())) {
dim2Values.get(element.getId()).add(schemaMatch);
} else {
dim2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
addToValues(semanticSchema, SchemaElementType.DIMENSION, dim2Values, schemaMatch);
break;
case DIMENSION:
parseInfo.getDimensions().add(element);
@@ -145,43 +133,53 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
}
}
if (!id2Values.isEmpty()) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : id2Values.entrySet()) {
addFilters(parseInfo, semanticSchema, entry, SchemaElementType.ENTITY);
}
}
addToFilters(id2Values, parseInfo, semanticSchema, SchemaElementType.ENTITY);
addToFilters(dim2Values, parseInfo, semanticSchema, SchemaElementType.DIMENSION);
addToFilters(tag2Values, parseInfo, semanticSchema, SchemaElementType.TAG);
}
if (!dim2Values.isEmpty()) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dim2Values.entrySet()) {
addFilters(parseInfo, semanticSchema, entry, SchemaElementType.DIMENSION);
private void addToFilters(Map<Long, List<SchemaElementMatch>> id2Values, SemanticParseInfo parseInfo,
SemanticSchema semanticSchema, SchemaElementType entity) {
if (Objects.isNull(id2Values) || id2Values.isEmpty()) {
return;
}
for (Entry<Long, List<SchemaElementMatch>> entry : id2Values.entrySet()) {
SchemaElement dimension = semanticSchema.getElement(entity, entry.getKey());
if (entry.getValue().size() == 1) {
SchemaElementMatch schemaMatch = entry.getValue().get(0);
QueryFilter dimensionFilter = new QueryFilter();
dimensionFilter.setValue(schemaMatch.getWord());
dimensionFilter.setBizName(dimension.getBizName());
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
dimensionFilter.setElementID(schemaMatch.getElement().getId());
parseInfo.setEntity(semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey()));
parseInfo.getDimensionFilters().add(dimensionFilter);
} else {
QueryFilter dimensionFilter = new QueryFilter();
List<String> vals = new ArrayList<>();
entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
dimensionFilter.setValue(vals);
dimensionFilter.setBizName(dimension.getBizName());
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.IN);
dimensionFilter.setElementID(entry.getKey());
parseInfo.getDimensionFilters().add(dimensionFilter);
}
}
}
private void addFilters(SemanticParseInfo parseInfo, SemanticSchema semanticSchema,
Entry<Long, List<SchemaElementMatch>> entry, SchemaElementType elementType) {
SchemaElement dimension = semanticSchema.getElement(elementType, entry.getKey());
if (entry.getValue().size() == 1) {
SchemaElementMatch schemaMatch = entry.getValue().get(0);
QueryFilter dimensionFilter = new QueryFilter();
dimensionFilter.setValue(schemaMatch.getWord());
dimensionFilter.setBizName(dimension.getBizName());
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
dimensionFilter.setElementID(schemaMatch.getElement().getId());
parseInfo.getDimensionFilters().add(dimensionFilter);
parseInfo.setEntity(semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey()));
} else {
QueryFilter dimensionFilter = new QueryFilter();
List<String> vals = new ArrayList<>();
entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
dimensionFilter.setValue(vals);
dimensionFilter.setBizName(dimension.getBizName());
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.IN);
dimensionFilter.setElementID(entry.getKey());
parseInfo.getDimensionFilters().add(dimensionFilter);
private void addToValues(SemanticSchema semanticSchema, SchemaElementType entity,
Map<Long, List<SchemaElementMatch>> id2Values, SchemaElementMatch schemaMatch) {
SchemaElement element = schemaMatch.getElement();
SchemaElement entityElement = semanticSchema.getElement(entity, element.getId());
if (entityElement != null) {
if (id2Values.containsKey(element.getId())) {
id2Values.get(element.getId()).add(schemaMatch);
} else {
id2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
}
@@ -199,11 +197,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
QueryResult queryResult = new QueryResult();
QueryStructReq queryStructReq = convertQueryStruct();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (optimizationConfig.isUseS2SqlSwitch()) {
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setCorrectS2SQL(parseInfo.getSqlInfo().getCorrectS2SQL());
}
SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
if (queryResp != null) {

8
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.core.query.rule.tag;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG;
import org.springframework.stereotype.Component;
@@ -14,7 +14,7 @@ public class TagDetailQuery extends TagSemanticQuery {
public TagDetailQuery() {
super();
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1)
queryMatcher.addOption(TAG, REQUIRED, AT_LEAST, 1)
.addOption(ID, REQUIRED, AT_LEAST, 1);
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java
View File

@@ -1,8 +1,10 @@
package com.tencent.supersonic.chat.core.query.rule.tag;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG_VALUE;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -16,7 +18,8 @@ public class TagFilterQuery extends TagListQuery {
public TagFilterQuery() {
super();
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1);
queryMatcher.addOption(TAG, OPTIONAL, AT_LEAST, 0);
queryMatcher.addOption(TAG_VALUE, REQUIRED, AT_LEAST, 1);
}
@Override

1
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java
View File

@@ -30,6 +30,7 @@ public class DataSetSchemaBuilder {
public static DataSetSchema build(DataSetSchemaResp resp) {
DataSetSchema dataSetSchema = new DataSetSchema();
dataSetSchema.setQueryConfig(resp.getQueryConfig());
dataSetSchema.setQueryType(resp.getQueryType());
SchemaElement dataSet = SchemaElement.builder()
.dataSet(resp.getId())
.id(resp.getId())

21
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java
View File

@@ -4,31 +4,28 @@ import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import java.util.List;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
@Slf4j
public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@@ -41,14 +38,10 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@SneakyThrows
@Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QuerySqlReq querySqlReq = new QuerySqlReq();
querySqlReq.setSql(queryStructReq.getCorrectS2SQL());
querySqlReq.setDataSetId(queryStructReq.getDataSetId());
querySqlReq.setParams(new ArrayList<>());
return queryByS2SQL(querySqlReq, user);
}
queryService = ContextUtils.getBean(QueryService.class);
if (queryStructReq.isConvertToSql()) {
return queryService.queryByReq(queryStructReq.convert(), user);
}
return queryService.queryByReq(queryStructReq, user);
}

38
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java
View File

@@ -1,5 +1,9 @@
package com.tencent.supersonic.chat.core.query.semantic;
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
import com.github.pagehelper.PageInfo;
import com.google.gson.Gson;
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
@@ -18,16 +22,20 @@ import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import java.net.URI;
import java.net.URL;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.core.ParameterizedTypeReference;
@@ -39,17 +47,6 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
@Slf4j
public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
@@ -67,18 +64,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
@Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QuerySqlReq querySqlReq = new QuerySqlReq();
querySqlReq.setSql(queryStructReq.getCorrectS2SQL());
querySqlReq.setModelIds(queryStructReq.getModelIdSet());
querySqlReq.setParams(new ArrayList<>());
return queryByS2SQL(querySqlReq, user);
}
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByStructPath(),
new Gson().toJson(queryStructReq));
return queryByS2SQL(queryStructReq.convert(), user);
}
@Override

14
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.core.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.core.query.QueryManager;
@@ -42,10 +43,7 @@ public class QueryReqBuilder {
queryStructReq.setQueryType(parseInfo.getQueryType());
queryStructReq.setDateInfo(rewrite2Between(parseInfo.getDateInfo()));
List<Filter> dimensionFilters = parseInfo.getDimensionFilters().stream()
.filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName()))
.map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue()))
.collect(Collectors.toList());
List<Filter> dimensionFilters = getFilters(parseInfo.getDimensionFilters());
queryStructReq.setDimensionFilters(dimensionFilters);
List<Filter> metricFilters = parseInfo.getMetricFilters().stream()
@@ -72,6 +70,14 @@ public class QueryReqBuilder {
return queryStructReq;
}
private static List<Filter> getFilters(Set<QueryFilter> queryFilters) {
List<Filter> dimensionFilters = queryFilters.stream()
.filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName()))
.map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue()))
.collect(Collectors.toList());
return dimensionFilters;
}
private static void deletionDuplicated(QueryStructReq queryStructReq) {
if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) && queryStructReq.getGroups().size() > 1) {
Set<String> groups = new HashSet<>();

4
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java
View File

@@ -26,8 +26,8 @@ public class S2SqlDateHelper {
return getDefaultDate(defaultDate, tagTypeTimeDefaultConfig).getLeft();
}
public static Pair<String, String> getStartEndDate(QueryContext queryContext,
Long dataSetId, QueryType queryType) {
public static Pair<String, String> getStartEndDate(QueryContext queryContext, Long dataSetId,
QueryType queryType) {
String defaultDate = DateUtils.getBeforeDate(0);
if (Objects.isNull(dataSetId)) {
return Pair.of(defaultDate, defaultDate);

1
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java
View File

@@ -131,6 +131,7 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField)));
queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult));
queryStructReq.setConvertToSql(false);
SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
MetricInfo metricInfo = new MetricInfo();

2
common/src/main/java/com/tencent/supersonic/common/util/SqlFilterUtils.java
View File

@@ -65,7 +65,7 @@ public class SqlFilterUtils {
joiner.add(SPACE + dealFilter(filter, isBizName) + SPACE);
}
});
log.info("getWhereClause, where sql : {}", joiner.toString());
log.info("getWhereClause, where sql : {}", joiner);
return joiner.toString();
}

8
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DataSetReq.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import lombok.Data;
import java.util.List;
import lombok.Data;
@Data
public class DataSetReq extends SchemaItem {
@@ -22,6 +22,8 @@ public class DataSetReq extends SchemaItem {
private List<String> adminOrgs;
private QueryType queryType;
public String getAdmin() {
if (admins == null) {
return null;

11
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
View File

@@ -53,16 +53,7 @@ public class QueryStructReq extends SemanticQueryReq {
private DateConf dateInfo;
private Long limit = 2000L;
private QueryType queryType = QueryType.ID;
/**
* Later deleted for compatibility only
*/
private String s2SQL;
/**
* Later deleted for compatibility only
*/
private String correctS2SQL;
private boolean convertToSql = true;
public List<String> getGroups() {
if (!CollectionUtils.isEmpty(this.groups)) {

12
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DataSetResp.java
View File

@@ -1,17 +1,17 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig;
import lombok.Data;
import org.springframework.util.CollectionUtils;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import lombok.Data;
import org.springframework.util.CollectionUtils;
@Data
public class DataSetResp extends SchemaItem {
@@ -30,6 +30,8 @@ public class DataSetResp extends SchemaItem {
private QueryConfig queryConfig;
private QueryType queryType;
public List<Long> getAllMetrics() {
return getDataSetModelConfigs().stream().map(DataSetModelConfig::getMetrics)
.flatMap(Collection::stream).collect(Collectors.toList());

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
import lombok.Data;
@Data
@TableName("s2_data_set")
@@ -41,4 +41,6 @@ public class DataSetDO {
private String adminOrg;
private String queryType;
}

24
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java
View File

@@ -8,10 +8,12 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.DataSetDetail;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
@@ -30,6 +32,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
@@ -168,6 +171,7 @@ public class DataSetServiceImpl
dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg())
? Lists.newArrayList() : Arrays.asList(dataSetDO.getAdminOrg().split(",")));
dataSetResp.setTypeEnum(TypeEnums.DATASET);
dataSetResp.setQueryType(QueryType.valueOf(dataSetDO.getQueryType()));
return dataSetResp;
}
@@ -176,6 +180,8 @@ public class DataSetServiceImpl
BeanMapper.mapper(dataSetReq, dataSetDO);
dataSetDO.setDataSetDetail(JSONObject.toJSONString(dataSetReq.getDataSetDetail()));
dataSetDO.setQueryConfig(JSONObject.toJSONString(dataSetReq.getQueryConfig()));
QueryType queryType = getQueryType(dataSetReq);
dataSetDO.setQueryType(queryType.name());
return dataSetDO;
}
@@ -188,6 +194,24 @@ public class DataSetServiceImpl
return queryReq;
}
private QueryType getQueryType(DataSetReq dataSetReq) {
QueryType queryType = dataSetReq.getQueryType();
if (Objects.nonNull(queryType)) {
return queryType;
}
List<DataSetModelConfig> dataSetModelConfigs = dataSetReq.getDataSetDetail().getDataSetModelConfigs();
if (CollectionUtils.isEmpty(dataSetModelConfigs)) {
return QueryType.METRIC;
}
Set<DataSetModelConfig> collect = dataSetModelConfigs.stream()
.filter(config -> !CollectionUtils.isEmpty(config.getTagIds())).collect(Collectors.toSet());
if (CollectionUtils.isEmpty(collect)) {
return QueryType.METRIC;
}
return QueryType.TAG;
}
public static boolean checkAdminPermission(User user, DataSetResp dataSetResp) {
List<String> admins = dataSetResp.getAdmins();
if (user.isSuperAdmin()) {

5
launchers/standalone/src/main/resources/config.update/sql-update.sql
View File

@@ -248,4 +248,7 @@ COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';
alter table s2_view rename to s2_data_set;
alter table s2_query_stat_info change view_id data_set_id bigint;
alter table s2_plugin change `view` data_set varchar(200);
alter table s2_data_set change view_detail data_set_detail text;
alter table s2_data_set change view_detail data_set_detail text;
--20240311
alter table s2_data_set add column query_type varchar(100) DEFAULT NULL;

3
launchers/standalone/src/main/resources/data/dictionary/custom/TagValue_4_1.txt Normal file
View File

@@ -0,0 +1,3 @@
内地 _4_1_tv 100
欧美 _4_1_tv 100
港台 _4_1_tv 100

2
launchers/standalone/src/main/resources/data/dictionary/custom/TagValue_4_2.txt Normal file
View File

@@ -0,0 +1,2 @@
流行 _4_2_tv 100
国风 _4_2_tv 100

3
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -571,7 +571,8 @@ CREATE TABLE IF NOT EXISTS `s2_data_set` (
updated_by VARCHAR(255),
query_config VARCHAR(3000),
`admin` varchar(3000) DEFAULT NULL,
`admin_org` varchar(3000) DEFAULT NULL
`admin_org` varchar(3000) DEFAULT NULL,
`query_type` varchar(100) DEFAULT NULL
);
CREATE TABLE IF NOT EXISTS `s2_tag` (

3
launchers/standalone/src/test/resources/data/dictionary/custom/TagValue_4_1.txt Normal file
View File

@@ -0,0 +1,3 @@
内地 _4_1_tv 100
欧美 _4_1_tv 100
港台 _4_1_tv 100

2
launchers/standalone/src/test/resources/data/dictionary/custom/TagValue_4_2.txt Normal file
View File

@@ -0,0 +1,2 @@
流行 _4_2_tv 100
国风 _4_2_tv 100

3
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -572,7 +572,8 @@ CREATE TABLE IF NOT EXISTS `s2_data_set` (
updated_by VARCHAR(255),
query_config VARCHAR(3000),
`admin` varchar(3000) DEFAULT NULL,
`admin_org` varchar(3000) DEFAULT NULL
`admin_org` varchar(3000) DEFAULT NULL,
`query_type` varchar(100) DEFAULT NULL
);
CREATE TABLE IF NOT EXISTS `s2_tag` (