topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-24 10:27:28 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Dataset supports query mode settings, and the chat layer supports tag mode (#802)

Browse Source
This commit is contained in:
lexluo09
2024-03-12 12:15:21 +08:00
committed by GitHub
parent 1d91a972da
commit ae7acb4817
26 changed files with 171 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java
View File

@@ -66,8 +66,10 @@ public class WhereCorrector extends BaseSemanticCorrector {
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
List<String> whereFields = SqlSelectHelper.getWhereFields(correctS2SQL);
if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
Pair<String, String> startEndDate = S2SqlDateHelper.getStartEndDate(queryContext,
semanticParseInfo.getDataSetId(), semanticParseInfo.getQueryType());
if (StringUtils.isNotBlank(startEndDate.getLeft())
&& StringUtils.isNotBlank(startEndDate.getRight())) {
correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL);

64
chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java
View File

@@ -1,26 +1,14 @@
package com.tencent.supersonic.chat.core.parser;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.DataSetSchema;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.chat.core.query.SemanticQuery;
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery;
import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
/**
* QueryTypeParser resolves query type as either METRIC or TAG, or ID.
@@ -38,50 +26,12 @@ public class QueryTypeParser implements SemanticParser {
// 1.init S2SQL
semanticQuery.initS2Sql(queryContext.getSemanticSchema(), user);
// 2.set queryType
QueryType queryType = getQueryType(queryContext, semanticQuery);
semanticQuery.getParseInfo().setQueryType(queryType);
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
Long dataSetId = parseInfo.getDataSetId();
DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId);
parseInfo.setQueryType(dataSetSchema.getQueryType());
}
}
private QueryType getQueryType(QueryContext queryContext, SemanticQuery semanticQuery) {
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
SqlInfo sqlInfo = parseInfo.getSqlInfo();
if (Objects.isNull(sqlInfo) || StringUtils.isBlank(sqlInfo.getS2SQL())) {
return QueryType.ID;
}
//1. entity queryType
Long dataSetId = parseInfo.getDataSetId();
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
if (semanticQuery instanceof RuleSemanticQuery || semanticQuery instanceof LLMSqlQuery) {
//If all the fields in the SELECT statement are of tag type.
List<String> whereFields = SqlSelectHelper.getWhereFields(sqlInfo.getS2SQL())
.stream().filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(whereFields)) {
Set<String> ids = semanticSchema.getEntities(dataSetId).stream().map(SchemaElement::getName)
.collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(ids) && ids.stream().anyMatch(whereFields::contains)) {
return QueryType.ID;
}
Set<String> tags = semanticSchema.getTags(dataSetId).stream().map(SchemaElement::getName)
.collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(tags) && tags.containsAll(whereFields)) {
return QueryType.TAG;
}
}
}
//2. metric queryType
List<String> selectFields = SqlSelectHelper.getSelectFields(sqlInfo.getS2SQL());
List<SchemaElement> metrics = semanticSchema.getMetrics(dataSetId);
if (CollectionUtils.isNotEmpty(metrics)) {
Set<String> metricNameSet = metrics.stream().map(SchemaElement::getName).collect(Collectors.toSet());
boolean containMetric = selectFields.stream().anyMatch(metricNameSet::contains);
if (containMetric) {
return QueryType.METRIC;
}
}
return QueryType.ID;
}
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java
View File

@@ -2,23 +2,21 @@ package com.tencent.supersonic.chat.core.query.llm.analytics;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.query.QueryManager;
import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.util.HashMap;
@@ -53,12 +51,6 @@ public class MetricAnalyzeQuery extends LLMSemanticQuery {
QueryStructReq queryStructReq = convertQueryStruct();
SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (optimizationConfig.isUseS2SqlSwitch()) {
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getQuerySQL());
}
SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
String text = generateTableText(semanticQueryResp);
Map<String, Object> properties = parseInfo.getProperties();

117
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java
View File

@@ -2,40 +2,38 @@
package com.tencent.supersonic.chat.core.query.rule;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.chat.core.query.BaseSemanticQuery;
import com.tencent.supersonic.chat.core.query.QueryManager;
import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@Slf4j
@ToString
@@ -107,30 +105,20 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
parseInfo.setDataSet(semanticSchema.getDataSet(dataSetIds.iterator().next()));
Map<Long, List<SchemaElementMatch>> dim2Values = new HashMap<>();
Map<Long, List<SchemaElementMatch>> id2Values = new HashMap<>();
Map<Long, List<SchemaElementMatch>> tag2Values = new HashMap<>();
for (SchemaElementMatch schemaMatch : parseInfo.getElementMatches()) {
SchemaElement element = schemaMatch.getElement();
element.setOrder(1 - schemaMatch.getSimilarity());
switch (element.getType()) {
case ID:
SchemaElement entityElement = semanticSchema.getElement(SchemaElementType.ENTITY, element.getId());
if (entityElement != null) {
if (id2Values.containsKey(element.getId())) {
id2Values.get(element.getId()).add(schemaMatch);
} else {
id2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
addToValues(semanticSchema, SchemaElementType.ENTITY, id2Values, schemaMatch);
break;
case TAG_VALUE:
addToValues(semanticSchema, SchemaElementType.TAG, tag2Values, schemaMatch);
break;
case VALUE:
SchemaElement dimElement = semanticSchema.getElement(SchemaElementType.DIMENSION, element.getId());
if (dimElement != null) {
if (dim2Values.containsKey(element.getId())) {
dim2Values.get(element.getId()).add(schemaMatch);
} else {
dim2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
addToValues(semanticSchema, SchemaElementType.DIMENSION, dim2Values, schemaMatch);
break;
case DIMENSION:
parseInfo.getDimensions().add(element);
@@ -145,43 +133,53 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
}
}
if (!id2Values.isEmpty()) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : id2Values.entrySet()) {
addFilters(parseInfo, semanticSchema, entry, SchemaElementType.ENTITY);
}
}
addToFilters(id2Values, parseInfo, semanticSchema, SchemaElementType.ENTITY);
addToFilters(dim2Values, parseInfo, semanticSchema, SchemaElementType.DIMENSION);
addToFilters(tag2Values, parseInfo, semanticSchema, SchemaElementType.TAG);
}
if (!dim2Values.isEmpty()) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dim2Values.entrySet()) {
addFilters(parseInfo, semanticSchema, entry, SchemaElementType.DIMENSION);
private void addToFilters(Map<Long, List<SchemaElementMatch>> id2Values, SemanticParseInfo parseInfo,
SemanticSchema semanticSchema, SchemaElementType entity) {
if (Objects.isNull(id2Values) || id2Values.isEmpty()) {
return;
}
for (Entry<Long, List<SchemaElementMatch>> entry : id2Values.entrySet()) {
SchemaElement dimension = semanticSchema.getElement(entity, entry.getKey());
if (entry.getValue().size() == 1) {
SchemaElementMatch schemaMatch = entry.getValue().get(0);
QueryFilter dimensionFilter = new QueryFilter();
dimensionFilter.setValue(schemaMatch.getWord());
dimensionFilter.setBizName(dimension.getBizName());
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
dimensionFilter.setElementID(schemaMatch.getElement().getId());
parseInfo.setEntity(semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey()));
parseInfo.getDimensionFilters().add(dimensionFilter);
} else {
QueryFilter dimensionFilter = new QueryFilter();
List<String> vals = new ArrayList<>();
entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
dimensionFilter.setValue(vals);
dimensionFilter.setBizName(dimension.getBizName());
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.IN);
dimensionFilter.setElementID(entry.getKey());
parseInfo.getDimensionFilters().add(dimensionFilter);
}
}
}
private void addFilters(SemanticParseInfo parseInfo, SemanticSchema semanticSchema,
Entry<Long, List<SchemaElementMatch>> entry, SchemaElementType elementType) {
SchemaElement dimension = semanticSchema.getElement(elementType, entry.getKey());
if (entry.getValue().size() == 1) {
SchemaElementMatch schemaMatch = entry.getValue().get(0);
QueryFilter dimensionFilter = new QueryFilter();
dimensionFilter.setValue(schemaMatch.getWord());
dimensionFilter.setBizName(dimension.getBizName());
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
dimensionFilter.setElementID(schemaMatch.getElement().getId());
parseInfo.getDimensionFilters().add(dimensionFilter);
parseInfo.setEntity(semanticSchema.getElement(SchemaElementType.ENTITY, entry.getKey()));
} else {
QueryFilter dimensionFilter = new QueryFilter();
List<String> vals = new ArrayList<>();
entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
dimensionFilter.setValue(vals);
dimensionFilter.setBizName(dimension.getBizName());
dimensionFilter.setName(dimension.getName());
dimensionFilter.setOperator(FilterOperatorEnum.IN);
dimensionFilter.setElementID(entry.getKey());
parseInfo.getDimensionFilters().add(dimensionFilter);
private void addToValues(SemanticSchema semanticSchema, SchemaElementType entity,
Map<Long, List<SchemaElementMatch>> id2Values, SchemaElementMatch schemaMatch) {
SchemaElement element = schemaMatch.getElement();
SchemaElement entityElement = semanticSchema.getElement(entity, element.getId());
if (entityElement != null) {
if (id2Values.containsKey(element.getId())) {
id2Values.get(element.getId()).add(schemaMatch);
} else {
id2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
}
@@ -199,11 +197,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
QueryResult queryResult = new QueryResult();
QueryStructReq queryStructReq = convertQueryStruct();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (optimizationConfig.isUseS2SqlSwitch()) {
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setCorrectS2SQL(parseInfo.getSqlInfo().getCorrectS2SQL());
}
SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
if (queryResp != null) {

8
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.core.query.rule.tag;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG;
import org.springframework.stereotype.Component;
@@ -14,7 +14,7 @@ public class TagDetailQuery extends TagSemanticQuery {
public TagDetailQuery() {
super();
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1)
queryMatcher.addOption(TAG, REQUIRED, AT_LEAST, 1)
.addOption(ID, REQUIRED, AT_LEAST, 1);
}

7
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java
View File

@@ -1,8 +1,10 @@
package com.tencent.supersonic.chat.core.query.rule.tag;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TAG_VALUE;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -16,7 +18,8 @@ public class TagFilterQuery extends TagListQuery {
public TagFilterQuery() {
super();
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1);
queryMatcher.addOption(TAG, OPTIONAL, AT_LEAST, 0);
queryMatcher.addOption(TAG_VALUE, REQUIRED, AT_LEAST, 1);
}
@Override

1
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/DataSetSchemaBuilder.java
View File

@@ -30,6 +30,7 @@ public class DataSetSchemaBuilder {
public static DataSetSchema build(DataSetSchemaResp resp) {
DataSetSchema dataSetSchema = new DataSetSchema();
dataSetSchema.setQueryConfig(resp.getQueryConfig());
dataSetSchema.setQueryType(resp.getQueryType());
SchemaElement dataSet = SchemaElement.builder()
.dataSet(resp.getId())
.id(resp.getId())

21
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java
View File

@@ -4,31 +4,28 @@ import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.DataSetFilterReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import java.util.List;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
@Slf4j
public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@@ -41,14 +38,10 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@SneakyThrows
@Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QuerySqlReq querySqlReq = new QuerySqlReq();
querySqlReq.setSql(queryStructReq.getCorrectS2SQL());
querySqlReq.setDataSetId(queryStructReq.getDataSetId());
querySqlReq.setParams(new ArrayList<>());
return queryByS2SQL(querySqlReq, user);
}
queryService = ContextUtils.getBean(QueryService.class);
if (queryStructReq.isConvertToSql()) {
return queryService.queryByReq(queryStructReq.convert(), user);
}
return queryService.queryByReq(queryStructReq, user);
}

38
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java
View File

@@ -1,5 +1,9 @@
package com.tencent.supersonic.chat.core.query.semantic;
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
import com.github.pagehelper.PageInfo;
import com.google.gson.Gson;
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
@@ -18,16 +22,20 @@ import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import java.net.URI;
import java.net.URL;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.core.ParameterizedTypeReference;
@@ -39,17 +47,6 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
@Slf4j
public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
@@ -67,18 +64,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
@Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QuerySqlReq querySqlReq = new QuerySqlReq();
querySqlReq.setSql(queryStructReq.getCorrectS2SQL());
querySqlReq.setModelIds(queryStructReq.getModelIdSet());
querySqlReq.setParams(new ArrayList<>());
return queryByS2SQL(querySqlReq, user);
}
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByStructPath(),
new Gson().toJson(queryStructReq));
return queryByS2SQL(queryStructReq.convert(), user);
}
@Override

14
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.core.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.core.query.QueryManager;
@@ -42,10 +43,7 @@ public class QueryReqBuilder {
queryStructReq.setQueryType(parseInfo.getQueryType());
queryStructReq.setDateInfo(rewrite2Between(parseInfo.getDateInfo()));
List<Filter> dimensionFilters = parseInfo.getDimensionFilters().stream()
.filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName()))
.map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue()))
.collect(Collectors.toList());
List<Filter> dimensionFilters = getFilters(parseInfo.getDimensionFilters());
queryStructReq.setDimensionFilters(dimensionFilters);
List<Filter> metricFilters = parseInfo.getMetricFilters().stream()
@@ -72,6 +70,14 @@ public class QueryReqBuilder {
return queryStructReq;
}
private static List<Filter> getFilters(Set<QueryFilter> queryFilters) {
List<Filter> dimensionFilters = queryFilters.stream()
.filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName()))
.map(chatFilter -> new Filter(chatFilter.getBizName(), chatFilter.getOperator(), chatFilter.getValue()))
.collect(Collectors.toList());
return dimensionFilters;
}
private static void deletionDuplicated(QueryStructReq queryStructReq) {
if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) && queryStructReq.getGroups().size() > 1) {
Set<String> groups = new HashSet<>();

4
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/S2SqlDateHelper.java
View File

@@ -26,8 +26,8 @@ public class S2SqlDateHelper {
return getDefaultDate(defaultDate, tagTypeTimeDefaultConfig).getLeft();
}
public static Pair<String, String> getStartEndDate(QueryContext queryContext,
Long dataSetId, QueryType queryType) {
public static Pair<String, String> getStartEndDate(QueryContext queryContext, Long dataSetId,
QueryType queryType) {
String defaultDate = DateUtils.getBeforeDate(0);
if (Objects.isNull(dataSetId)) {
return Pair.of(defaultDate, defaultDate);