mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 11:07:06 +00:00
(improvement)(chat) Rules and large model queries should be queried using s2sql (#334)
This commit is contained in:
@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.api.component;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import org.apache.calcite.sql.parser.SqlParseException;
|
||||
|
||||
/**
|
||||
@@ -15,7 +15,7 @@ public interface SemanticQuery {
|
||||
|
||||
QueryResult execute(User user) throws SqlParseException;
|
||||
|
||||
ExplainResp explain(User user);
|
||||
SqlInfo explain(User user);
|
||||
|
||||
SemanticParseInfo getParseInfo();
|
||||
|
||||
|
||||
@@ -68,4 +68,12 @@ public class SemanticSchema implements Serializable {
|
||||
modelSchemaList.stream().forEach(d -> entities.add(d.getEntity()));
|
||||
return entities;
|
||||
}
|
||||
|
||||
public Map<String, String> getBizNameToName(Long modelId) {
|
||||
List<SchemaElement> allElements = new ArrayList<>();
|
||||
allElements.addAll(getDimensions(modelId));
|
||||
allElements.addAll(getMetrics(modelId));
|
||||
return allElements.stream()
|
||||
.collect(Collectors.toMap(a -> a.getBizName(), a -> a.getName(), (k1, k2) -> k1));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,8 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
||||
@@ -47,7 +47,14 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
||||
return elements.stream();
|
||||
})
|
||||
.collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1));
|
||||
result.put(DateUtils.DATE_FIELD, DateUtils.DATE_FIELD);
|
||||
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
|
||||
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
|
||||
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
|
||||
|
||||
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
|
||||
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
|
||||
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -61,7 +68,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
||||
}
|
||||
|
||||
needAddFields.removeAll(selectFields);
|
||||
needAddFields.remove(DateUtils.DATE_FIELD);
|
||||
needAddFields.remove(TimeDimensionEnum.DAY.getChName());
|
||||
String replaceFields = SqlParserAddHelper.addFieldsToSelect(sql, new ArrayList<>(needAddFields));
|
||||
semanticCorrectInfo.setSql(replaceFields);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.tencent.supersonic.chat.corrector;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
|
||||
public interface CorrectorService {
|
||||
|
||||
SemanticCorrectInfo correctorSql(QueryFilters queryFilters, SemanticParseInfo parseInfo, String sql);
|
||||
|
||||
void addS2QLAndLoginSql(QueryStructReq queryStructReq, SemanticParseInfo parseInfo);
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package com.tencent.supersonic.chat.corrector;
|
||||
|
||||
import com.tencent.supersonic.chat.api.component.SemanticCorrector;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class CorrectorServiceImpl implements CorrectorService {
|
||||
|
||||
@Autowired
|
||||
private SchemaService schemaService;
|
||||
|
||||
public SemanticCorrectInfo correctorSql(QueryFilters queryFilters, SemanticParseInfo parseInfo, String sql) {
|
||||
|
||||
SemanticCorrectInfo correctInfo = SemanticCorrectInfo.builder()
|
||||
.queryFilters(queryFilters).sql(sql)
|
||||
.parseInfo(parseInfo).build();
|
||||
|
||||
List<SemanticCorrector> corrections = ComponentFactory.getSqlCorrections();
|
||||
|
||||
corrections.forEach(correction -> {
|
||||
try {
|
||||
correction.correct(correctInfo);
|
||||
log.info("sqlCorrection:{} sql:{}", correction.getClass().getSimpleName(), correctInfo.getSql());
|
||||
} catch (Exception e) {
|
||||
log.error(String.format("correct error,correctInfo:%s", correctInfo), e);
|
||||
}
|
||||
});
|
||||
return correctInfo;
|
||||
}
|
||||
|
||||
|
||||
public void addS2QLAndLoginSql(QueryStructReq queryStructReq, SemanticParseInfo parseInfo) {
|
||||
convertBizNameToName(queryStructReq, parseInfo);
|
||||
QueryS2QLReq queryS2QLReq = queryStructReq.convert(queryStructReq);
|
||||
parseInfo.getSqlInfo().setS2QL(queryS2QLReq.getSql());
|
||||
queryStructReq.setS2QL(queryS2QLReq.getSql());
|
||||
|
||||
SemanticCorrectInfo semanticCorrectInfo = correctorSql(new QueryFilters(), parseInfo,
|
||||
queryS2QLReq.getSql());
|
||||
parseInfo.getSqlInfo().setLogicSql(semanticCorrectInfo.getSql());
|
||||
|
||||
queryStructReq.setLogicSql(semanticCorrectInfo.getSql());
|
||||
}
|
||||
|
||||
|
||||
private void convertBizNameToName(QueryStructReq queryStructReq, SemanticParseInfo parseInfo) {
|
||||
Map<String, String> bizNameToName = schemaService.getSemanticSchema()
|
||||
.getBizNameToName(queryStructReq.getModelId());
|
||||
List<Order> orders = queryStructReq.getOrders();
|
||||
if (CollectionUtils.isNotEmpty(orders)) {
|
||||
for (Order order : orders) {
|
||||
order.setColumn(bizNameToName.get(order.getColumn()));
|
||||
}
|
||||
}
|
||||
List<Aggregator> aggregators = queryStructReq.getAggregators();
|
||||
if (CollectionUtils.isNotEmpty(aggregators)) {
|
||||
for (Aggregator aggregator : aggregators) {
|
||||
aggregator.setColumn(bizNameToName.get(aggregator.getColumn()));
|
||||
}
|
||||
}
|
||||
List<String> groups = queryStructReq.getGroups();
|
||||
if (CollectionUtils.isNotEmpty(groups)) {
|
||||
groups = groups.stream().map(group -> bizNameToName.get(group)).collect(Collectors.toList());
|
||||
queryStructReq.setGroups(groups);
|
||||
}
|
||||
List<Filter> dimensionFilters = queryStructReq.getDimensionFilters();
|
||||
if (CollectionUtils.isNotEmpty(dimensionFilters)) {
|
||||
dimensionFilters.stream().forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName())));
|
||||
}
|
||||
List<Filter> metricFilters = queryStructReq.getMetricFilters();
|
||||
if (CollectionUtils.isNotEmpty(dimensionFilters)) {
|
||||
metricFilters.stream().forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName())));
|
||||
}
|
||||
|
||||
queryStructReq.setModelName(parseInfo.getModelName());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.corrector;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
||||
@@ -44,7 +44,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
|
||||
return elements.stream();
|
||||
}
|
||||
).collect(Collectors.toSet());
|
||||
dimensions.add(DateUtils.DATE_FIELD);
|
||||
dimensions.add(TimeDimensionEnum.DAY.getChName());
|
||||
|
||||
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sql);
|
||||
|
||||
@@ -52,7 +52,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
|
||||
return;
|
||||
}
|
||||
// if only date in select not add group by.
|
||||
if (selectFields.size() == 1 && selectFields.contains(DateUtils.DATE_FIELD)) {
|
||||
if (selectFields.size() == 1 && selectFields.contains(TimeDimensionEnum.DAY.getChName())) {
|
||||
return;
|
||||
}
|
||||
if (SqlParserSelectHelper.hasGroupBy(sql)) {
|
||||
|
||||
@@ -7,8 +7,8 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||
import com.tencent.supersonic.chat.parser.llm.s2ql.S2QLDateHelper;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.common.util.StringUtil;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||
@@ -71,11 +71,11 @@ public class WhereCorrector extends BaseSemanticCorrector {
|
||||
private void addDateIfNotExist(SemanticCorrectInfo semanticCorrectInfo) {
|
||||
String sql = semanticCorrectInfo.getSql();
|
||||
List<String> whereFields = SqlParserSelectHelper.getWhereFields(sql);
|
||||
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(DateUtils.DATE_FIELD)) {
|
||||
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(TimeDimensionEnum.DAY.getChName())) {
|
||||
String currentDate = S2QLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId());
|
||||
if (StringUtils.isNotBlank(currentDate)) {
|
||||
sql = SqlParserAddHelper.addParenthesisToWhere(sql);
|
||||
sql = SqlParserAddHelper.addWhere(sql, DateUtils.DATE_FIELD, currentDate);
|
||||
sql = SqlParserAddHelper.addWhere(sql, TimeDimensionEnum.DAY.getChName(), currentDate);
|
||||
}
|
||||
}
|
||||
semanticCorrectInfo.setSql(sql);
|
||||
|
||||
@@ -24,7 +24,7 @@ import com.tencent.supersonic.chat.service.AgentService;
|
||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@@ -18,6 +18,7 @@ import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
|
||||
import com.tencent.supersonic.chat.service.AgentService;
|
||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
||||
@@ -118,7 +119,7 @@ public class LLMRequestService {
|
||||
String priorExts = getPriorExts(modelId, fieldNameList);
|
||||
llmReq.setPriorExts(priorExts);
|
||||
|
||||
fieldNameList.add(DateUtils.DATE_FIELD);
|
||||
fieldNameList.add(TimeDimensionEnum.DAY.getChName());
|
||||
llmSchema.setFieldNameList(fieldNameList);
|
||||
llmReq.setSchema(llmSchema);
|
||||
|
||||
|
||||
@@ -2,27 +2,27 @@ package com.tencent.supersonic.chat.parser.llm.s2ql;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticCorrector;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||
import com.tencent.supersonic.chat.corrector.CorrectorService;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
|
||||
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
|
||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
@@ -34,6 +34,7 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@@ -41,11 +42,15 @@ import org.springframework.util.CollectionUtils;
|
||||
@Service
|
||||
public class LLMResponseService {
|
||||
|
||||
@Autowired
|
||||
private CorrectorService correctorService;
|
||||
|
||||
public void addParseInfo(QueryContext queryCtx, ParseResult parseResult, String sql, Double weight) {
|
||||
|
||||
SemanticParseInfo parseInfo = getParseInfo(queryCtx, parseResult, weight);
|
||||
|
||||
SemanticCorrectInfo semanticCorrectInfo = getCorrectorSql(queryCtx, parseInfo, sql);
|
||||
QueryFilters queryFilters = queryCtx.getRequest().getQueryFilters();
|
||||
SemanticCorrectInfo semanticCorrectInfo = correctorService.correctorSql(queryFilters, parseInfo, sql);
|
||||
|
||||
parseInfo.getSqlInfo().setLogicSql(semanticCorrectInfo.getSql());
|
||||
|
||||
@@ -64,7 +69,7 @@ public class LLMResponseService {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
return allFields.stream()
|
||||
.filter(entry -> !DateUtils.DATE_FIELD.equalsIgnoreCase(entry))
|
||||
.filter(entry -> !TimeDimensionEnum.DAY.getChName().equalsIgnoreCase(entry))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@@ -140,7 +145,7 @@ public class LLMResponseService {
|
||||
|
||||
private DateConf getDateInfo(List<FilterExpression> filterExpressions) {
|
||||
List<FilterExpression> dateExpressions = filterExpressions.stream()
|
||||
.filter(expression -> DateUtils.DATE_FIELD.equalsIgnoreCase(expression.getFieldName()))
|
||||
.filter(expression -> TimeDimensionEnum.DAY.getChName().equalsIgnoreCase(expression.getFieldName()))
|
||||
.collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(dateExpressions)) {
|
||||
return new DateConf();
|
||||
@@ -182,24 +187,6 @@ public class LLMResponseService {
|
||||
return dateExpressions.size() > 1 && Objects.nonNull(dateExpressions.get(1).getFieldValue());
|
||||
}
|
||||
|
||||
private SemanticCorrectInfo getCorrectorSql(QueryContext queryCtx, SemanticParseInfo parseInfo, String sql) {
|
||||
|
||||
SemanticCorrectInfo correctInfo = SemanticCorrectInfo.builder()
|
||||
.queryFilters(queryCtx.getRequest().getQueryFilters()).sql(sql)
|
||||
.parseInfo(parseInfo).build();
|
||||
|
||||
List<SemanticCorrector> corrections = ComponentFactory.getSqlCorrections();
|
||||
|
||||
corrections.forEach(correction -> {
|
||||
try {
|
||||
correction.correct(correctInfo);
|
||||
log.info("sqlCorrection:{} sql:{}", correction.getClass().getSimpleName(), correctInfo.getSql());
|
||||
} catch (Exception e) {
|
||||
log.error(String.format("correct error,correctInfo:%s", correctInfo), e);
|
||||
}
|
||||
});
|
||||
return correctInfo;
|
||||
}
|
||||
|
||||
private SemanticParseInfo getParseInfo(QueryContext queryCtx, ParseResult parseResult, Double weight) {
|
||||
if (Objects.isNull(weight)) {
|
||||
|
||||
@@ -10,6 +10,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
|
||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
||||
import com.tencent.supersonic.chat.corrector.CorrectorService;
|
||||
import com.tencent.supersonic.chat.plugin.PluginManager;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
|
||||
@@ -29,6 +30,7 @@ import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.calcite.sql.parser.SqlParseException;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
@@ -40,6 +42,9 @@ public class MetricInterpretQuery extends PluginSemanticQuery {
|
||||
|
||||
public static final String QUERY_MODE = "METRIC_INTERPRET";
|
||||
|
||||
@Autowired
|
||||
private CorrectorService correctorService;
|
||||
|
||||
public MetricInterpretQuery() {
|
||||
QueryManager.register(this);
|
||||
}
|
||||
@@ -58,6 +63,9 @@ public class MetricInterpretQuery extends PluginSemanticQuery {
|
||||
|
||||
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
|
||||
queryStructReq.setUseS2qlSwitch(optimizationConfig.isUseS2qlSwitch());
|
||||
if (optimizationConfig.isUseS2qlSwitch()) {
|
||||
correctorService.addS2QLAndLoginSql(queryStructReq, parseInfo);
|
||||
}
|
||||
|
||||
QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user);
|
||||
String text = generateTableText(queryResultWithSchemaResp);
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
|
||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
@@ -42,7 +43,7 @@ public class S2QLQuery extends PluginSemanticQuery {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
String querySql = parseInfo.getSqlInfo().getLogicSql();
|
||||
QueryS2QLReq queryS2QLReq = getQueryS2QLReq(querySql);
|
||||
QueryS2QLReq queryS2QLReq = QueryReqBuilder.buildS2QLReq(querySql, parseInfo.getModelId());
|
||||
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2QL(queryS2QLReq, user);
|
||||
|
||||
log.info("queryByS2QL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql);
|
||||
@@ -64,22 +65,22 @@ public class S2QLQuery extends PluginSemanticQuery {
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
private QueryS2QLReq getQueryS2QLReq(String sql) {
|
||||
return QueryReqBuilder.buildS2QLReq(sql, parseInfo.getModelId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExplainResp explain(User user) {
|
||||
public SqlInfo explain(User user) {
|
||||
SqlInfo sqlInfo = parseInfo.getSqlInfo();
|
||||
ExplainSqlReq explainSqlReq = null;
|
||||
try {
|
||||
QueryS2QLReq queryS2QLReq = QueryReqBuilder.buildS2QLReq(sqlInfo.getLogicSql(), parseInfo.getModelId());
|
||||
explainSqlReq = ExplainSqlReq.builder()
|
||||
.queryTypeEnum(QueryTypeEnum.SQL)
|
||||
.queryReq(getQueryS2QLReq(parseInfo.getSqlInfo().getLogicSql()))
|
||||
.queryReq(queryS2QLReq)
|
||||
.build();
|
||||
return semanticInterpreter.explain(explainSqlReq, user);
|
||||
ExplainResp explain = semanticInterpreter.explain(explainSqlReq, user);
|
||||
sqlInfo.setQuerySql(explain.getSql());
|
||||
} catch (Exception e) {
|
||||
log.error("explain error explainSqlReq:{}", explainSqlReq, e);
|
||||
}
|
||||
return null;
|
||||
return sqlInfo;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.query.plugin;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@@ -20,7 +20,7 @@ public abstract class PluginSemanticQuery implements SemanticQuery {
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExplainResp explain(User user) {
|
||||
public SqlInfo explain(User user) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,9 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
||||
import com.tencent.supersonic.chat.corrector.CorrectorService;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
@@ -27,6 +29,7 @@ import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
@@ -201,7 +204,10 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
|
||||
|
||||
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
|
||||
queryStructReq.setUseS2qlSwitch(optimizationConfig.isUseS2qlSwitch());
|
||||
|
||||
if (optimizationConfig.isUseS2qlSwitch()) {
|
||||
CorrectorService correctorService = ContextUtils.getBean(CorrectorService.class);
|
||||
correctorService.addS2QLAndLoginSql(queryStructReq, parseInfo);
|
||||
}
|
||||
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
|
||||
|
||||
if (queryResp != null) {
|
||||
@@ -221,20 +227,27 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public ExplainResp explain(User user) {
|
||||
public SqlInfo explain(User user) {
|
||||
SqlInfo sqlInfo = parseInfo.getSqlInfo();
|
||||
ExplainSqlReq explainSqlReq = null;
|
||||
try {
|
||||
QueryStructReq queryStructReq = convertQueryStruct();
|
||||
CorrectorService correctorService = ContextUtils.getBean(CorrectorService.class);
|
||||
correctorService.addS2QLAndLoginSql(queryStructReq, parseInfo);
|
||||
|
||||
QueryS2QLReq queryS2QLReq = QueryReqBuilder.buildS2QLReq(sqlInfo.getLogicSql(), parseInfo.getModelId());
|
||||
explainSqlReq = ExplainSqlReq.builder()
|
||||
.queryTypeEnum(QueryTypeEnum.STRUCT)
|
||||
.queryReq(isMultiStructQuery()
|
||||
? convertQueryMultiStruct() : convertQueryStruct())
|
||||
.queryTypeEnum(QueryTypeEnum.SQL)
|
||||
.queryReq(queryS2QLReq)
|
||||
.build();
|
||||
return semanticInterpreter.explain(explainSqlReq, user);
|
||||
ExplainResp explain = semanticInterpreter.explain(explainSqlReq, user);
|
||||
sqlInfo.setQuerySql(explain.getSql());
|
||||
} catch (Exception e) {
|
||||
log.error("explain error explainSqlReq:{}", explainSqlReq, e);
|
||||
}
|
||||
return null;
|
||||
return sqlInfo;
|
||||
}
|
||||
|
||||
protected boolean isMultiStructQuery() {
|
||||
|
||||
@@ -6,26 +6,26 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ParseTimeCostDO;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
|
||||
import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
public class ExplainSqlParseResponder implements ParseResponder {
|
||||
public class SqlInfoParseResponder implements ParseResponder {
|
||||
|
||||
@Override
|
||||
public void fillResponse(ParseResp parseResp, QueryContext queryContext,
|
||||
List<ChatParseDO> chatParseDOS) {
|
||||
List<ChatParseDO> chatParseDOS) {
|
||||
QueryReq queryReq = queryContext.getRequest();
|
||||
Long startTime = System.currentTimeMillis();
|
||||
addExplainSql(queryReq, parseResp.getSelectedParses());
|
||||
addExplainSql(queryReq, parseResp.getCandidateParses());
|
||||
addSqlInfo(queryReq, parseResp.getSelectedParses());
|
||||
addSqlInfo(queryReq, parseResp.getCandidateParses());
|
||||
parseResp.setParseTimeCost(new ParseTimeCostDO());
|
||||
parseResp.getParseTimeCost().setSqlTime(System.currentTimeMillis() - startTime);
|
||||
if (!CollectionUtils.isEmpty(chatParseDOS)) {
|
||||
@@ -49,26 +49,26 @@ public class ExplainSqlParseResponder implements ParseResponder {
|
||||
}
|
||||
}
|
||||
|
||||
private void addExplainSql(QueryReq queryReq, List<SemanticParseInfo> semanticParseInfos) {
|
||||
private void addSqlInfo(QueryReq queryReq, List<SemanticParseInfo> semanticParseInfos) {
|
||||
if (CollectionUtils.isEmpty(semanticParseInfos)) {
|
||||
return;
|
||||
}
|
||||
semanticParseInfos.forEach(parseInfo -> {
|
||||
addExplainSql(queryReq, parseInfo);
|
||||
addSqlInfo(queryReq, parseInfo);
|
||||
});
|
||||
}
|
||||
|
||||
private void addExplainSql(QueryReq queryReq, SemanticParseInfo parseInfo) {
|
||||
private void addSqlInfo(QueryReq queryReq, SemanticParseInfo parseInfo) {
|
||||
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
|
||||
if (Objects.isNull(semanticQuery)) {
|
||||
return;
|
||||
}
|
||||
semanticQuery.setParseInfo(parseInfo);
|
||||
ExplainResp explain = semanticQuery.explain(queryReq.getUser());
|
||||
if (Objects.isNull(explain)) {
|
||||
SqlInfo sqlInfo = semanticQuery.explain(queryReq.getUser());
|
||||
if (Objects.isNull(sqlInfo)) {
|
||||
return;
|
||||
}
|
||||
parseInfo.getSqlInfo().setQuerySql(explain.getSql());
|
||||
parseInfo.setSqlInfo(sqlInfo);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -264,8 +264,6 @@ public class SemanticService {
|
||||
QueryResultWithSchemaResp queryResultWithColumns = null;
|
||||
try {
|
||||
QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo);
|
||||
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
|
||||
queryStructReq.setUseS2qlSwitch(optimizationConfig.isUseS2qlSwitch());
|
||||
queryResultWithColumns = semanticInterpreter.queryByStruct(queryStructReq, user);
|
||||
} catch (Exception e) {
|
||||
log.warn("setMainModel queryByStruct error, e:", e);
|
||||
|
||||
@@ -20,6 +20,7 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
|
||||
@@ -39,6 +40,8 @@ import com.tencent.supersonic.chat.utils.SolvedQueryManager;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
@@ -52,9 +55,8 @@ import com.tencent.supersonic.knowledge.dictionary.MultiCustomDictionary;
|
||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
@@ -65,8 +67,6 @@ import java.util.Objects;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.sf.jsqlparser.expression.Expression;
|
||||
import net.sf.jsqlparser.expression.LongValue;
|
||||
@@ -252,6 +252,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
queryResult.setQueryTimeCost(System.currentTimeMillis() - executeTime);
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
// save time cost data
|
||||
public void saveInfo(List<StatisticsDO> timeCostDOList,
|
||||
String queryText, Long queryId,
|
||||
@@ -329,6 +330,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
ChatContext context = chatService.getOrCreateContext(queryCtx.getChatId());
|
||||
return context.getParseInfo();
|
||||
}
|
||||
|
||||
//mainly used for executing after revising filters,for example:"fans_cnt>=100000"->"fans_cnt>500000",
|
||||
//"style='流行'"->"style in ['流行','爱国']"
|
||||
@Override
|
||||
@@ -371,9 +373,9 @@ public class QueryServiceImpl implements QueryService {
|
||||
log.info("correctorSql after replacing:{}", correctorSql);
|
||||
parseInfo.getSqlInfo().setLogicSql(correctorSql);
|
||||
semanticQuery.setParseInfo(parseInfo);
|
||||
ExplainResp explain = semanticQuery.explain(user);
|
||||
if (!Objects.isNull(explain)) {
|
||||
parseInfo.getSqlInfo().setQuerySql(explain.getSql());
|
||||
SqlInfo sqlInfo = semanticQuery.explain(user);
|
||||
if (!Objects.isNull(sqlInfo)) {
|
||||
parseInfo.setSqlInfo(sqlInfo);
|
||||
}
|
||||
}
|
||||
semanticQuery.setParseInfo(parseInfo);
|
||||
@@ -402,7 +404,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
return;
|
||||
}
|
||||
Map<String, String> map = new HashMap<>();
|
||||
String dateField = DateUtils.DATE_FIELD;
|
||||
String dateField = TimeDimensionEnum.DAY.getChName();
|
||||
if (queryData.getDateInfo().getUnit() > 1) {
|
||||
queryData.getDateInfo().setStartDate(DateUtils.getBeforeDate(queryData.getDateInfo().getUnit() + 1));
|
||||
queryData.getDateInfo().setEndDate(DateUtils.getBeforeDate(1));
|
||||
@@ -410,7 +412,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
// startDate equals to endDate
|
||||
if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) {
|
||||
for (FilterExpression filterExpression : filterExpressionList) {
|
||||
if (DateUtils.DATE_FIELD.equals(filterExpression.getFieldName())) {
|
||||
if (TimeDimensionEnum.DAY.getChName().equals(filterExpression.getFieldName())) {
|
||||
//sql where condition exists 'equals' operator about date,just replace
|
||||
if (filterExpression.getOperator().equals(FilterOperatorEnum.EQUALS)) {
|
||||
dateField = filterExpression.getFieldName();
|
||||
@@ -419,9 +421,9 @@ public class QueryServiceImpl implements QueryService {
|
||||
filedNameToValueMap.put(dateField, map);
|
||||
} else {
|
||||
// first remove,then add
|
||||
removeFieldNames.add(DateUtils.DATE_FIELD);
|
||||
removeFieldNames.add(TimeDimensionEnum.DAY.getChName());
|
||||
EqualsTo equalsTo = new EqualsTo();
|
||||
Column column = new Column(DateUtils.DATE_FIELD);
|
||||
Column column = new Column(TimeDimensionEnum.DAY.getChName());
|
||||
StringValue stringValue = new StringValue(queryData.getDateInfo().getStartDate());
|
||||
equalsTo.setLeftExpression(column);
|
||||
equalsTo.setRightExpression(stringValue);
|
||||
@@ -432,7 +434,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
}
|
||||
} else {
|
||||
for (FilterExpression filterExpression : filterExpressionList) {
|
||||
if (DateUtils.DATE_FIELD.equals(filterExpression.getFieldName())) {
|
||||
if (TimeDimensionEnum.DAY.getChName().equals(filterExpression.getFieldName())) {
|
||||
dateField = filterExpression.getFieldName();
|
||||
//just replace
|
||||
if (FilterOperatorEnum.GREATER_THAN_EQUALS.getValue().equals(filterExpression.getOperator())
|
||||
@@ -448,7 +450,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
filedNameToValueMap.put(dateField, map);
|
||||
// first remove,then add
|
||||
if (FilterOperatorEnum.EQUALS.getValue().equals(filterExpression.getOperator())) {
|
||||
removeFieldNames.add(DateUtils.DATE_FIELD);
|
||||
removeFieldNames.add(TimeDimensionEnum.DAY.getChName());
|
||||
GreaterThanEquals greaterThanEquals = new GreaterThanEquals();
|
||||
addTimeFilters(queryData.getDateInfo().getStartDate(), greaterThanEquals, addConditions);
|
||||
MinorThanEquals minorThanEquals = new MinorThanEquals();
|
||||
@@ -463,7 +465,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
public <T extends ComparisonOperator> void addTimeFilters(String date,
|
||||
T comparisonExpression,
|
||||
List<Expression> addConditions) {
|
||||
Column column = new Column(DateUtils.DATE_FIELD);
|
||||
Column column = new Column(TimeDimensionEnum.DAY.getChName());
|
||||
StringValue stringValue = new StringValue(date);
|
||||
comparisonExpression.setLeftExpression(column);
|
||||
comparisonExpression.setRightExpression(stringValue);
|
||||
@@ -508,6 +510,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// add in condition to sql where condition
|
||||
public void addWhereInFilters(QueryFilter dslQueryFilter,
|
||||
InExpression inExpression,
|
||||
@@ -536,6 +539,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// add where filter
|
||||
public <T extends ComparisonOperator> void addWhereFilters(QueryFilter dslQueryFilter,
|
||||
T comparisonExpression,
|
||||
|
||||
@@ -50,6 +50,7 @@ public class DictQueryHelper {
|
||||
@Value("${dimension.white.weight:60000000}")
|
||||
private Long dimensionWhiteWeight;
|
||||
|
||||
|
||||
public List<String> fetchDimValueSingle(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict,
|
||||
User user) {
|
||||
List<String> data = new ArrayList<>();
|
||||
|
||||
@@ -11,7 +11,7 @@ import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
@@ -25,6 +25,7 @@ import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
@@ -141,8 +142,14 @@ public class QueryReqBuilder {
|
||||
private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {
|
||||
List<Aggregator> aggregators = new ArrayList<>();
|
||||
if (metric != null) {
|
||||
String agg = (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)) ? ""
|
||||
: aggregateType.name();
|
||||
String agg = "";
|
||||
if (Objects.isNull(aggregateType) || aggregateType.equals(AggregateTypeEnum.NONE)) {
|
||||
if (StringUtils.isNotBlank(metric.getDefaultAgg())) {
|
||||
agg = metric.getDefaultAgg();
|
||||
}
|
||||
} else {
|
||||
agg = aggregateType.name();
|
||||
}
|
||||
aggregators.add(new Aggregator(metric.getBizName(), AggOperatorEnum.of(agg)));
|
||||
}
|
||||
return aggregators;
|
||||
|
||||
@@ -30,6 +30,7 @@ class QueryReqBuilderTest {
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setModelId(1L);
|
||||
queryStructReq.setNativeQuery(false);
|
||||
queryStructReq.setModelName("内容库");
|
||||
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setFunc(AggOperatorEnum.UNKNOWN);
|
||||
@@ -51,13 +52,13 @@ class QueryReqBuilderTest {
|
||||
|
||||
QueryS2QLReq queryS2QLReq = queryStructReq.convert(queryStructReq);
|
||||
Assert.assertEquals(
|
||||
"SELECT department, SUM(pv) FROM t_1 WHERE (sys_imp_date IN ('2023-08-01')) "
|
||||
"SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
|
||||
+ "GROUP BY department ORDER BY uv LIMIT 2000", queryS2QLReq.getSql());
|
||||
|
||||
queryStructReq.setNativeQuery(true);
|
||||
queryS2QLReq = queryStructReq.convert(queryStructReq);
|
||||
Assert.assertEquals(
|
||||
"SELECT department, pv FROM t_1 WHERE (sys_imp_date IN ('2023-08-01')) "
|
||||
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
|
||||
+ "ORDER BY uv LIMIT 2000",
|
||||
queryS2QLReq.getSql());
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ import com.tencent.supersonic.semantic.model.domain.DimensionService;
|
||||
import com.tencent.supersonic.semantic.model.domain.MetricService;
|
||||
import com.tencent.supersonic.semantic.query.service.QueryService;
|
||||
import com.tencent.supersonic.semantic.query.service.SchemaService;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -40,12 +41,13 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
|
||||
QueryS2QLReq queryS2QLReq = queryStructReq.convert(queryStructReq);
|
||||
if (queryStructReq.isUseS2qlSwitch() && StringUtils.isNotBlank(queryS2QLReq.getSql())) {
|
||||
log.info("queryStructReq convert to sql:{},queryStructReq:{}", queryS2QLReq.getSql(), queryStructReq);
|
||||
if (StringUtils.isNotBlank(queryStructReq.getLogicSql())) {
|
||||
QueryS2QLReq queryS2QLReq = new QueryS2QLReq();
|
||||
queryS2QLReq.setSql(queryStructReq.getLogicSql());
|
||||
queryS2QLReq.setModelId(queryStructReq.getModelId());
|
||||
queryS2QLReq.setVariables(new HashMap<>());
|
||||
return queryByS2QL(queryS2QLReq, user);
|
||||
}
|
||||
|
||||
queryService = ContextUtils.getBean(QueryService.class);
|
||||
return queryService.queryByStructWithAuth(queryStructReq, user);
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
@@ -69,9 +70,11 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
|
||||
|
||||
@Override
|
||||
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
|
||||
QueryS2QLReq queryS2QLReq = queryStructReq.convert(queryStructReq);
|
||||
if (queryStructReq.isUseS2qlSwitch() && StringUtils.isNotBlank(queryS2QLReq.getSql())) {
|
||||
log.info("queryStructReq convert to sql:{},queryStructReq:{}", queryS2QLReq.getSql(), queryStructReq);
|
||||
if (StringUtils.isNotBlank(queryStructReq.getLogicSql())) {
|
||||
QueryS2QLReq queryS2QLReq = new QueryS2QLReq();
|
||||
queryS2QLReq.setSql(queryStructReq.getLogicSql());
|
||||
queryS2QLReq.setModelId(queryStructReq.getModelId());
|
||||
queryS2QLReq.setVariables(new HashMap<>());
|
||||
return queryByS2QL(queryS2QLReq, user);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.tencent.supersonic.common.pojo.enums;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
public enum TimeDimensionEnum {
|
||||
|
||||
DAY("sys_imp_date", "数据日期"),
|
||||
WEEK("sys_imp_week", "数据日期_周"),
|
||||
|
||||
MONTH("sys_imp_month", "数据日期_月");
|
||||
|
||||
private String name;
|
||||
|
||||
private String chName;
|
||||
|
||||
TimeDimensionEnum(String name, String chName) {
|
||||
this.name = name;
|
||||
this.chName = chName;
|
||||
}
|
||||
|
||||
public static List<String> getNameList() {
|
||||
return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getName).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static Set<String> getChNameSet() {
|
||||
return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getChName).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public String getChName() {
|
||||
return chName;
|
||||
}
|
||||
}
|
||||
@@ -15,8 +15,6 @@ import lombok.extern.slf4j.Slf4j;
|
||||
public class DateUtils {
|
||||
|
||||
public static final String DATE_FORMAT = "yyyy-MM-dd";
|
||||
|
||||
public static final String DATE_FIELD = "数据日期";
|
||||
public static final String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
|
||||
public static final String FORMAT = "yyyyMMddHHmmss";
|
||||
|
||||
|
||||
@@ -55,15 +55,14 @@ public class SqlFilterUtils {
|
||||
return filterCols;
|
||||
}
|
||||
|
||||
|
||||
public String getWhereClause(List<Filter> filters) {
|
||||
public String getWhereClause(List<Filter> filters, boolean isBizName) {
|
||||
StringJoiner joiner = new StringJoiner(Constants.AND_UPPER);
|
||||
|
||||
if (!CollectionUtils.isEmpty(filters)) {
|
||||
filters.stream()
|
||||
.forEach(filter -> {
|
||||
if (Strings.isNotEmpty(dealFilter(filter))) {
|
||||
joiner.add(SPACE + dealFilter(filter) + SPACE);
|
||||
if (Strings.isNotEmpty(dealFilter(filter, isBizName))) {
|
||||
joiner.add(SPACE + dealFilter(filter, isBizName) + SPACE);
|
||||
}
|
||||
});
|
||||
log.info("getWhereClause, where sql : {}", joiner.toString());
|
||||
@@ -73,7 +72,12 @@ public class SqlFilterUtils {
|
||||
return "";
|
||||
}
|
||||
|
||||
public String dealFilter(Filter filter) {
|
||||
|
||||
public String getWhereClause(List<Filter> filters) {
|
||||
return getWhereClause(filters, true);
|
||||
}
|
||||
|
||||
public String dealFilter(Filter filter, boolean isBizName) {
|
||||
if (Objects.isNull(filter)) {
|
||||
return "";
|
||||
}
|
||||
@@ -82,14 +86,14 @@ public class SqlFilterUtils {
|
||||
}
|
||||
StringBuilder condition = new StringBuilder();
|
||||
if (Filter.Relation.FILTER.equals(filter.getRelation())) {
|
||||
return dealSingleFilter(filter);
|
||||
return dealSingleFilter(filter, isBizName);
|
||||
}
|
||||
|
||||
List<Filter> children = filter.getChildren();
|
||||
condition.append(PARENTHESES_START);
|
||||
StringJoiner joiner = new StringJoiner(SPACE + filter.getRelation().name() + SPACE);
|
||||
for (Filter child : children) {
|
||||
joiner.add(dealFilter(child));
|
||||
joiner.add(dealFilter(child, isBizName));
|
||||
}
|
||||
condition.append(joiner.toString());
|
||||
condition.append(PARENTHESES_END);
|
||||
@@ -97,8 +101,11 @@ public class SqlFilterUtils {
|
||||
}
|
||||
|
||||
// todo deal metric filter
|
||||
private String dealSingleFilter(Filter filter) {
|
||||
private String dealSingleFilter(Filter filter, boolean isBizName) {
|
||||
String name = filter.getBizName();
|
||||
if (!isBizName) {
|
||||
name = filter.getName();
|
||||
}
|
||||
Object value = filter.getValue();
|
||||
FilterOperatorEnum operator = filter.getOperator();
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
|
||||
|
||||
com.tencent.supersonic.chat.responder.parse.ParseResponder=\
|
||||
com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
|
||||
com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder
|
||||
com.tencent.supersonic.chat.responder.parse.SqlInfoParseResponder
|
||||
|
||||
com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
|
||||
com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder
|
||||
@@ -42,7 +42,7 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
|
||||
|
||||
com.tencent.supersonic.chat.responder.parse.ParseResponder=\
|
||||
com.tencent.supersonic.chat.responder.parse.EntityInfoParseResponder, \
|
||||
com.tencent.supersonic.chat.responder.parse.ExplainSqlParseResponder
|
||||
com.tencent.supersonic.chat.responder.parse.SqlInfoParseResponder
|
||||
|
||||
com.tencent.supersonic.chat.responder.execute.ExecuteResponder=\
|
||||
com.tencent.supersonic.chat.responder.execute.EntityInfoExecuteResponder, \
|
||||
|
||||
@@ -8,4 +8,4 @@ long.text.threshold=0.8
|
||||
short.text.threshold=0.5
|
||||
query.text.length.threshold=10
|
||||
candidate.threshold=0.2
|
||||
user.s2ql.switch=false
|
||||
user.s2ql.switch=true
|
||||
@@ -1,28 +0,0 @@
|
||||
package com.tencent.supersonic.semantic.api.model.enums;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
public enum TimeDimensionEnum {
|
||||
|
||||
DAY("sys_imp_date"),
|
||||
WEEK("sys_imp_week"),
|
||||
|
||||
MONTH("sys_imp_month");
|
||||
|
||||
private String name;
|
||||
|
||||
TimeDimensionEnum(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public static List<String> getNameList() {
|
||||
return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getName).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,23 @@
|
||||
package com.tencent.supersonic.semantic.api.query.request;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateModeUtils;
|
||||
import com.tencent.supersonic.common.util.SqlFilterUtils;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||
import com.tencent.supersonic.semantic.api.query.pojo.Cache;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.semantic.api.query.pojo.Param;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.sf.jsqlparser.JSQLParserException;
|
||||
@@ -49,6 +47,7 @@ public class QueryStructReq {
|
||||
|
||||
private Long modelId;
|
||||
|
||||
private String modelName;
|
||||
private List<String> groups = new ArrayList<>();
|
||||
private List<Aggregator> aggregators = new ArrayList<>();
|
||||
private List<Order> orders = new ArrayList<>();
|
||||
@@ -62,6 +61,15 @@ public class QueryStructReq {
|
||||
|
||||
private boolean useS2qlSwitch;
|
||||
|
||||
/**
|
||||
* Later deleted for compatibility only
|
||||
*/
|
||||
private String s2QL;
|
||||
/**
|
||||
* Later deleted for compatibility only
|
||||
*/
|
||||
private String logicSql;
|
||||
|
||||
public List<String> getGroups() {
|
||||
if (!CollectionUtils.isEmpty(this.groups)) {
|
||||
this.groups = groups.stream().filter(group -> !Strings.isEmpty(group)).collect(Collectors.toList());
|
||||
@@ -203,6 +211,10 @@ public class QueryStructReq {
|
||||
func = AggOperatorEnum.SUM;
|
||||
}
|
||||
sumFunction.setName(func.getOperator());
|
||||
if (AggOperatorEnum.COUNT_DISTINCT.equals(func)) {
|
||||
sumFunction.setName("count");
|
||||
sumFunction.setDistinct(true);
|
||||
}
|
||||
sumFunction.setParameters(new ExpressionList(new Column(aggregator.getColumn())));
|
||||
selectItems.add(new SelectExpressionItem(sumFunction));
|
||||
}
|
||||
@@ -210,7 +222,7 @@ public class QueryStructReq {
|
||||
}
|
||||
plainSelect.setSelectItems(selectItems);
|
||||
//2.Set the table name
|
||||
Table table = new Table(Constants.TABLE_PREFIX + queryStructReq.getModelId());
|
||||
Table table = new Table(queryStructReq.getModelName());
|
||||
plainSelect.setFromItem(table);
|
||||
|
||||
//3.Set the order by clause
|
||||
@@ -218,6 +230,9 @@ public class QueryStructReq {
|
||||
if (!CollectionUtils.isEmpty(orders)) {
|
||||
List<OrderByElement> orderByElements = new ArrayList<>();
|
||||
for (Order order : orders) {
|
||||
if (StringUtils.isBlank(order.getColumn())) {
|
||||
continue;
|
||||
}
|
||||
OrderByElement orderByElement = new OrderByElement();
|
||||
orderByElement.setExpression(new Column(order.getColumn()));
|
||||
orderByElement.setAsc(false);
|
||||
@@ -249,7 +264,7 @@ public class QueryStructReq {
|
||||
//6.Set where
|
||||
List<Filter> dimensionFilters = queryStructReq.getDimensionFilters();
|
||||
SqlFilterUtils sqlFilterUtils = ContextUtils.getBean(SqlFilterUtils.class);
|
||||
String whereClause = sqlFilterUtils.getWhereClause(dimensionFilters);
|
||||
String whereClause = sqlFilterUtils.getWhereClause(dimensionFilters, false);
|
||||
|
||||
String sql = select.toString();
|
||||
if (StringUtils.isNotBlank(whereClause)) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter;
|
||||
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
|
||||
public class H2Adaptor extends EngineAdaptor {
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter;
|
||||
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.tencent.supersonic.semantic.model.domain.utils;
|
||||
|
||||
import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams;
|
||||
import com.tencent.supersonic.semantic.model.domain.adaptor.engineadapter.EngineAdaptor;
|
||||
|
||||
@@ -4,11 +4,10 @@ package com.tencent.supersonic.semantic.query.parser.convert;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.semantic.api.model.request.SqlExecuteReq;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
|
||||
@@ -185,7 +184,14 @@ public class QueryReqConverter {
|
||||
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
|
||||
.collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1));
|
||||
|
||||
dimensionResults.put(DateUtils.DATE_FIELD, TimeDimensionEnum.DAY.getName());
|
||||
dimensionResults.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getName());
|
||||
dimensionResults.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getName());
|
||||
dimensionResults.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getName());
|
||||
|
||||
dimensionResults.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getName());
|
||||
dimensionResults.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getName());
|
||||
dimensionResults.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getName());
|
||||
|
||||
dimensionResults.putAll(metricResults);
|
||||
return dimensionResults;
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.util.cache.CacheUtils;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
|
||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.semantic.query.utils;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
Reference in New Issue
Block a user