topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 04:57:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Rules, large models, and query dimension values support timelessness. (#1522)

Browse Source
This commit is contained in:
lexluo09
2024-08-07 13:29:07 +08:00
committed by GitHub
parent c8fe6d2d04
commit 208686de46
28 changed files with 442 additions and 613 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/ChatQueryContext.java
View File

@@ -7,6 +7,7 @@ import com.tencent.supersonic.common.pojo.ChatModelConfig;
import com.tencent.supersonic.common.pojo.Text2SQLExemplar;
import com.tencent.supersonic.common.pojo.enums.Text2SQLType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.QueryDataType;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -66,4 +67,10 @@ public class ChatQueryContext {
.collect(Collectors.toList());
return candidateQueries;
}
public boolean containsPartitionDimensions(Long dataSetId) {
SemanticSchema semanticSchema = this.getSemanticSchema();
DataSetSchema dataSetSchema = semanticSchema.getDataSetSchemaMap().get(dataSetId);
return dataSetSchema.containsPartitionDimensions();
}
}

22
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java
View File

@@ -7,6 +7,10 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
@@ -15,10 +19,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
/**
* basic semantic correction functionality, offering common methods and an
@@ -61,14 +61,16 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
return elements.stream();
})
.collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1));
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
if (chatQueryContext.containsPartitionDimensions(dataSetId)) {
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
}
return result;
}

5
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/ParserConfig.java
View File

@@ -61,11 +61,6 @@ public class ParserConfig extends ParameterConfig {
"解析结果展示个数", "前端展示的解析个数",
"number", "Parser相关配置");
public static final Parameter PARSER_S2SQL_ENABLE =
new Parameter("s2.parser.s2sql.switch", "true",
"", "",
"bool", "Parser相关配置");
@Override
public List<Parameter> getSysParameters() {
return Lists.newArrayList(

5
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java
View File

@@ -86,8 +86,9 @@ public class LLMRequestService {
&& Objects.nonNull(semanticSchema.getDataSetSchemaMap().get(dataSetId))) {
TimeDefaultConfig timeDefaultConfig = semanticSchema.getDataSetSchemaMap()
.get(dataSetId).getTagTypeTimeDefaultConfig();
if (!Objects.equals(timeDefaultConfig.getUnit(), -1)) {
// 数据集查询设置 时间不为-1时才添加 '数据日期' 字段
if (!Objects.equals(timeDefaultConfig.getUnit(), -1)
&& queryCtx.containsPartitionDimensions(dataSetId)) {
// 数据集配置了数据日期字段，并查询设置 时间不为-1时才添加 '数据日期' 字段
fieldNameList.add(TimeDimensionEnum.DAY.getChName());
}
}

188
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.chat.parser.rule;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.parser.SemanticParser;
import com.tencent.supersonic.headless.chat.query.QueryManager;
@@ -10,7 +11,6 @@ import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery;
import com.xkzhangsan.time.nlp.TimeNLP;
import com.xkzhangsan.time.nlp.TimeNLPUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.text.DateFormat;
import java.text.ParseException;
@@ -22,8 +22,6 @@ import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
;
/**
* TimeRangeParser extracts time range specified in the user query
* based on keyword matching.
@@ -52,123 +50,127 @@ public class TimeRangeParser implements SemanticParser {
}
if (dateConf != null) {
if (queryContext.getCandidateQueries().size() > 0) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
query.getParseInfo().setDateInfo(dateConf);
query.getParseInfo().setScore(query.getParseInfo().getScore()
+ dateConf.getDetectWord().length());
}
} else if (QueryManager.containsRuleQuery(queryContext.getContextParseInfo().getQueryMode())) {
RuleSemanticQuery semanticQuery = QueryManager.createRuleQuery(
queryContext.getContextParseInfo().getQueryMode());
// inherit parse info from context
queryContext.getContextParseInfo().setDateInfo(dateConf);
queryContext.getContextParseInfo().setScore(queryContext.getContextParseInfo().getScore()
+ dateConf.getDetectWord().length());
semanticQuery.setParseInfo(queryContext.getContextParseInfo());
updateQueryContext(queryContext, dateConf);
}
}
private void updateQueryContext(ChatQueryContext queryContext, DateConf dateConf) {
if (!queryContext.getCandidateQueries().isEmpty()) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
query.getParseInfo().setDateInfo(dateConf);
query.getParseInfo().setScore(query.getParseInfo().getScore() + dateConf.getDetectWord().length());
}
} else {
SemanticParseInfo contextParseInfo = queryContext.getContextParseInfo();
if (QueryManager.containsRuleQuery(contextParseInfo.getQueryMode())) {
RuleSemanticQuery semanticQuery = QueryManager.createRuleQuery(contextParseInfo.getQueryMode());
contextParseInfo.setDateInfo(dateConf);
contextParseInfo.setScore(contextParseInfo.getScore() + dateConf.getDetectWord().length());
semanticQuery.setParseInfo(contextParseInfo);
queryContext.getCandidateQueries().add(semanticQuery);
}
}
}
private DateConf parseDateCN(String queryText) {
Date startDate = null;
Date endDate;
String detectWord = null;
List<TimeNLP> times = TimeNLPUtil.parse(queryText);
if (times.size() > 0) {
startDate = times.get(0).getTime();
detectWord = times.get(0).getTimeExpression();
} else {
if (times.isEmpty()) {
return null;
}
Date startDate = times.get(0).getTime();
String detectWord = times.get(0).getTimeExpression();
Date endDate = times.size() > 1 ? times.get(1).getTime() : startDate;
if (times.size() > 1) {
endDate = times.get(1).getTime();
detectWord += "~" + times.get(0).getTimeExpression();
} else {
endDate = startDate;
detectWord += "~" + times.get(1).getTimeExpression();
}
return getDateConf(startDate, endDate, detectWord);
}
private DateConf parseDateNumber(String queryText) {
String startDate;
String endDate = null;
String detectWord = null;
Matcher dateMatcher = DATE_PATTERN_NUMBER.matcher(queryText);
if (dateMatcher.find()) {
startDate = dateMatcher.group();
detectWord = startDate;
} else {
if (!dateMatcher.find()) {
return null;
}
if (dateMatcher.find()) {
endDate = dateMatcher.group();
detectWord += "~" + endDate;
String startDateStr = dateMatcher.group();
String detectWord = startDateStr;
String endDateStr = dateMatcher.find() ? dateMatcher.group() : startDateStr;
if (!startDateStr.equals(endDateStr)) {
detectWord += "~" + endDateStr;
}
endDate = endDate != null ? endDate : startDate;
try {
return getDateConf(DATE_FORMAT_NUMBER.parse(startDate), DATE_FORMAT_NUMBER.parse(endDate), detectWord);
Date startDate = DATE_FORMAT_NUMBER.parse(startDateStr);
Date endDate = DATE_FORMAT_NUMBER.parse(endDateStr);
return getDateConf(startDate, endDate, detectWord);
} catch (ParseException e) {
return null;
}
}
private DateConf parseRecent(String queryText) {
Matcher m = RECENT_PATTERN_CN.matcher(queryText);
if (m.matches()) {
int num = 0;
String enNum = m.group("enNum");
String zhNum = m.group("zhNum");
if (enNum != null) {
num = Integer.parseInt(enNum);
} else if (zhNum != null) {
num = zhNumParse(zhNum);
}
if (num > 0) {
DateConf info = new DateConf();
String zhPeriod = m.group("zhPeriod");
int days;
switch (zhPeriod) {
case "周":
days = 7;
info.setPeriod(Constants.WEEK);
break;
case "月":
days = 30;
info.setPeriod(Constants.MONTH);
break;
case "年":
days = 365;
info.setPeriod(Constants.YEAR);
break;
default:
days = 1;
info.setPeriod(Constants.DAY);
}
days = days * num;
info.setDateMode(DateConf.DateMode.RECENT);
String detectWord = "近" + num + zhPeriod;
if (StringUtils.isNotEmpty(m.group("periodStr"))) {
detectWord = m.group("periodStr");
}
info.setDetectWord(detectWord);
info.setStartDate(LocalDate.now().minusDays(days).toString());
info.setEndDate(LocalDate.now().minusDays(1).toString());
info.setUnit(num);
return info;
}
Matcher matcher = RECENT_PATTERN_CN.matcher(queryText);
if (!matcher.matches()) {
return null;
}
int num = parseNumber(matcher);
if (num <= 0) {
return null;
}
String zhPeriod = matcher.group("zhPeriod");
int days = getDaysByPeriod(zhPeriod) * num;
String detectWord = matcher.group("periodStr");
return null;
DateConf info = new DateConf();
info.setPeriod(getPeriodConstant(zhPeriod));
info.setDateMode(DateConf.DateMode.RECENT);
info.setDetectWord(detectWord);
info.setStartDate(LocalDate.now().minusDays(days).toString());
info.setEndDate(LocalDate.now().minusDays(1).toString());
info.setUnit(num);
return info;
}
private int parseNumber(Matcher matcher) {
String enNum = matcher.group("enNum");
String zhNum = matcher.group("zhNum");
if (enNum != null) {
return Integer.parseInt(enNum);
} else if (zhNum != null) {
return zhNumParse(zhNum);
}
return 0;
}
private int getDaysByPeriod(String zhPeriod) {
switch (zhPeriod) {
case "周":
return 7;
case "月":
return 30;
case "年":
return 365;
default:
return 1;
}
}
private String getPeriodConstant(String zhPeriod) {
switch (zhPeriod) {
case "周":
return Constants.WEEK;
case "月":
return Constants.MONTH;
case "年":
return Constants.YEAR;
default:
return Constants.DAY;
}
}
private int zhNumParse(String zhNumStr) {
@@ -176,10 +178,9 @@ public class TimeRangeParser implements SemanticParser {
String numStr = "一二三四五六七八九";
String unitStr = "十百千万亿";
String[] ssArr = zhNumStr.split("");
for (String e : ssArr) {
int numIndex = numStr.indexOf(e);
int unitIndex = unitStr.indexOf(e);
for (char c : zhNumStr.toCharArray()) {
int numIndex = numStr.indexOf(c);
int unitIndex = unitStr.indexOf(c);
if (numIndex != -1) {
stack.push(numIndex + 1);
} else if (unitIndex != -1) {
@@ -192,7 +193,7 @@ public class TimeRangeParser implements SemanticParser {
}
}
return stack.stream().mapToInt(s -> s).sum();
return stack.stream().mapToInt(Integer::intValue).sum();
}
private DateConf getDateConf(Date startDate, Date endDate, String detectWord) {
@@ -207,5 +208,4 @@ public class TimeRangeParser implements SemanticParser {
info.setDetectWord(detectWord);
return info;
}
}

31
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/BaseSemanticQuery.java
View File

@@ -5,12 +5,11 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.chat.parser.ParserConfig;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
@@ -21,8 +20,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static com.tencent.supersonic.headless.chat.parser.ParserConfig.PARSER_S2SQL_ENABLE;
@Slf4j
@ToString
public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
@@ -43,6 +40,19 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
return QueryReqBuilder.buildStructReq(parseInfo);
}
@Override
public SemanticQueryReq buildSemanticQueryReq() {
return QueryReqBuilder.buildS2SQLReq(parseInfo.getSqlInfo(), parseInfo.getDataSetId());
}
protected void initS2SqlByStruct(DataSetSchema dataSetSchema) {
QueryStructReq queryStructReq = convertQueryStruct();
convertBizNameToName(dataSetSchema, queryStructReq);
QuerySqlReq querySQLReq = queryStructReq.convert();
parseInfo.getSqlInfo().setParsedS2SQL(querySQLReq.getSql());
parseInfo.getSqlInfo().setCorrectedS2SQL(querySQLReq.getSql());
}
protected void convertBizNameToName(DataSetSchema dataSetSchema, QueryStructReq queryStructReq) {
Map<String, String> bizNameToName = dataSetSchema.getBizNameToName();
bizNameToName.putAll(TimeDimensionEnum.getNameToNameMap());
@@ -74,17 +84,4 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
}
}
protected void initS2SqlByStruct(DataSetSchema dataSetSchema) {
ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class);
boolean s2sqlEnable = Boolean.valueOf(parserConfig.getParameterValue(PARSER_S2SQL_ENABLE));
if (!s2sqlEnable) {
return;
}
QueryStructReq queryStructReq = convertQueryStruct();
convertBizNameToName(dataSetSchema, queryStructReq);
QuerySqlReq querySQLReq = queryStructReq.convert();
parseInfo.getSqlInfo().setParsedS2SQL(querySQLReq.getSql());
parseInfo.getSqlInfo().setCorrectedS2SQL(querySQLReq.getSql());
}
}

7
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlQuery.java
View File

@@ -3,10 +3,8 @@ package com.tencent.supersonic.headless.chat.query.llm.s2sql;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.chat.query.QueryManager;
import com.tencent.supersonic.headless.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -25,11 +23,6 @@ public class LLMSqlQuery extends LLMSemanticQuery {
return QUERY_MODE;
}
@Override
public SemanticQueryReq buildSemanticQueryReq() {
return QueryReqBuilder.buildS2SQLReq(parseInfo.getSqlInfo(), parseInfo.getDataSetId());
}
@Override
public void initS2Sql(DataSetSchema dataSetSchema, User user) {
SqlInfo sqlInfo = parseInfo.getSqlInfo();

61
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java
View File

@@ -27,6 +27,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -41,7 +42,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
}
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches,
ChatQueryContext queryCtx) {
ChatQueryContext queryCtx) {
return queryMatcher.match(candidateElementMatches);
}
@@ -56,20 +57,30 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
fillSchemaElement(parseInfo, semanticSchema);
fillScore(parseInfo);
fillDateConf(parseInfo, chatQueryContext.getContextParseInfo());
fillDateConfByInherited(parseInfo, chatQueryContext);
}
private void fillDateConf(SemanticParseInfo queryParseInfo, SemanticParseInfo chatParseInfo) {
if (queryParseInfo.getDateInfo() != null || chatParseInfo.getDateInfo() == null) {
public boolean needFillDateConf(ChatQueryContext chatQueryContext) {
Long dataSetId = parseInfo.getDataSetId();
if (Objects.isNull(dataSetId) || dataSetId <= 0L) {
return false;
}
return chatQueryContext.containsPartitionDimensions(dataSetId);
}
private void fillDateConfByInherited(SemanticParseInfo queryParseInfo, ChatQueryContext chatQueryContext) {
SemanticParseInfo contextParseInfo = chatQueryContext.getContextParseInfo();
if (queryParseInfo.getDateInfo() != null || contextParseInfo.getDateInfo() == null
|| needFillDateConf(chatQueryContext)) {
return;
}
if ((QueryManager.isTagQuery(queryParseInfo.getQueryMode())
&& QueryManager.isTagQuery(chatParseInfo.getQueryMode()))
&& QueryManager.isTagQuery(contextParseInfo.getQueryMode()))
|| (QueryManager.isMetricQuery(queryParseInfo.getQueryMode())
&& QueryManager.isMetricQuery(chatParseInfo.getQueryMode()))) {
&& QueryManager.isMetricQuery(contextParseInfo.getQueryMode()))) {
// inherit date info from context
queryParseInfo.setDateInfo(chatParseInfo.getDateInfo());
queryParseInfo.setDateInfo(contextParseInfo.getDateInfo());
queryParseInfo.getDateInfo().setInherited(true);
}
}
@@ -142,13 +153,15 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
}
private void addToFilters(Map<Long, List<SchemaElementMatch>> id2Values, SemanticParseInfo parseInfo,
SemanticSchema semanticSchema, SchemaElementType entity) {
SemanticSchema semanticSchema, SchemaElementType entity) {
if (id2Values == null || id2Values.isEmpty()) {
return;
}
for (Entry<Long, List<SchemaElementMatch>> entry : id2Values.entrySet()) {
SchemaElement dimension = semanticSchema.getElement(entity, entry.getKey());
if (dimension.containsPartitionTime()) {
continue;
}
if (entry.getValue().size() == 1) {
SchemaElementMatch schemaMatch = entry.getValue().get(0);
QueryFilter dimensionFilter = new QueryFilter();
@@ -173,34 +186,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
}
}
private void addToValues(SemanticSchema semanticSchema, SchemaElementType entity,
Map<Long, List<SchemaElementMatch>> id2Values, SchemaElementMatch schemaMatch) {
SchemaElement element = schemaMatch.getElement();
SchemaElement entityElement = semanticSchema.getElement(entity, element.getId());
if (entityElement != null) {
if (id2Values.containsKey(element.getId())) {
id2Values.get(element.getId()).add(schemaMatch);
} else {
id2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch)));
}
}
}
@Override
public SemanticQueryReq buildSemanticQueryReq() {
String queryMode = parseInfo.getQueryMode();
if (parseInfo.getDataSetId() == null || StringUtils.isEmpty(queryMode)
|| !QueryManager.containsRuleQuery(queryMode)) {
// reach here some error may happen
log.error("not find QueryMode");
throw new RuntimeException("not find QueryMode");
}
QueryStructReq queryStructReq = convertQueryStruct();
return queryStructReq.convert(true);
}
protected boolean isMultiStructQuery() {
return false;
}
@@ -224,7 +209,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
}
public static List<RuleSemanticQuery> resolve(Long dataSetId, List<SchemaElementMatch> candidateElementMatches,
ChatQueryContext chatQueryContext) {
ChatQueryContext chatQueryContext) {
List<RuleSemanticQuery> matchedQueries = new ArrayList<>();
for (RuleSemanticQuery semanticQuery : QueryManager.getRuleQueries()) {
List<SchemaElementMatch> matches = semanticQuery.match(candidateElementMatches, chatQueryContext);

53
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailListQuery.java
View File

@@ -25,34 +25,35 @@ public abstract class DetailListQuery extends DetailSemanticQuery {
private void addEntityDetailAndOrderByMetric(ChatQueryContext chatQueryContext, SemanticParseInfo parseInfo) {
Long dataSetId = parseInfo.getDataSetId();
if (Objects.nonNull(dataSetId) && dataSetId > 0L) {
DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
if (dataSetSchema != null && Objects.nonNull(dataSetSchema.getEntity())) {
Set<SchemaElement> dimensions = new LinkedHashSet<>();
Set<SchemaElement> metrics = new LinkedHashSet<>();
Set<Order> orders = new LinkedHashSet<>();
TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig();
if (tagTypeDefaultConfig != null && tagTypeDefaultConfig.getDefaultDisplayInfo() != null) {
if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) {
metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds()
.stream().map(id -> {
SchemaElement metric = dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (metric != null) {
orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER));
}
return metric;
}).filter(Objects::nonNull).collect(Collectors.toSet());
}
if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) {
dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> dataSetSchema.getElement(SchemaElementType.DIMENSION, id))
.filter(Objects::nonNull).collect(Collectors.toSet());
}
if (Objects.isNull(dataSetId) || dataSetId <= 0L) {
return;
}
DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
if (dataSetSchema != null && Objects.nonNull(dataSetSchema.getEntity())) {
Set<SchemaElement> dimensions = new LinkedHashSet<>();
Set<SchemaElement> metrics = new LinkedHashSet<>();
Set<Order> orders = new LinkedHashSet<>();
TagTypeDefaultConfig tagTypeDefaultConfig = dataSetSchema.getTagTypeDefaultConfig();
if (tagTypeDefaultConfig != null && tagTypeDefaultConfig.getDefaultDisplayInfo() != null) {
if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds())) {
metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getMetricIds()
.stream().map(id -> {
SchemaElement metric = dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (metric != null) {
orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER));
}
return metric;
}).filter(Objects::nonNull).collect(Collectors.toSet());
}
if (CollectionUtils.isNotEmpty(tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds())) {
dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> dataSetSchema.getElement(SchemaElementType.DIMENSION, id))
.filter(Objects::nonNull).collect(Collectors.toSet());
}
parseInfo.setDimensions(dimensions);
parseInfo.setMetrics(metrics);
parseInfo.setOrders(orders);
}
parseInfo.setDimensions(dimensions);
parseInfo.setMetrics(metrics);
parseInfo.setOrders(orders);
}
}

44
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java
View File

@@ -14,6 +14,7 @@ import lombok.extern.slf4j.Slf4j;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@Slf4j
@@ -39,28 +40,31 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery {
parseInfo.setQueryType(QueryType.DETAIL);
parseInfo.setLimit(DETAIL_MAX_RESULTS);
if (parseInfo.getDateInfo() == null) {
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
if (!needFillDateConf(chatQueryContext)) {
return;
}
Map<Long, DataSetSchema> dataSetSchemaMap = chatQueryContext.getSemanticSchema().getDataSetSchemaMap();
DataSetSchema dataSetSchema = dataSetSchemaMap.get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
if (Objects.nonNull(timeDefaultConfig)
&& Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {
DateConf dateInfo = new DateConf();
if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {
int unit = timeDefaultConfig.getUnit();
String startDate = LocalDate.now().plusDays(-unit).toString();
String endDate = startDate;
if (TimeMode.LAST.equals(timeDefaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
} else if (TimeMode.RECENT.equals(timeDefaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.RECENT);
endDate = LocalDate.now().plusDays(-1).toString();
}
dateInfo.setUnit(unit);
dateInfo.setPeriod(timeDefaultConfig.getPeriod());
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
parseInfo.setDateInfo(dateInfo);
int unit = timeDefaultConfig.getUnit();
String startDate = LocalDate.now().plusDays(-unit).toString();
String endDate = startDate;
if (TimeMode.LAST.equals(timeDefaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
} else if (TimeMode.RECENT.equals(timeDefaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.RECENT);
endDate = LocalDate.now().plusDays(-1).toString();
}
dateInfo.setUnit(unit);
dateInfo.setPeriod(timeDefaultConfig.getPeriod());
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
parseInfo.setDateInfo(dateInfo);
}
}

14
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricIdQuery.java
View File

@@ -1,24 +1,24 @@
package com.tencent.supersonic.headless.chat.query.rule.metric;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
@Slf4j
@Component

51
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java
View File

@@ -38,30 +38,35 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
public void fillParseInfo(ChatQueryContext chatQueryContext) {
super.fillParseInfo(chatQueryContext);
parseInfo.setLimit(METRIC_MAX_RESULTS);
if (parseInfo.getDateInfo() == null) {
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig();
DateConf dateInfo = new DateConf();
//加上时间!=-1 判断
if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {
int unit = timeDefaultConfig.getUnit();
String startDate = LocalDate.now().plusDays(-unit).toString();
String endDate = startDate;
if (TimeMode.LAST.equals(timeDefaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
} else if (TimeMode.RECENT.equals(timeDefaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.RECENT);
endDate = LocalDate.now().plusDays(-1).toString();
}
dateInfo.setUnit(unit);
dateInfo.setPeriod(timeDefaultConfig.getPeriod());
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
// 时间不为-1才设置时间，所以移到这里
parseInfo.setDateInfo(dateInfo);
fillDateInfo(chatQueryContext);
}
private void fillDateInfo(ChatQueryContext chatQueryContext) {
if (parseInfo.getDateInfo() != null || !needFillDateConf(chatQueryContext)) {
return;
}
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig();
DateConf dateInfo = new DateConf();
//加上时间!=-1 判断
if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {
int unit = timeDefaultConfig.getUnit();
String startDate = LocalDate.now().plusDays(-unit).toString();
String endDate = startDate;
if (TimeMode.LAST.equals(timeDefaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
} else if (TimeMode.RECENT.equals(timeDefaultConfig.getTimeMode())) {
dateInfo.setDateMode(DateConf.DateMode.RECENT);
endDate = LocalDate.now().plusDays(-1).toString();
}
dateInfo.setUnit(unit);
dateInfo.setPeriod(timeDefaultConfig.getPeriod());
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
// 时间不为-1才设置时间，所以移到这里
parseInfo.setDateInfo(dateInfo);
}
}
}

18
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java
View File

@@ -18,6 +18,11 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.chat.query.QueryManager;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
@@ -28,10 +33,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
@Slf4j
public class QueryReqBuilder {
@@ -88,9 +89,12 @@ public class QueryReqBuilder {
}
private static DateConf rewrite2Between(DateConf dateInfo) {
if (Objects.isNull(dateInfo)) {
return null;
}
DateConf dateInfoNew = new DateConf();
BeanUtils.copyProperties(dateInfo, dateInfoNew);
if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT.equals(dateInfo.getDateMode())) {
if (DateConf.DateMode.RECENT.equals(dateInfo.getDateMode())) {
int unit = dateInfo.getUnit();
int days = 1;
switch (dateInfo.getPeriod()) {
@@ -222,7 +226,7 @@ public class QueryReqBuilder {
}
public static Set<Order> getOrder(Set<Order> existingOrders,
AggregateTypeEnum aggregator, SchemaElement metric) {
AggregateTypeEnum aggregator, SchemaElement metric) {
if (existingOrders != null && !existingOrders.isEmpty()) {
return existingOrders;
}
@@ -259,7 +263,7 @@ public class QueryReqBuilder {
}
public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, SchemaElement metric,
AggOperatorEnum aggOperatorEnum) {
AggOperatorEnum aggOperatorEnum) {
QueryStructReq queryStructReq = buildStructReq(parseInfo);
queryStructReq.setQueryType(QueryType.METRIC);
queryStructReq.setOrders(new ArrayList<>());