topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Unified display of time filtering conditions to the frontend. (#1600)

Browse Source
This commit is contained in:
lexluo09
2024-08-23 19:01:37 +08:00
committed by GitHub
parent 371e2f1e05
commit d2306464a6
3 changed files with 45 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java
View File

@@ -62,15 +62,13 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
})
.collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1));
if (chatQueryContext.containsPartitionDimensions(dataSetId)) {
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
}
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
return result;
}

11
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java
View File

@@ -57,14 +57,19 @@ public class TimeRangeParser implements SemanticParser {
private void updateQueryContext(ChatQueryContext queryContext, DateConf dateConf) {
if (!queryContext.getCandidateQueries().isEmpty()) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
query.getParseInfo().setDateInfo(dateConf);
query.getParseInfo().setScore(query.getParseInfo().getScore() + dateConf.getDetectWord().length());
SemanticParseInfo parseInfo = query.getParseInfo();
if (queryContext.containsPartitionDimensions(parseInfo.getDataSetId())) {
parseInfo.setDateInfo(dateConf);
}
parseInfo.setScore(parseInfo.getScore() + dateConf.getDetectWord().length());
}
} else {
SemanticParseInfo contextParseInfo = queryContext.getContextParseInfo();
if (QueryManager.containsRuleQuery(contextParseInfo.getQueryMode())) {
RuleSemanticQuery semanticQuery = QueryManager.createRuleQuery(contextParseInfo.getQueryMode());
contextParseInfo.setDateInfo(dateConf);
if (queryContext.containsPartitionDimensions(contextParseInfo.getDataSetId())) {
contextParseInfo.setDateInfo(dateConf);
}
contextParseInfo.setScore(contextParseInfo.getScore() + dateConf.getDetectWord().length());
semanticQuery.setParseInfo(contextParseInfo);
queryContext.getCandidateQueries().add(semanticQuery);

48
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/ParseInfoProcessor.java
View File

@@ -48,41 +48,40 @@ public class ParseInfoProcessor implements ResultProcessor {
return;
}
List<FieldExpression> expressions = SqlSelectHelper.getFilterExpression(s2SQL);
Long dataSetId = parseInfo.getDataSetId();
SemanticLayerService semanticLayerService = ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dsSchema = semanticLayerService.getDataSetSchema(dataSetId);
//extract date filter from S2SQL
try {
if (parseInfo.getDateInfo() == null && !CollectionUtils.isEmpty(expressions)) {
parseInfo.setDateInfo(extractDateFilter(expressions));
parseInfo.setDateInfo(extractDateFilter(expressions, dsSchema));
}
} catch (Exception e) {
log.error("failed to extract date range:", e);
}
//extract dimension filters from S2SQL
Long dataSetId = parseInfo.getDataSetId();
SemanticLayerService semanticLayerService = ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dsSchema = semanticLayerService.getDataSetSchema(dataSetId);
try {
Map<String, SchemaElement> fieldNameToElement = getNameToElement(dsSchema);
parseInfo.getDimensionFilters().addAll(extractDimensionFilter(fieldNameToElement, expressions));
List<QueryFilter> queryFilters = extractDimensionFilter(dsSchema, expressions);
parseInfo.getDimensionFilters().addAll(queryFilters);
} catch (Exception e) {
log.error("failed to extract dimension filters:", e);
}
//extract metrics from S2SQL
List<String> allFields = filterDateField(SqlSelectHelper.getAllSelectFields(s2SQL));
List<String> allFields = filterDateField(dsSchema, SqlSelectHelper.getAllSelectFields(s2SQL));
Set<SchemaElement> metrics = matchSchemaElements(allFields, dsSchema.getMetrics());
parseInfo.setMetrics(metrics);
//extract dimensions from S2SQL
if (QueryType.METRIC.equals(parseInfo.getQueryType())) {
List<String> groupByFields = SqlSelectHelper.getGroupByFields(s2SQL);
List<String> groupByDimensions = filterDateField(groupByFields);
List<String> groupByDimensions = filterDateField(dsSchema, groupByFields);
parseInfo.setDimensions(matchSchemaElements(groupByDimensions, dsSchema.getDimensions()));
} else if (QueryType.DETAIL.equals(parseInfo.getQueryType())) {
List<String> selectFields = SqlSelectHelper.getSelectFields(s2SQL);
List<String> selectDimensions = filterDateField(selectFields);
List<String> selectDimensions = filterDateField(dsSchema, selectFields);
parseInfo.setDimensions(matchSchemaElements(selectDimensions, dsSchema.getDimensions()));
}
}
@@ -103,20 +102,22 @@ public class ParseInfoProcessor implements ResultProcessor {
).collect(Collectors.toSet());
}
private List<String> filterDateField(List<String> allFields) {
private List<String> filterDateField(DataSetSchema dataSetSchema, List<String> allFields) {
return allFields.stream()
.filter(entry -> !TimeDimensionEnum.DAY.getChName().equalsIgnoreCase(entry))
.filter(entry -> !isPartitionDimension(dataSetSchema, entry))
.collect(Collectors.toList());
}
private List<QueryFilter> extractDimensionFilter(Map<String, SchemaElement> fieldNameToElement,
private List<QueryFilter> extractDimensionFilter(DataSetSchema dsSchema,
List<FieldExpression> fieldExpressions) {
Map<String, SchemaElement> fieldNameToElement = getNameToElement(dsSchema);
List<QueryFilter> result = Lists.newArrayList();
for (FieldExpression expression : fieldExpressions) {
QueryFilter dimensionFilter = new QueryFilter();
dimensionFilter.setValue(expression.getFieldValue());
SchemaElement schemaElement = fieldNameToElement.get(expression.getFieldName());
if (Objects.isNull(schemaElement) || schemaElement.containsPartitionTime()) {
if (Objects.isNull(schemaElement) || isPartitionDimension(dsSchema, schemaElement.getName())) {
continue;
}
dimensionFilter.setName(schemaElement.getName());
@@ -131,11 +132,12 @@ public class ParseInfoProcessor implements ResultProcessor {
return result;
}
private DateConf extractDateFilter(List<FieldExpression> fieldExpressions) {
private DateConf extractDateFilter(List<FieldExpression> fieldExpressions,
DataSetSchema dataSetSchema) {
List<FieldExpression> dateExpressions = fieldExpressions.stream()
.filter(expression -> TimeDimensionEnum.DAY.getChName().equalsIgnoreCase(expression.getFieldName()))
.filter(expression -> isPartitionDimension(dataSetSchema, expression.getFieldName()))
.collect(Collectors.toList());
if (org.apache.commons.collections.CollectionUtils.isEmpty(dateExpressions)) {
if (CollectionUtils.isEmpty(dateExpressions)) {
return null;
}
DateConf dateInfo = new DateConf();
@@ -166,6 +168,18 @@ public class ParseInfoProcessor implements ResultProcessor {
return dateInfo;
}
private static boolean isPartitionDimension(DataSetSchema dataSetSchema, String sqlFieldName) {
if (TimeDimensionEnum.containsTimeDimension(sqlFieldName)) {
return true;
}
if (Objects.isNull(dataSetSchema)
|| Objects.isNull(dataSetSchema.getPartitionDimension())
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
return false;
}
return sqlFieldName.equalsIgnoreCase(dataSetSchema.getPartitionDimension().getName());
}
private boolean containOperators(FieldExpression expression, FilterOperatorEnum firstOperator,
FilterOperatorEnum... operatorEnums) {
return (Arrays.asList(operatorEnums).contains(firstOperator) && Objects.nonNull(