topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(feature)(chat) support querying of dimension aliases and metric aliases (#260)

Browse Source
This commit is contained in:
lexluo09
2023-10-18 21:35:12 +08:00
committed by GitHub
parent ba1d14f40a
commit 34eb94320e
5 changed files with 68 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java
View File

@@ -35,9 +35,18 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
dbAllFields.addAll(semanticSchema.getMetrics());
dbAllFields.addAll(semanticSchema.getDimensions());
// support fieldName and field alias
Map<String, String> result = dbAllFields.stream()
.filter(entry -> entry.getModel().equals(modelId))
.collect(Collectors.toMap(SchemaElement::getName, a -> a.getName(), (k1, k2) -> k1));
.flatMap(schemaElement -> {
Set<String> elements = new HashSet<>();
elements.add(schemaElement.getName());
if (!CollectionUtils.isEmpty(schemaElement.getAlias())) {
elements.addAll(schemaElement.getAlias());
}
return elements.stream();
})
.collect(Collectors.toMap(a -> a, a -> a, (k1, k2) -> k1));
result.put(DateUtils.DATE_FIELD, DateUtils.DATE_FIELD);
return result;
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java
View File

@@ -356,6 +356,9 @@ public class LLMS2QLParser implements SemanticParser {
llmReq.setLinking(linking);
String currentDate = S2QLDateHelper.getReferenceDate(modelId);
if (StringUtils.isEmpty(currentDate)) {
currentDate = DateUtils.getBeforeDate(0);
}
llmReq.setCurrentDate(currentDate);
return llmReq;
}
@@ -423,14 +426,13 @@ public class LLMS2QLParser implements SemanticParser {
|| SchemaElementType.VALUE.equals(elementType);
})
.map(schemaElementMatch -> {
SchemaElementType elementType = schemaElementMatch.getElement().getType();
if (!SchemaElementType.VALUE.equals(elementType)) {
return schemaElementMatch.getWord();
SchemaElement element = schemaElementMatch.getElement();
SchemaElementType elementType = element.getType();
if (SchemaElementType.VALUE.equals(elementType)) {
return itemIdToName.get(element.getId());
}
return itemIdToName.get(schemaElementMatch.getElement().getId());
return schemaElementMatch.getWord();
})
.filter(name -> StringUtils.isNotEmpty(name) && !name.contains("%"))
.collect(Collectors.toSet());
return fieldNameList;
}

35
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/ModelSchemaBuilder.java
View File

@@ -9,28 +9,24 @@ import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.pojo.Entity;
import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Objects;
import java.util.Map;
import java.util.stream.Collectors;
public class ModelSchemaBuilder {
private static String aliasSplit = ",";
public static ModelSchema build(ModelSchemaResp resp) {
ModelSchema modelSchema = new ModelSchema();
SchemaElement domain = SchemaElement.builder()
@@ -39,14 +35,14 @@ public class ModelSchemaBuilder {
.name(resp.getName())
.bizName(resp.getBizName())
.type(SchemaElementType.MODEL)
.alias(getAliasList(resp.getAlias()))
.alias(SchemaItem.getAliasList(resp.getAlias()))
.build();
modelSchema.setModel(domain);
Set<SchemaElement> metrics = new HashSet<>();
for (MetricSchemaResp metric : resp.getMetrics()) {
List<String> alias = getAliasList(metric.getAlias());
List<String> alias = SchemaItem.getAliasList(metric.getAlias());
SchemaElement metricToAdd = SchemaElement.builder()
.model(resp.getId())
@@ -68,7 +64,7 @@ public class ModelSchemaBuilder {
Set<SchemaElement> dimensionValues = new HashSet<>();
for (DimSchemaResp dim : resp.getDimensions()) {
List<String> alias = getAliasList(dim.getAlias());
List<String> alias = SchemaItem.getAliasList(dim.getAlias());
Set<String> dimValueAlias = new HashSet<>();
List<DimValueMap> dimValueMaps = dim.getDimValueMaps();
List<SchemaValueMap> schemaValueMaps = new ArrayList<>();
@@ -133,13 +129,6 @@ public class ModelSchemaBuilder {
return modelSchema;
}
private static List<String> getAliasList(String alias) {
if (StringUtils.isEmpty(alias)) {
return new ArrayList<>();
}
return Arrays.asList(alias.split(aliasSplit));
}
private static List<RelateSchemaElement> getRelateSchemaElement(MetricSchemaResp metricSchemaResp) {
RelateDimension relateDimension = metricSchemaResp.getRelateDimension();
if (relateDimension == null || CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {

12
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/pojo/SchemaItem.java
View File

@@ -5,13 +5,18 @@ import com.tencent.supersonic.common.pojo.RecordInfo;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import lombok.Data;
import lombok.ToString;
import org.apache.commons.lang3.StringUtils;
@Data
@ToString(callSuper = true)
public class SchemaItem extends RecordInfo {
private static String aliasSplit = ",";
private Long id;
private String name;
@@ -49,4 +54,11 @@ public class SchemaItem extends RecordInfo {
public int hashCode() {
return Objects.hashCode(super.hashCode(), id, name, bizName, description, status, typeEnum, sensitiveLevel);
}
public static List<String> getAliasList(String alias) {
if (StringUtils.isEmpty(alias)) {
return new ArrayList<>();
}
return Arrays.asList(alias.split(aliasSplit));
}
}

33
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -27,13 +27,16 @@ import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -172,14 +175,30 @@ public class QueryReqConverter {
protected Map<String, String> getFieldNameToBizNameMap(ModelSchemaResp modelSchemaResp) {
List<SchemaItem> allSchemaItems = new ArrayList<>();
allSchemaItems.addAll(modelSchemaResp.getDimensions());
allSchemaItems.addAll(modelSchemaResp.getMetrics());
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = modelSchemaResp.getDimensions().stream()
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1));
Map<String, String> result = allSchemaItems.stream()
.collect(Collectors.toMap(SchemaItem::getName, a -> a.getBizName(), (k1, k2) -> k1));
result.put(DateUtils.DATE_FIELD, TimeDimensionEnum.DAY.getName());
return result;
Map<String, String> metricResults = modelSchemaResp.getMetrics().stream()
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(a -> a.getLeft(), a -> a.getRight(), (k1, k2) -> k1));
dimensionResults.put(DateUtils.DATE_FIELD, TimeDimensionEnum.DAY.getName());
dimensionResults.putAll(metricResults);
return dimensionResults;
}
private Stream<Pair<String, String>> getPairStream(String aliasStr, String name, String bizName) {
Set<Pair<String, String>> elements = new HashSet<>();
elements.add(Pair.of(name, bizName));
if (StringUtils.isNotBlank(aliasStr)) {
List<String> aliasList = SchemaItem.getAliasList(aliasStr);
for (String alias : aliasList) {
elements.add(Pair.of(alias, bizName));
}
}
return elements.stream();
}
public void correctTableName(QueryS2QLReq databaseReq) {