topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-20 06:34:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(project) dsl support get topN dimension/metric by useCount and fix semanticService get dimension/metric usrCount error (#105)

Browse Source
This commit is contained in:
lexluo09
2023-09-20 11:21:50 +08:00
committed by GitHub
parent c8ff37e304
commit 98656eb445
12 changed files with 154 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
chat/core/src/main/java/com/tencent/supersonic/chat/config/LLMParserConfig.java
View File

@@ -16,4 +16,10 @@ public class LLMParserConfig {
@Value("${query2sql.path:/query2sql}")
private String queryToSqlPath;
@Value("${dimension.topn:5}")
private Integer dimensionTopN;
@Value("${metric.topn:5}")
private Integer metricTopN;
}

35
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java
View File

@@ -39,6 +39,7 @@ import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -87,7 +88,7 @@ public class LLMDslParser implements SemanticParser {
return;
}
LLMReq llmReq = getLlmReq(queryCtx, modelId);
LLMReq llmReq = getLlmReq(queryCtx, modelId, llmParserConfig);
LLMResp llmResp = requestLLM(llmReq, modelId, llmParserConfig);
if (Objects.isNull(llmResp)) {
@@ -340,22 +341,28 @@ public class LLMDslParser implements SemanticParser {
return null;
}
private LLMReq getLlmReq(QueryContext queryCtx, Long modelId) {
private LLMReq getLlmReq(QueryContext queryCtx, Long modelId, LLMParserConfig llmParserConfig) {
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
Map<Long, String> modelIdToName = semanticSchema.getModelIdToName();
String queryText = queryCtx.getRequest().getQueryText();
LLMReq llmReq = new LLMReq();
llmReq.setQueryText(queryText);
LLMReq.LLMSchema llmSchema = new LLMReq.LLMSchema();
llmSchema.setModelName(modelIdToName.get(modelId));
llmSchema.setDomainName(modelIdToName.get(modelId));
List<String> fieldNameList = getFieldNameList(queryCtx, modelId, semanticSchema);
List<String> fieldNameList = getFieldNameList(queryCtx, modelId, semanticSchema, llmParserConfig);
fieldNameList.add(BaseSemanticCorrector.DATE_FIELD);
llmSchema.setFieldNameList(fieldNameList);
llmReq.setSchema(llmSchema);
List<ElementValue> linking = new ArrayList<>();
linking.addAll(getValueList(queryCtx, modelId, semanticSchema));
llmReq.setLinking(linking);
String currentDate = DSLDateHelper.getReferenceDate(modelId);
llmReq.setCurrentDate(currentDate);
return llmReq;
@@ -399,12 +406,27 @@ public class LLMDslParser implements SemanticParser {
}
protected List<String> getFieldNameList(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema) {
protected List<String> getFieldNameList(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema,
LLMParserConfig llmParserConfig) {
Map<Long, String> itemIdToName = getItemIdToName(modelId, semanticSchema);
Set<String> results = semanticSchema.getDimensions().stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(llmParserConfig.getDimensionTopN())
.map(entry -> entry.getName())
.collect(Collectors.toSet());
Set<String> metrics = semanticSchema.getMetrics().stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(llmParserConfig.getMetricTopN())
.map(entry -> entry.getName())
.collect(Collectors.toSet());
results.addAll(metrics);
List<SchemaElementMatch> matchedElements = queryCtx.getMapInfo().getMatchedElements(modelId);
if (CollectionUtils.isEmpty(matchedElements)) {
return new ArrayList<>();
return new ArrayList<>(results);
}
Set<String> fieldNameList = matchedElements.stream()
.filter(schemaElementMatch -> {
@@ -423,7 +445,8 @@ public class LLMDslParser implements SemanticParser {
})
.filter(name -> StringUtils.isNotEmpty(name) && !name.contains("%"))
.collect(Collectors.toSet());
return new ArrayList<>(fieldNameList);
results.addAll(fieldNameList);
return new ArrayList<>(results);
}
protected Map<Long, String> getItemIdToName(Long modelId, SemanticSchema semanticSchema) {