topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Modify s2ql to s2sql and add LLMSemanticQuery containing S2SQLQuery and MetricInterpretQuery. (#350)

Browse Source
This commit is contained in:
lexluo09
2023-11-09 21:59:27 +08:00
committed by GitHub
parent acee0a36da
commit 7d33c49db8
90 changed files with 379 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java
View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
@@ -38,7 +38,7 @@ public interface SemanticInterpreter {
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user); QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user);
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);

6
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java
View File

@@ -5,7 +5,7 @@ import lombok.Data;
@Data @Data
public class SqlInfo { public class SqlInfo {
private String s2QL; private String s2SQL;
private String logicSql; private String correctS2SQL;
private String querySql; private String querySQL;
} }

2
chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentToolType.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.agent.tool;
public enum AgentToolType { public enum AgentToolType {
RULE, RULE,
LLM_S2QL, LLM_S2SQL,
PLUGIN, PLUGIN,
INTERPRET INTERPRET
} }

4
chat/core/src/main/java/com/tencent/supersonic/chat/config/OptimizationConfig.java
View File

@@ -39,8 +39,8 @@ public class OptimizationConfig {
@Value("${candidate.threshold}") @Value("${candidate.threshold}")
private Double candidateThreshold; private Double candidateThreshold;
@Value("${user.s2ql.switch:false}") @Value("${user.s2SQL.switch:false}")
private boolean useS2qlSwitch; private boolean useS2SqlSwitch;
@Value("${embedding.mapper.word.min:4}") @Value("${embedding.mapper.word.min:4}")
private int embeddingMapperWordMin; private int embeddingMapperWordMin;

8
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/BaseSemanticCorrector.java
View File

@@ -27,7 +27,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
public void correct(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { public void correct(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
try { try {
if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getLogicSql())) { if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getCorrectS2SQL())) {
return; return;
} }
work(queryReq, semanticParseInfo); work(queryReq, semanticParseInfo);
@@ -83,12 +83,12 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
needAddFields.removeAll(selectFields); needAddFields.removeAll(selectFields);
needAddFields.remove(TimeDimensionEnum.DAY.getChName()); needAddFields.remove(TimeDimensionEnum.DAY.getChName());
String replaceFields = SqlParserAddHelper.addFieldsToSelect(logicSql, new ArrayList<>(needAddFields)); String replaceFields = SqlParserAddHelper.addFieldsToSelect(logicSql, new ArrayList<>(needAddFields));
semanticParseInfo.getSqlInfo().setLogicSql(replaceFields); semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceFields);
} }
protected void addAggregateToMetric(SemanticParseInfo semanticParseInfo) { protected void addAggregateToMetric(SemanticParseInfo semanticParseInfo) {
//add aggregate to all metric //add aggregate to all metric
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql(); String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
Long modelId = semanticParseInfo.getModel().getModel(); Long modelId = semanticParseInfo.getModel().getModel();
List<SchemaElement> metrics = getMetricElements(modelId); List<SchemaElement> metrics = getMetricElements(modelId);
@@ -105,7 +105,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
return; return;
} }
String aggregateSql = SqlParserAddHelper.addAggregateToField(logicSql, metricToAggregate); String aggregateSql = SqlParserAddHelper.addAggregateToField(logicSql, metricToAggregate);
semanticParseInfo.getSqlInfo().setLogicSql(aggregateSql); semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql);
} }
protected List<SchemaElement> getMetricElements(Long modelId) { protected List<SchemaElement> getMetricElements(Long modelId) {

4
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalAfterCorrector.java
View File

@@ -15,14 +15,14 @@ public class GlobalAfterCorrector extends BaseSemanticCorrector {
@Override @Override
public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql(); String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
if (!SqlParserSelectFunctionHelper.hasAggregateFunction(logicSql)) { if (!SqlParserSelectFunctionHelper.hasAggregateFunction(logicSql)) {
return; return;
} }
Expression havingExpression = SqlParserSelectHelper.getHavingExpression(logicSql); Expression havingExpression = SqlParserSelectHelper.getHavingExpression(logicSql);
if (Objects.nonNull(havingExpression)) { if (Objects.nonNull(havingExpression)) {
String replaceSql = SqlParserAddHelper.addFunctionToSelect(logicSql, havingExpression); String replaceSql = SqlParserAddHelper.addFunctionToSelect(logicSql, havingExpression);
semanticParseInfo.getSqlInfo().setLogicSql(replaceSql); semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceSql);
} }
return; return;
} }

22
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java
View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult; import com.tencent.supersonic.chat.parser.llm.s2sql.ParseResult;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -34,15 +34,15 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
private void replaceAlias(SemanticParseInfo semanticParseInfo) { private void replaceAlias(SemanticParseInfo semanticParseInfo) {
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String replaceAlias = SqlParserReplaceHelper.replaceAlias(sqlInfo.getLogicSql()); String replaceAlias = SqlParserReplaceHelper.replaceAlias(sqlInfo.getCorrectS2SQL());
sqlInfo.setLogicSql(replaceAlias); sqlInfo.setCorrectS2SQL(replaceAlias);
} }
private void correctFieldName(SemanticParseInfo semanticParseInfo) { private void correctFieldName(SemanticParseInfo semanticParseInfo) {
Map<String, String> fieldNameMap = getFieldNameMap(semanticParseInfo.getModelId()); Map<String, String> fieldNameMap = getFieldNameMap(semanticParseInfo.getModelId());
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getLogicSql(), fieldNameMap); String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap);
sqlInfo.setLogicSql(sql); sqlInfo.setCorrectS2SQL(sql);
} }
private void updateFieldNameByLinkingValue(SemanticParseInfo semanticParseInfo) { private void updateFieldNameByLinkingValue(SemanticParseInfo semanticParseInfo) {
@@ -57,8 +57,8 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String sql = SqlParserReplaceHelper.replaceFieldNameByValue(sqlInfo.getLogicSql(), fieldValueToFieldNames); String sql = SqlParserReplaceHelper.replaceFieldNameByValue(sqlInfo.getCorrectS2SQL(), fieldValueToFieldNames);
sqlInfo.setLogicSql(sql); sqlInfo.setCorrectS2SQL(sql);
} }
private List<ElementValue> getLinkingValues(SemanticParseInfo semanticParseInfo) { private List<ElementValue> getLinkingValues(SemanticParseInfo semanticParseInfo) {
@@ -91,7 +91,7 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
))); )));
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String sql = SqlParserReplaceHelper.replaceValue(sqlInfo.getLogicSql(), filedNameToValueMap, false); String sql = SqlParserReplaceHelper.replaceValue(sqlInfo.getCorrectS2SQL(), filedNameToValueMap, false);
sqlInfo.setLogicSql(sql); sqlInfo.setCorrectS2SQL(sql);
} }
} }

6
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GroupByCorrector.java
View File

@@ -31,7 +31,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
//add dimension group by //add dimension group by
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo(); SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String logicSql = sqlInfo.getLogicSql(); String logicSql = sqlInfo.getCorrectS2SQL();
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
//add alias field name //add alias field name
Set<String> dimensions = semanticSchema.getDimensions(modelId).stream() Set<String> dimensions = semanticSchema.getDimensions(modelId).stream()
@@ -71,14 +71,14 @@ public class GroupByCorrector extends BaseSemanticCorrector {
return true; return true;
}) })
.collect(Collectors.toSet()); .collect(Collectors.toSet());
semanticParseInfo.getSqlInfo().setLogicSql(SqlParserAddHelper.addGroupBy(logicSql, groupByFields)); semanticParseInfo.getSqlInfo().setCorrectS2SQL(SqlParserAddHelper.addGroupBy(logicSql, groupByFields));
addAggregate(semanticParseInfo); addAggregate(semanticParseInfo);
} }
private void addAggregate(SemanticParseInfo semanticParseInfo) { private void addAggregate(SemanticParseInfo semanticParseInfo) {
List<String> sqlGroupByFields = SqlParserSelectHelper.getGroupByFields( List<String> sqlGroupByFields = SqlParserSelectHelper.getGroupByFields(
semanticParseInfo.getSqlInfo().getLogicSql()); semanticParseInfo.getSqlInfo().getCorrectS2SQL());
if (CollectionUtils.isEmpty(sqlGroupByFields)) { if (CollectionUtils.isEmpty(sqlGroupByFields)) {
return; return;
} }

4
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/HavingCorrector.java
View File

@@ -28,8 +28,8 @@ public class HavingCorrector extends BaseSemanticCorrector {
if (CollectionUtils.isEmpty(metrics)) { if (CollectionUtils.isEmpty(metrics)) {
return; return;
} }
String havingSql = SqlParserAddHelper.addHaving(semanticParseInfo.getSqlInfo().getLogicSql(), metrics); String havingSql = SqlParserAddHelper.addHaving(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), metrics);
semanticParseInfo.getSqlInfo().setLogicSql(havingSql); semanticParseInfo.getSqlInfo().setCorrectS2SQL(havingSql);
} }
} }

2
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/SelectCorrector.java
View File

@@ -12,7 +12,7 @@ public class SelectCorrector extends BaseSemanticCorrector {
@Override @Override
public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql(); String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
List<String> aggregateFields = SqlParserSelectHelper.getAggregateFields(logicSql); List<String> aggregateFields = SqlParserSelectHelper.getAggregateFields(logicSql);
List<String> selectFields = SqlParserSelectHelper.getSelectFields(logicSql); List<String> selectFields = SqlParserSelectHelper.getSelectFields(logicSql);
// If the number of aggregated fields is equal to the number of queried fields, do not add fields to select. // If the number of aggregated fields is equal to the number of queried fields, do not add fields to select.

20
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java
View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.llm.s2ql.S2QLDateHelper; import com.tencent.supersonic.chat.parser.llm.s2sql.S2SQLDateHelper;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
@@ -46,7 +46,7 @@ public class WhereCorrector extends BaseSemanticCorrector {
private void addQueryFilter(QueryReq queryReq, SemanticParseInfo semanticParseInfo) { private void addQueryFilter(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
String queryFilter = getQueryFilter(queryReq.getQueryFilters()); String queryFilter = getQueryFilter(queryReq.getQueryFilters());
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql(); String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
if (StringUtils.isNotEmpty(queryFilter)) { if (StringUtils.isNotEmpty(queryFilter)) {
log.info("add queryFilter to logicSql :{}", queryFilter); log.info("add queryFilter to logicSql :{}", queryFilter);
@@ -57,27 +57,27 @@ public class WhereCorrector extends BaseSemanticCorrector {
log.error("parseCondExpression", e); log.error("parseCondExpression", e);
} }
logicSql = SqlParserAddHelper.addWhere(logicSql, expression); logicSql = SqlParserAddHelper.addWhere(logicSql, expression);
semanticParseInfo.getSqlInfo().setLogicSql(logicSql); semanticParseInfo.getSqlInfo().setCorrectS2SQL(logicSql);
} }
} }
private void parserDateDiffFunction(SemanticParseInfo semanticParseInfo) { private void parserDateDiffFunction(SemanticParseInfo semanticParseInfo) {
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql(); String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
logicSql = SqlParserReplaceHelper.replaceFunction(logicSql); logicSql = SqlParserReplaceHelper.replaceFunction(logicSql);
semanticParseInfo.getSqlInfo().setLogicSql(logicSql); semanticParseInfo.getSqlInfo().setCorrectS2SQL(logicSql);
} }
private void addDateIfNotExist(SemanticParseInfo semanticParseInfo) { private void addDateIfNotExist(SemanticParseInfo semanticParseInfo) {
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql(); String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
List<String> whereFields = SqlParserSelectHelper.getWhereFields(logicSql); List<String> whereFields = SqlParserSelectHelper.getWhereFields(logicSql);
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(TimeDimensionEnum.DAY.getChName())) { if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(TimeDimensionEnum.DAY.getChName())) {
String currentDate = S2QLDateHelper.getReferenceDate(semanticParseInfo.getModelId()); String currentDate = S2SQLDateHelper.getReferenceDate(semanticParseInfo.getModelId());
if (StringUtils.isNotBlank(currentDate)) { if (StringUtils.isNotBlank(currentDate)) {
logicSql = SqlParserAddHelper.addParenthesisToWhere(logicSql); logicSql = SqlParserAddHelper.addParenthesisToWhere(logicSql);
logicSql = SqlParserAddHelper.addWhere(logicSql, TimeDimensionEnum.DAY.getChName(), currentDate); logicSql = SqlParserAddHelper.addWhere(logicSql, TimeDimensionEnum.DAY.getChName(), currentDate);
} }
} }
semanticParseInfo.getSqlInfo().setLogicSql(logicSql); semanticParseInfo.getSqlInfo().setCorrectS2SQL(logicSql);
} }
private String getQueryFilter(QueryFilters queryFilters) { private String getQueryFilter(QueryFilters queryFilters) {
@@ -106,9 +106,9 @@ public class WhereCorrector extends BaseSemanticCorrector {
} }
Map<String, Map<String, String>> aliasAndBizNameToTechName = getAliasAndBizNameToTechName(dimensions); Map<String, Map<String, String>> aliasAndBizNameToTechName = getAliasAndBizNameToTechName(dimensions);
String logicSql = SqlParserReplaceHelper.replaceValue(semanticParseInfo.getSqlInfo().getLogicSql(), String logicSql = SqlParserReplaceHelper.replaceValue(semanticParseInfo.getSqlInfo().getCorrectS2SQL(),
aliasAndBizNameToTechName); aliasAndBizNameToTechName);
semanticParseInfo.getSqlInfo().setLogicSql(logicSql); semanticParseInfo.getSqlInfo().setCorrectS2SQL(logicSql);
return; return;
} }

4
chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.OptimizationConfig; import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -20,7 +20,7 @@ public class SatisfactionChecker {
// check all the parse info in candidate // check all the parse info in candidate
public static boolean check(QueryContext queryContext) { public static boolean check(QueryContext queryContext) {
for (SemanticQuery query : queryContext.getCandidateQueries()) { for (SemanticQuery query : queryContext.getCandidateQueries()) {
if (query.getQueryMode().equals(S2QLQuery.QUERY_MODE)) { if (query.getQueryMode().equals(S2SQLQuery.QUERY_MODE)) {
continue; continue;
} }
if (checkThreshold(queryContext.getRequest().getQueryText(), query.getParseInfo())) { if (checkThreshold(queryContext.getRequest().getQueryText(), query.getParseInfo())) {

23
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/interpret/MetricInterpretParser.java
View File

@@ -7,33 +7,32 @@ import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.MetricInterpretTool; import com.tencent.supersonic.chat.agent.tool.MetricInterpretTool;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import lombok.extern.slf4j.Slf4j; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import org.springframework.util.CollectionUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.HashMap;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Slf4j @Slf4j
public class MetricInterpretParser implements SemanticParser { public class MetricInterpretParser implements SemanticParser {
@@ -71,7 +70,7 @@ public class MetricInterpretParser implements SemanticParser {
private void buildQuery(Long modelId, QueryContext queryContext, private void buildQuery(Long modelId, QueryContext queryContext,
List<Long> metricIds, List<SchemaElementMatch> schemaElementMatches, String toolName) { List<Long> metricIds, List<SchemaElementMatch> schemaElementMatches, String toolName) {
PluginSemanticQuery metricInterpretQuery = QueryManager.createPluginQuery(MetricInterpretQuery.QUERY_MODE); LLMSemanticQuery metricInterpretQuery = QueryManager.createLLMQuery(MetricInterpretQuery.QUERY_MODE);
Set<SchemaElement> metrics = getMetrics(metricIds, modelId); Set<SchemaElement> metrics = getMetrics(metricIds, modelId);
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, queryContext.getRequest(), SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, queryContext.getRequest(),
metrics, schemaElementMatches, toolName); metrics, schemaElementMatches, toolName);

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/HeuristicModelResolver.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/HeuristicModelResolver.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;

18
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMRequestService.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMRequestService.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.agent.tool.AgentToolType; import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
@@ -12,9 +12,9 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.config.LLMParserConfig; import com.tencent.supersonic.chat.config.LLMParserConfig;
import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp; import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
@@ -66,18 +66,18 @@ public class LLMRequestService {
public boolean check(QueryContext queryCtx) { public boolean check(QueryContext queryCtx) {
QueryReq request = queryCtx.getRequest(); QueryReq request = queryCtx.getRequest();
if (StringUtils.isEmpty(llmParserConfig.getUrl())) { if (StringUtils.isEmpty(llmParserConfig.getUrl())) {
log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMS2QLParser.class, llmParserConfig); log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMS2SQLParser.class, llmParserConfig);
return true; return true;
} }
if (SatisfactionChecker.check(queryCtx)) { if (SatisfactionChecker.check(queryCtx)) {
log.info("skip {}, queryText:{}", LLMS2QLParser.class, request.getQueryText()); log.info("skip {}, queryText:{}", LLMS2SQLParser.class, request.getQueryText());
return true; return true;
} }
return false; return false;
} }
public Long getModelId(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) { public Long getModelId(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) {
Set<Long> distinctModelIds = agentService.getModelIds(agentId, AgentToolType.LLM_S2QL); Set<Long> distinctModelIds = agentService.getModelIds(agentId, AgentToolType.LLM_S2SQL);
if (agentService.containsAllModel(distinctModelIds)) { if (agentService.containsAllModel(distinctModelIds)) {
distinctModelIds = new HashSet<>(); distinctModelIds = new HashSet<>();
} }
@@ -89,7 +89,7 @@ public class LLMRequestService {
public CommonAgentTool getParserTool(QueryReq request, Long modelId) { public CommonAgentTool getParserTool(QueryReq request, Long modelId) {
List<CommonAgentTool> commonAgentTools = agentService.getParserTools(request.getAgentId(), List<CommonAgentTool> commonAgentTools = agentService.getParserTools(request.getAgentId(),
AgentToolType.LLM_S2QL); AgentToolType.LLM_S2SQL);
Optional<CommonAgentTool> llmParserTool = commonAgentTools.stream() Optional<CommonAgentTool> llmParserTool = commonAgentTools.stream()
.filter(tool -> { .filter(tool -> {
List<Long> modelIds = tool.getModelIds(); List<Long> modelIds = tool.getModelIds();
@@ -131,7 +131,7 @@ public class LLMRequestService {
linking.addAll(getValueList(queryCtx, modelId, semanticSchema)); linking.addAll(getValueList(queryCtx, modelId, semanticSchema));
llmReq.setLinking(linking); llmReq.setLinking(linking);
String currentDate = S2QLDateHelper.getReferenceDate(modelId); String currentDate = S2SQLDateHelper.getReferenceDate(modelId);
if (StringUtils.isEmpty(currentDate)) { if (StringUtils.isEmpty(currentDate)) {
currentDate = DateUtils.getBeforeDate(0); currentDate = DateUtils.getBeforeDate(0);
} }

12
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMResponseService.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMResponseService.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.QueryContext;
@@ -6,8 +6,8 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.service.SchemaService; import com.tencent.supersonic.knowledge.service.SchemaService;
@@ -20,11 +20,11 @@ import org.springframework.stereotype.Service;
@Slf4j @Slf4j
@Service @Service
public class LLMResponseService { public class LLMResponseService {
public SemanticParseInfo addParseInfo(QueryContext queryCtx, ParseResult parseResult, String s2ql, Double weight) { public SemanticParseInfo addParseInfo(QueryContext queryCtx, ParseResult parseResult, String s2SQL, Double weight) {
if (Objects.isNull(weight)) { if (Objects.isNull(weight)) {
weight = 0D; weight = 0D;
} }
PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(S2QLQuery.QUERY_MODE); LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(S2SQLQuery.QUERY_MODE);
SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
Long modelId = parseResult.getModelId(); Long modelId = parseResult.getModelId();
CommonAgentTool commonAgentTool = parseResult.getCommonAgentTool(); CommonAgentTool commonAgentTool = parseResult.getCommonAgentTool();
@@ -38,7 +38,7 @@ public class LLMResponseService {
parseInfo.setProperties(properties); parseInfo.setProperties(properties);
parseInfo.setScore(queryCtx.getRequest().getQueryText().length() * (1 + weight)); parseInfo.setScore(queryCtx.getRequest().getQueryText().length() * (1 + weight));
parseInfo.setQueryMode(semanticQuery.getQueryMode()); parseInfo.setQueryMode(semanticQuery.getQueryMode());
parseInfo.getSqlInfo().setS2QL(s2ql); parseInfo.getSqlInfo().setS2SQL(s2SQL);
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
Map<Long, String> modelIdToName = semanticSchema.getModelIdToName(); Map<Long, String> modelIdToName = semanticSchema.getModelIdToName();

12
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMS2SQLParser.java
View File

@@ -1,19 +1,19 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp; import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public class LLMS2QLParser implements SemanticParser { public class LLMS2SQLParser implements SemanticParser {
@Override @Override
public void parse(QueryContext queryCtx, ChatContext chatCtx) { public void parse(QueryContext queryCtx, ChatContext chatCtx) {
@@ -32,7 +32,7 @@ public class LLMS2QLParser implements SemanticParser {
//3.get agent tool and determine whether to skip this parser. //3.get agent tool and determine whether to skip this parser.
CommonAgentTool commonAgentTool = requestService.getParserTool(request, modelId); CommonAgentTool commonAgentTool = requestService.getParserTool(request, modelId);
if (Objects.isNull(commonAgentTool)) { if (Objects.isNull(commonAgentTool)) {
log.info("no tool in this agent, skip {}", LLMS2QLParser.class); log.info("no tool in this agent, skip {}", LLMS2SQLParser.class);
return; return;
} }
//4.construct a request, call the API for the large model, and retrieve the results. //4.construct a request, call the API for the large model, and retrieve the results.
@@ -62,7 +62,7 @@ public class LLMS2QLParser implements SemanticParser {
} }
} catch (Exception e) { } catch (Exception e) {
log.error("LLMS2QLParser error", e); log.error("parse", e);
} }
} }

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelMatchResult.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ModelMatchResult.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import lombok.Data; import lombok.Data;

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelResolver.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ModelResolver.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;

6
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ParseResult.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/ParseResult.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq; import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp; import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

4
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/S2QLDateHelper.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2sql/S2SQLDateHelper.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
@@ -11,7 +11,7 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
public class S2QLDateHelper { public class S2SQLDateHelper {
public static String getReferenceDate(Long modelId) { public static String getReferenceDate(Long modelId) {
String defaultDate = DateUtils.getBeforeDate(0); String defaultDate = DateUtils.getBeforeDate(0);

4
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseConfig; import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.plugin.PluginRecallResult; import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.chat.service.PluginService; import com.tencent.supersonic.chat.service.PluginService;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.net.URI; import java.net.URI;
@@ -101,7 +101,7 @@ public class FunctionBasedParser extends PluginParser {
log.info("user decide Model:{}", modelId); log.info("user decide Model:{}", modelId);
List<Plugin> plugins = getPluginList(queryContext); List<Plugin> plugins = getPluginList(queryContext);
List<PluginParseConfig> functionDOList = plugins.stream().filter(plugin -> { List<PluginParseConfig> functionDOList = plugins.stream().filter(plugin -> {
if (S2QLQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) { if (S2SQLQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
return false; return false;
} }
if (plugin.getParseModeConfig() == null) { if (plugin.getParseModeConfig() == null) {

12
chat/core/src/main/java/com/tencent/supersonic/chat/query/BaseSemanticQuery.java
View File

@@ -16,7 +16,7 @@ import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;
@@ -40,10 +40,10 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
ExplainSqlReq explainSqlReq = null; ExplainSqlReq explainSqlReq = null;
SqlInfo sqlInfo = parseInfo.getSqlInfo(); SqlInfo sqlInfo = parseInfo.getSqlInfo();
try { try {
QueryS2QLReq queryS2QLReq = QueryReqBuilder.buildS2QLReq(sqlInfo.getLogicSql(), parseInfo.getModelId()); QueryS2SQLReq queryS2SQLReq = QueryReqBuilder.buildS2SQLReq(sqlInfo.getCorrectS2SQL(), parseInfo.getModelId());
explainSqlReq = ExplainSqlReq.builder() explainSqlReq = ExplainSqlReq.builder()
.queryTypeEnum(QueryTypeEnum.SQL) .queryTypeEnum(QueryTypeEnum.SQL)
.queryReq(queryS2QLReq) .queryReq(queryS2SQLReq)
.build(); .build();
ExplainResp explain = semanticInterpreter.explain(explainSqlReq, user); ExplainResp explain = semanticInterpreter.explain(explainSqlReq, user);
if (Objects.nonNull(explain)) { if (Objects.nonNull(explain)) {
@@ -105,9 +105,9 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
protected void initS2SqlByStruct() { protected void initS2SqlByStruct() {
QueryStructReq queryStructReq = convertQueryStruct(); QueryStructReq queryStructReq = convertQueryStruct();
convertBizNameToName(queryStructReq); convertBizNameToName(queryStructReq);
QueryS2QLReq queryS2QLReq = queryStructReq.convert(queryStructReq); QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq);
parseInfo.getSqlInfo().setS2QL(queryS2QLReq.getSql()); parseInfo.getSqlInfo().setS2SQL(queryS2SQLReq.getSql());
parseInfo.getSqlInfo().setLogicSql(queryS2QLReq.getSql()); parseInfo.getSqlInfo().setCorrectS2SQL(queryS2SQLReq.getSql());
} }
} }

37
chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryManager.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.chat.query; package com.tencent.supersonic.chat.query;
import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.query.rule.entity.EntitySemanticQuery; import com.tencent.supersonic.chat.query.rule.entity.EntitySemanticQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -16,25 +16,45 @@ public class QueryManager {
private static Map<String, RuleSemanticQuery> ruleQueryMap = new ConcurrentHashMap<>(); private static Map<String, RuleSemanticQuery> ruleQueryMap = new ConcurrentHashMap<>();
private static Map<String, PluginSemanticQuery> pluginQueryMap = new ConcurrentHashMap<>(); private static Map<String, PluginSemanticQuery> pluginQueryMap = new ConcurrentHashMap<>();
private static Map<String, LLMSemanticQuery> llmQueryMap = new ConcurrentHashMap<>();
public static void register(SemanticQuery query) { public static void register(SemanticQuery query) {
if (query instanceof RuleSemanticQuery) { if (query instanceof RuleSemanticQuery) {
ruleQueryMap.put(query.getQueryMode(), (RuleSemanticQuery) query); ruleQueryMap.put(query.getQueryMode(), (RuleSemanticQuery) query);
} else if (query instanceof PluginSemanticQuery) { } else if (query instanceof PluginSemanticQuery) {
pluginQueryMap.put(query.getQueryMode(), (PluginSemanticQuery) query); pluginQueryMap.put(query.getQueryMode(), (PluginSemanticQuery) query);
} else if (query instanceof LLMSemanticQuery) {
llmQueryMap.put(query.getQueryMode(), (LLMSemanticQuery) query);
} }
} }
public static SemanticQuery createQuery(String queryMode) { public static SemanticQuery createQuery(String queryMode) {
if (containsRuleQuery(queryMode)) { if (containsRuleQuery(queryMode)) {
return createRuleQuery(queryMode); return createRuleQuery(queryMode);
} else { }
if (containsPluginQuery(queryMode)) {
return createPluginQuery(queryMode); return createPluginQuery(queryMode);
} }
return createLLMQuery(queryMode);
} }
public static RuleSemanticQuery createRuleQuery(String queryMode) { public static RuleSemanticQuery createRuleQuery(String queryMode) {
RuleSemanticQuery semanticQuery = ruleQueryMap.get(queryMode); RuleSemanticQuery semanticQuery = ruleQueryMap.get(queryMode);
return (RuleSemanticQuery) getSemanticQuery(queryMode, semanticQuery);
}
public static PluginSemanticQuery createPluginQuery(String queryMode) {
PluginSemanticQuery semanticQuery = pluginQueryMap.get(queryMode);
return (PluginSemanticQuery) getSemanticQuery(queryMode, semanticQuery);
}
public static LLMSemanticQuery createLLMQuery(String queryMode) {
LLMSemanticQuery semanticQuery = llmQueryMap.get(queryMode);
return (LLMSemanticQuery) getSemanticQuery(queryMode, semanticQuery);
}
private static SemanticQuery getSemanticQuery(String queryMode, SemanticQuery semanticQuery) {
if (Objects.isNull(semanticQuery)) { if (Objects.isNull(semanticQuery)) {
throw new RuntimeException("no supported queryMode :" + queryMode); throw new RuntimeException("no supported queryMode :" + queryMode);
} }
@@ -45,17 +65,6 @@ public class QueryManager {
} }
} }
public static PluginSemanticQuery createPluginQuery(String queryMode) {
PluginSemanticQuery semanticQuery = pluginQueryMap.get(queryMode);
if (Objects.isNull(semanticQuery)) {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
try {
return semanticQuery.getClass().getDeclaredConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
}
public static boolean containsRuleQuery(String queryMode) { public static boolean containsRuleQuery(String queryMode) {
if (queryMode == null) { if (queryMode == null) {
return false; return false;
@@ -77,7 +86,7 @@ public class QueryManager {
return ruleQueryMap.get(queryMode) instanceof EntitySemanticQuery; return ruleQueryMap.get(queryMode) instanceof EntitySemanticQuery;
} }
public static boolean isPluginQuery(String queryMode) { public static boolean containsPluginQuery(String queryMode) {
return queryMode != null && pluginQueryMap.containsKey(queryMode); return queryMode != null && pluginQueryMap.containsKey(queryMode);
} }

14
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/LLMSemanticQuery.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.chat.query.llm;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.query.BaseSemanticQuery;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class LLMSemanticQuery extends BaseSemanticQuery {
@Override
public void initS2Sql(User user) {
}
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLmAnswerReq.java → chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLMAnswerReq.java
View File

@@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.query.llm.interpret;
import lombok.Data; import lombok.Data;
@Data @Data
public class LLmAnswerReq { public class LLMAnswerReq {
private String queryText; private String queryText;

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLmAnswerResp.java → chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/LLMAnswerResp.java
View File

@@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.query.llm.interpret;
import lombok.Data; import lombok.Data;
@Data @Data
public class LLmAnswerResp { public class LLMAnswerResp {
private String assistantMessage; private String assistantMessage;
} }

14
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/MetricInterpretQuery.java
View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.config.OptimizationConfig; import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder; import com.tencent.supersonic.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Aggregator;
@@ -35,7 +35,7 @@ import org.springframework.util.CollectionUtils;
@Slf4j @Slf4j
@Component @Component
public class MetricInterpretQuery extends PluginSemanticQuery { public class MetricInterpretQuery extends LLMSemanticQuery {
public static final String QUERY_MODE = "METRIC_INTERPRET"; public static final String QUERY_MODE = "METRIC_INTERPRET";
@@ -56,9 +56,9 @@ public class MetricInterpretQuery extends PluginSemanticQuery {
SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class); OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (optimizationConfig.isUseS2qlSwitch()) { if (optimizationConfig.isUseS2SqlSwitch()) {
queryStructReq.setS2QL(parseInfo.getSqlInfo().getS2QL()); queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setS2QL(parseInfo.getSqlInfo().getQuerySql()); queryStructReq.setS2SQL(parseInfo.getSqlInfo().getQuerySQL());
} }
QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user); QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user);
@@ -151,12 +151,12 @@ public class MetricInterpretQuery extends PluginSemanticQuery {
public String fetchInterpret(String queryText, String dataText) { public String fetchInterpret(String queryText, String dataText) {
PluginManager pluginManager = ContextUtils.getBean(PluginManager.class); PluginManager pluginManager = ContextUtils.getBean(PluginManager.class);
LLmAnswerReq lLmAnswerReq = new LLmAnswerReq(); LLMAnswerReq lLmAnswerReq = new LLMAnswerReq();
lLmAnswerReq.setQueryText(queryText); lLmAnswerReq.setQueryText(queryText);
lLmAnswerReq.setPluginOutput(dataText); lLmAnswerReq.setPluginOutput(dataText);
ResponseEntity<String> responseEntity = pluginManager.doRequest("answer_with_plugin_call", ResponseEntity<String> responseEntity = pluginManager.doRequest("answer_with_plugin_call",
JSONObject.toJSONString(lLmAnswerReq)); JSONObject.toJSONString(lLmAnswerReq));
LLmAnswerResp lLmAnswerResp = JSONObject.parseObject(responseEntity.getBody(), LLmAnswerResp.class); LLMAnswerResp lLmAnswerResp = JSONObject.parseObject(responseEntity.getBody(), LLMAnswerResp.class);
if (lLmAnswerResp != null) { if (lLmAnswerResp != null) {
return lLmAnswerResp.getAssistantMessage(); return lLmAnswerResp.getAssistantMessage();
} }

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMReq.java → chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2sql/LLMReq.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.s2ql; package com.tencent.supersonic.chat.query.llm.s2sql;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMResp.java → chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2sql/LLMResp.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.s2ql; package com.tencent.supersonic.chat.query.llm.s2sql;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;

22
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/S2QLQuery.java → chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2sql/S2SQLQuery.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.s2ql; package com.tencent.supersonic.chat.query.llm.s2sql;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
@@ -6,12 +6,12 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder; import com.tencent.supersonic.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -21,12 +21,12 @@ import org.springframework.stereotype.Component;
@Slf4j @Slf4j
@Component @Component
public class S2QLQuery extends PluginSemanticQuery { public class S2SQLQuery extends LLMSemanticQuery {
public static final String QUERY_MODE = "LLM_S2QL"; public static final String QUERY_MODE = "LLM_S2SQL";
protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
public S2QLQuery() { public S2SQLQuery() {
QueryManager.register(this); QueryManager.register(this);
} }
@@ -39,11 +39,11 @@ public class S2QLQuery extends PluginSemanticQuery {
public QueryResult execute(User user) { public QueryResult execute(User user) {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
String querySql = parseInfo.getSqlInfo().getLogicSql(); String querySql = parseInfo.getSqlInfo().getCorrectS2SQL();
QueryS2QLReq queryS2QLReq = QueryReqBuilder.buildS2QLReq(querySql, parseInfo.getModelId()); QueryS2SQLReq queryS2SQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModelId());
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2QL(queryS2QLReq, user); QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2SQL(queryS2SQLReq, user);
log.info("queryByS2QL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql); log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql);
QueryResult queryResult = new QueryResult(); QueryResult queryResult = new QueryResult();
if (Objects.nonNull(queryResp)) { if (Objects.nonNull(queryResp)) {
@@ -65,6 +65,6 @@ public class S2QLQuery extends PluginSemanticQuery {
@Override @Override
public void initS2Sql(User user) { public void initS2Sql(User user) {
SqlInfo sqlInfo = parseInfo.getSqlInfo(); SqlInfo sqlInfo = parseInfo.getSqlInfo();
sqlInfo.setLogicSql(sqlInfo.getS2QL()); sqlInfo.setCorrectS2SQL(sqlInfo.getS2SQL());
} }
} }

6
chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java
View File

@@ -200,9 +200,9 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
QueryStructReq queryStructReq = convertQueryStruct(); QueryStructReq queryStructReq = convertQueryStruct();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class); OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (optimizationConfig.isUseS2qlSwitch()) { if (optimizationConfig.isUseS2SqlSwitch()) {
queryStructReq.setS2QL(parseInfo.getSqlInfo().getS2QL()); queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setLogicSql(parseInfo.getSqlInfo().getLogicSql()); queryStructReq.setCorrectS2SQL(parseInfo.getSqlInfo().getCorrectS2SQL());
} }
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);

4
chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java
View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List; import java.util.List;
@@ -24,7 +24,7 @@ public class EntityInfoExecuteResponder implements ExecuteResponder {
return; return;
} }
String queryMode = semanticParseInfo.getQueryMode(); String queryMode = semanticParseInfo.getQueryMode();
if (QueryManager.isPluginQuery(queryMode) && !S2QLQuery.QUERY_MODE.equals(queryMode)) { if (QueryManager.containsPluginQuery(queryMode) || MetricInterpretQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) {
return; return;
} }
SemanticService semanticService = ContextUtils.getBean(SemanticService.class); SemanticService semanticService = ContextUtils.getBean(SemanticService.class);

12
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List; import java.util.List;
@@ -18,22 +18,22 @@ public class EntityInfoParseResponder implements ParseResponder {
@Override @Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext, public void fillResponse(ParseResp parseResp, QueryContext queryContext,
List<ChatParseDO> chatParseDOS) { List<ChatParseDO> chatParseDOS) {
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses(); List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
if (CollectionUtils.isEmpty(selectedParses)) { if (CollectionUtils.isEmpty(selectedParses)) {
return; return;
} }
QueryReq queryReq = queryContext.getRequest(); QueryReq queryReq = queryContext.getRequest();
selectedParses.forEach(parseInfo -> { selectedParses.forEach(parseInfo -> {
if (QueryManager.isPluginQuery(parseInfo.getQueryMode()) String queryMode = parseInfo.getQueryMode();
&& !S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { if (QueryManager.containsPluginQuery(queryMode) || MetricInterpretQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) {
return; return;
} }
//1. set entity info //1. set entity info
SemanticService semanticService = ContextUtils.getBean(SemanticService.class); SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryReq.getUser()); EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryReq.getUser());
if (QueryManager.isEntityQuery(parseInfo.getQueryMode()) if (QueryManager.isEntityQuery(queryMode)
|| QueryManager.isMetricQuery(parseInfo.getQueryMode())) { || QueryManager.isMetricQuery(queryMode)) {
parseInfo.setEntityInfo(entityInfo); parseInfo.setEntityInfo(entityInfo);
} }
//2. set native value //2. set native value

2
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/SqlInfoParseResponder.java
View File

@@ -68,7 +68,7 @@ public class SqlInfoParseResponder implements ParseResponder {
if (StringUtils.isBlank(explainSql)) { if (StringUtils.isBlank(explainSql)) {
return; return;
} }
parseInfo.getSqlInfo().setQuerySql(explainSql); parseInfo.getSqlInfo().setQuerySQL(explainSql);
} }
} }

10
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java
View File

@@ -73,7 +73,7 @@ public class ParserInfoServiceImpl implements ParseInfoService {
public void updateParseInfo(SemanticParseInfo parseInfo) { public void updateParseInfo(SemanticParseInfo parseInfo) {
SqlInfo sqlInfo = parseInfo.getSqlInfo(); SqlInfo sqlInfo = parseInfo.getSqlInfo();
String logicSql = sqlInfo.getLogicSql(); String logicSql = sqlInfo.getCorrectS2SQL();
if (StringUtils.isBlank(logicSql)) { if (StringUtils.isBlank(logicSql)) {
return; return;
} }
@@ -103,20 +103,20 @@ public class ParserInfoServiceImpl implements ParseInfoService {
if (Objects.isNull(semanticSchema)) { if (Objects.isNull(semanticSchema)) {
return; return;
} }
List<String> allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getLogicSql())); List<String> allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL()));
Set<SchemaElement> metrics = getElements(parseInfo.getModelId(), allFields, semanticSchema.getMetrics()); Set<SchemaElement> metrics = getElements(parseInfo.getModelId(), allFields, semanticSchema.getMetrics());
parseInfo.setMetrics(metrics); parseInfo.setMetrics(metrics);
if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getLogicSql())) { if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectS2SQL())) {
parseInfo.setNativeQuery(false); parseInfo.setNativeQuery(false);
List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getLogicSql()); List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL());
List<String> groupByDimensions = getFieldsExceptDate(groupByFields); List<String> groupByDimensions = getFieldsExceptDate(groupByFields);
parseInfo.setDimensions( parseInfo.setDimensions(
getElements(parseInfo.getModelId(), groupByDimensions, semanticSchema.getDimensions())); getElements(parseInfo.getModelId(), groupByDimensions, semanticSchema.getDimensions()));
} else { } else {
parseInfo.setNativeQuery(true); parseInfo.setNativeQuery(true);
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getLogicSql()); List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL());
List<String> selectDimensions = getFieldsExceptDate(selectFields); List<String> selectDimensions = getFieldsExceptDate(selectFields);
parseInfo.setDimensions( parseInfo.setDimensions(
getElements(parseInfo.getModelId(), selectDimensions, semanticSchema.getDimensions())); getElements(parseInfo.getModelId(), selectDimensions, semanticSchema.getDimensions()));

12
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -27,7 +27,7 @@ import com.tencent.supersonic.chat.persistence.dataobject.CostType;
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO; import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.QuerySelector; import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery; import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder;
import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.service.ChatService;
@@ -291,11 +291,11 @@ public class QueryServiceImpl implements QueryService {
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { if (S2SQLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>(); Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
Map<String, Map<String, String>> havingFiledNameToValueMap = new HashMap<>(); Map<String, Map<String, String>> havingFiledNameToValueMap = new HashMap<>();
String correctorSql = parseInfo.getSqlInfo().getLogicSql(); String correctorSql = parseInfo.getSqlInfo().getCorrectS2SQL();
log.info("correctorSql before replacing:{}", correctorSql); log.info("correctorSql before replacing:{}", correctorSql);
// get where filter and having filter // get where filter and having filter
List<FilterExpression> whereExpressionList = SqlParserSelectHelper.getWhereExpressions(correctorSql); List<FilterExpression> whereExpressionList = SqlParserSelectHelper.getWhereExpressions(correctorSql);
@@ -321,11 +321,11 @@ public class QueryServiceImpl implements QueryService {
correctorSql = SqlParserAddHelper.addHaving(correctorSql, addHavingConditions); correctorSql = SqlParserAddHelper.addHaving(correctorSql, addHavingConditions);
log.info("correctorSql after replacing:{}", correctorSql); log.info("correctorSql after replacing:{}", correctorSql);
parseInfo.getSqlInfo().setLogicSql(correctorSql); parseInfo.getSqlInfo().setCorrectS2SQL(correctorSql);
semanticQuery.setParseInfo(parseInfo); semanticQuery.setParseInfo(parseInfo);
String explainSql = semanticQuery.explain(user); String explainSql = semanticQuery.explain(user);
if (StringUtils.isNotBlank(explainSql)) { if (StringUtils.isNotBlank(explainSql)) {
parseInfo.getSqlInfo().setQuerySql(explainSql); parseInfo.getSqlInfo().setQuerySQL(explainSql);
} }
} }
semanticQuery.setParseInfo(parseInfo); semanticQuery.setParseInfo(parseInfo);
@@ -522,7 +522,7 @@ public class QueryServiceImpl implements QueryService {
private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) { private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) {
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { if (S2SQLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
return parseInfo; return parseInfo;
} }
if (CollectionUtils.isNotEmpty(queryData.getDimensions())) { if (CollectionUtils.isNotEmpty(queryData.getDimensions())) {

8
chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java
View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver; import com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver;
import com.tencent.supersonic.chat.query.QuerySelector; import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder;
@@ -20,7 +20,7 @@ public class ComponentFactory {
private static List<SchemaMapper> schemaMappers = new ArrayList<>(); private static List<SchemaMapper> schemaMappers = new ArrayList<>();
private static List<SemanticParser> semanticParsers = new ArrayList<>(); private static List<SemanticParser> semanticParsers = new ArrayList<>();
private static List<SemanticCorrector> s2QLCorrections = new ArrayList<>(); private static List<SemanticCorrector> s2SQLCorrections = new ArrayList<>();
private static SemanticInterpreter semanticInterpreter; private static SemanticInterpreter semanticInterpreter;
private static List<ParseResponder> parseResponders = new ArrayList<>(); private static List<ParseResponder> parseResponders = new ArrayList<>();
private static List<ExecuteResponder> executeResponders = new ArrayList<>(); private static List<ExecuteResponder> executeResponders = new ArrayList<>();
@@ -35,8 +35,8 @@ public class ComponentFactory {
} }
public static List<SemanticCorrector> getSqlCorrections() { public static List<SemanticCorrector> getSqlCorrections() {
return CollectionUtils.isEmpty(s2QLCorrections) ? init(SemanticCorrector.class, return CollectionUtils.isEmpty(s2SQLCorrections) ? init(SemanticCorrector.class,
s2QLCorrections) : s2QLCorrections; s2SQLCorrections) : s2SQLCorrections;
} }
public static List<ParseResponder> getParseResponders() { public static List<ParseResponder> getParseResponders() {

14
chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java
View File

@@ -13,7 +13,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.time.LocalDate; import java.time.LocalDate;
import java.util.ArrayList; import java.util.ArrayList;
@@ -124,19 +124,19 @@ public class QueryReqBuilder {
} }
/** /**
* convert to QueryS2QLReq * convert to QueryS2SQLReq
* *
* @param querySql * @param querySql
* @param modelId * @param modelId
* @return * @return
*/ */
public static QueryS2QLReq buildS2QLReq(String querySql, Long modelId) { public static QueryS2SQLReq buildS2SQLReq(String querySql, Long modelId) {
QueryS2QLReq queryS2QLReq = new QueryS2QLReq(); QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
if (Objects.nonNull(querySql)) { if (Objects.nonNull(querySql)) {
queryS2QLReq.setSql(querySql); queryS2SQLReq.setSql(querySql);
} }
queryS2QLReq.setModelId(modelId); queryS2SQLReq.setModelId(modelId);
return queryS2QLReq; return queryS2SQLReq;
} }
private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) { private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {

4
chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParserTest.java → chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/s2sql/LLMS2SQLParserTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql; package com.tencent.supersonic.chat.parser.llm.s2sql;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@@ -14,7 +14,7 @@ import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic; import org.mockito.MockedStatic;
import org.mockito.Mockito; import org.mockito.Mockito;
class LLMS2QLParserTest { class LLMS2SQLParserTest {
@Test @Test
void setFilter() { void setFilter() {

12
chat/core/src/test/java/com/tencent/supersonic/chat/utils/QueryReqBuilderTest.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@@ -25,7 +25,7 @@ import org.mockito.Mockito;
class QueryReqBuilderTest { class QueryReqBuilderTest {
@Test @Test
void buildS2QLReq() { void buildS2SQLReq() {
init(); init();
QueryStructReq queryStructReq = new QueryStructReq(); QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setModelId(1L); queryStructReq.setModelId(1L);
@@ -50,17 +50,17 @@ class QueryReqBuilderTest {
orders.add(order); orders.add(order);
queryStructReq.setOrders(orders); queryStructReq.setOrders(orders);
QueryS2QLReq queryS2QLReq = queryStructReq.convert(queryStructReq); QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals( Assert.assertEquals(
"SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " "SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "GROUP BY department ORDER BY uv LIMIT 2000", queryS2QLReq.getSql()); + "GROUP BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
queryStructReq.setNativeQuery(true); queryStructReq.setNativeQuery(true);
queryS2QLReq = queryStructReq.convert(queryStructReq); queryS2SQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals( Assert.assertEquals(
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "ORDER BY uv LIMIT 2000", + "ORDER BY uv LIMIT 2000",
queryS2QLReq.getSql()); queryS2SQLReq.getSql());
} }

18
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java
View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DimensionService;
@@ -41,12 +41,12 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@SneakyThrows @SneakyThrows
@Override @Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) { public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getLogicSql())) { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QueryS2QLReq queryS2QLReq = new QueryS2QLReq(); QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
queryS2QLReq.setSql(queryStructReq.getLogicSql()); queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
queryS2QLReq.setModelId(queryStructReq.getModelId()); queryS2SQLReq.setModelId(queryStructReq.getModelId());
queryS2QLReq.setVariables(new HashMap<>()); queryS2SQLReq.setVariables(new HashMap<>());
return queryByS2QL(queryS2QLReq, user); return queryByS2SQL(queryS2SQLReq, user);
} }
queryService = ContextUtils.getBean(QueryService.class); queryService = ContextUtils.getBean(QueryService.class);
return queryService.queryByStructWithAuth(queryStructReq, user); return queryService.queryByStructWithAuth(queryStructReq, user);
@@ -65,9 +65,9 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@Override @Override
@SneakyThrows @SneakyThrows
public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) { public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) {
queryService = ContextUtils.getBean(QueryService.class); queryService = ContextUtils.getBean(QueryService.class);
Object object = queryService.queryBySql(queryS2QLReq, user); Object object = queryService.queryBySql(queryS2SQLReq, user);
return JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp.class); return JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp.class);
} }

18
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java
View File

@@ -31,7 +31,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.net.URI; import java.net.URI;
@@ -70,12 +70,12 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
@Override @Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) { public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getLogicSql())) { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QueryS2QLReq queryS2QLReq = new QueryS2QLReq(); QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
queryS2QLReq.setSql(queryStructReq.getLogicSql()); queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
queryS2QLReq.setModelId(queryStructReq.getModelId()); queryS2SQLReq.setModelId(queryStructReq.getModelId());
queryS2QLReq.setVariables(new HashMap<>()); queryS2SQLReq.setVariables(new HashMap<>());
return queryByS2QL(queryS2QLReq, user); return queryByS2SQL(queryS2SQLReq, user);
} }
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
@@ -93,10 +93,10 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
} }
@Override @Override
public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) { public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(), return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
new Gson().toJson(queryS2QLReq)); new Gson().toJson(queryS2SQLReq));
} }
public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) { public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) {

6
launchers/chat/src/main/resources/META-INF/spring.factories
View File

@@ -7,7 +7,7 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\
com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \ com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \
com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \ com.tencent.supersonic.chat.parser.llm.s2sql.LLMS2SQLParser, \
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser
com.tencent.supersonic.chat.api.component.SemanticCorrector=\ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
@@ -24,8 +24,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\ com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver com.tencent.supersonic.chat.parser.llm.s2sql.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

6
launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java
View File

@@ -230,7 +230,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
LLMParserTool llmParserTool = new LLMParserTool(); LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1"); llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.LLM_S2QL); llmParserTool.setType(AgentToolType.LLM_S2SQL);
llmParserTool.setModelIds(Lists.newArrayList(-1L)); llmParserTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(llmParserTool); agentConfig.getTools().add(llmParserTool);
@@ -257,7 +257,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
LLMParserTool llmParserTool = new LLMParserTool(); LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1"); llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.LLM_S2QL); llmParserTool.setType(AgentToolType.LLM_S2SQL);
llmParserTool.setModelIds(Lists.newArrayList(-1L)); llmParserTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(llmParserTool); agentConfig.getTools().add(llmParserTool);
@@ -279,7 +279,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
LLMParserTool llmParserTool = new LLMParserTool(); LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1"); llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.LLM_S2QL); llmParserTool.setType(AgentToolType.LLM_S2SQL);
llmParserTool.setModelIds(Lists.newArrayList(3L)); llmParserTool.setModelIds(Lists.newArrayList(3L));
agentConfig.getTools().add(llmParserTool); agentConfig.getTools().add(llmParserTool);

6
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -7,7 +7,7 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\
com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \ com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \
com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \ com.tencent.supersonic.chat.parser.llm.s2sql.LLMS2SQLParser, \
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \ com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser
@@ -25,8 +25,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\ com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver com.tencent.supersonic.chat.parser.llm.s2sql.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

4
launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricInterpretTest.java
View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.query.llm.interpret.LLmAnswerResp; import com.tencent.supersonic.chat.query.llm.interpret.LLMAnswerResp;
import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.service.QueryService; import com.tencent.supersonic.chat.service.QueryService;
import com.tencent.supersonic.util.DataUtils; import com.tencent.supersonic.util.DataUtils;
@@ -45,7 +45,7 @@ public class MetricInterpretTest {
public void testMetricInterpret() throws Exception { public void testMetricInterpret() throws Exception {
MockConfiguration.mockAgent(agentService); MockConfiguration.mockAgent(agentService);
MockConfiguration.mockEmbeddingUrl(embeddingConfig); MockConfiguration.mockEmbeddingUrl(embeddingConfig);
LLmAnswerResp lLmAnswerResp = new LLmAnswerResp(); LLMAnswerResp lLmAnswerResp = new LLMAnswerResp();
lLmAnswerResp.setAssistantMessage("alice最近在超音数的访问情况有增多"); lLmAnswerResp.setAssistantMessage("alice最近在超音数的访问情况有增多");
MockConfiguration.mockPluginManagerDoRequest(pluginManager, "answer_with_plugin_call", MockConfiguration.mockPluginManagerDoRequest(pluginManager, "answer_with_plugin_call",
ResponseEntity.ok(JSONObject.toJSONString(lLmAnswerResp))); ResponseEntity.ok(JSONObject.toJSONString(lLmAnswerResp)));

2
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2QLReq.java → semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2SQLReq.java
View File

@@ -6,7 +6,7 @@ import lombok.ToString;
@Data @Data
@ToString @ToString
public class QueryS2QLReq { public class QueryS2SQLReq {
private Long modelId; private Long modelId;

8
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java
View File

@@ -62,11 +62,11 @@ public class QueryStructReq {
/** /**
* Later deleted for compatibility only * Later deleted for compatibility only
*/ */
private String s2QL; private String s2SQL;
/** /**
* Later deleted for compatibility only * Later deleted for compatibility only
*/ */
private String logicSql; private String correctS2SQL;
public List<String> getGroups() { public List<String> getGroups() {
if (!CollectionUtils.isEmpty(this.groups)) { if (!CollectionUtils.isEmpty(this.groups)) {
@@ -171,7 +171,7 @@ public class QueryStructReq {
* @param queryStructReq * @param queryStructReq
* @return * @return
*/ */
public QueryS2QLReq convert(QueryStructReq queryStructReq) { public QueryS2SQLReq convert(QueryStructReq queryStructReq) {
String sql = null; String sql = null;
try { try {
sql = buildSql(queryStructReq); sql = buildSql(queryStructReq);
@@ -179,7 +179,7 @@ public class QueryStructReq {
log.error("buildSql error", e); log.error("buildSql error", e);
} }
QueryS2QLReq result = new QueryS2QLReq(); QueryS2SQLReq result = new QueryS2SQLReq();
result.setSql(sql); result.setSql(sql);
result.setModelId(queryStructReq.getModelId()); result.setModelId(queryStructReq.getModelId());
result.setVariables(new HashMap<>()); result.setVariables(new HashMap<>());

20
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java
View File

@@ -20,16 +20,16 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager; import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.MaterializationPlanner; import com.tencent.supersonic.semantic.query.parser.calcite.planner.MaterializationPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataType; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils; import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SqlParser; import com.tencent.supersonic.semantic.query.parser.SqlParser;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;

20
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java
View File

@@ -11,16 +11,16 @@ import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataType; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DimensionTimeTypeParams; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MetricTypeParams; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;

18
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/MaterializationPlanner.java
View File

@@ -4,15 +4,15 @@ import com.tencent.supersonic.common.util.calcite.SqlParseUtils;
import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Constants.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/Constants.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
public class Constants { public class Constants {

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DataSource.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/DataSource.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import java.util.List; import java.util.List;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DataType.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/DataType.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.Arrays; import java.util.Arrays;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Dimension.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/Dimension.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DimensionTimeTypeParams.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/DimensionTimeTypeParams.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import lombok.Data; import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Identify.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/Identify.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Materialization.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/Materialization.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MaterializationElement.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/MaterializationElement.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.List; import java.util.List;
import lombok.Builder; import lombok.Builder;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Measure.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/Measure.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Metric.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/Metric.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MetricTypeParams.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/MetricTypeParams.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/SemanticModel.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/SemanticModel.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/TimeRange.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2sql/TimeRange.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql; package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SchemaBuilder.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.S2QLSqlValidatorImpl; import com.tencent.supersonic.semantic.query.parser.calcite.sql.S2SQLSqlValidatorImpl;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@@ -35,9 +35,9 @@ public class SchemaBuilder {
Configuration.typeFactory, Configuration.typeFactory,
Configuration.config Configuration.config
); );
S2QLSqlValidatorImpl s2QLSqlValidator = new S2QLSqlValidatorImpl(Configuration.operatorTable, catalogReader, S2SQLSqlValidatorImpl s2SQLSqlValidator = new S2SQLSqlValidatorImpl(Configuration.operatorTable, catalogReader,
Configuration.typeFactory, Configuration.validatorConfig); Configuration.typeFactory, Configuration.validatorConfig);
return new ParameterScope(s2QLSqlValidator, nameToTypeMap); return new ParameterScope(s2SQLSqlValidator, nameToTypeMap);
} }
public static CalciteSchema getMaterializationSchema() { public static CalciteSchema getMaterializationSchema() {

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.semantic.query.parser.calcite.schema; package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Renderer.java
View File

@@ -5,11 +5,11 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/S2QLSqlValidatorImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/S2SQLSqlValidatorImpl.java
View File

@@ -5,9 +5,9 @@ import org.apache.calcite.sql.SqlOperatorTable;
import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; import org.apache.calcite.sql.validate.SqlValidatorCatalogReader;
import org.apache.calcite.sql.validate.SqlValidatorImpl; import org.apache.calcite.sql.validate.SqlValidatorImpl;
public class S2QLSqlValidatorImpl extends SqlValidatorImpl { public class S2SQLSqlValidatorImpl extends SqlValidatorImpl {
public S2QLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader, public S2SQLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader,
RelDataTypeFactory typeFactory, Config config) { RelDataTypeFactory typeFactory, Config config) {
super(opTab, catalogReader, typeFactory, config); super(opTab, catalogReader, typeFactory, config);
} }

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql; package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.extend.LateralViewExplodeNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.extend.LateralViewExplodeNode;
import java.util.ArrayList; import java.util.ArrayList;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DimensionNode.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify.Type;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MeasureNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorScope;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MetricNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import lombok.Data; import lombok.Data;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/FilterRender.java
View File

@@ -6,9 +6,9 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList; import java.util.ArrayList;

14
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java
View File

@@ -1,15 +1,15 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render; package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/OutputRender.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList; import java.util.ArrayList;

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java
View File

@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable; import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.ModelService;
@@ -56,7 +56,7 @@ public class QueryReqConverter {
@Autowired @Autowired
private Catalog catalog; private Catalog catalog;
public QueryStatement convert(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception { public QueryStatement convert(QueryS2SQLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception {
if (Objects.isNull(modelSchemaResp)) { if (Objects.isNull(modelSchemaResp)) {
return new QueryStatement(); return new QueryStatement();
@@ -119,9 +119,9 @@ public class QueryReqConverter {
return queryStatement; return queryStatement;
} }
private AggOption getAggOption(QueryS2QLReq databaseReq) { private AggOption getAggOption(QueryS2SQLReq databaseReq) {
// if there is no group by in S2QL,set MetricTable's aggOption to "NATIVE" // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2QL,set MetricTable's aggOption to "NATIVE" // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql(); String sql = databaseReq.getSql();
if (!SqlParserSelectHelper.hasGroupBy(sql) if (!SqlParserSelectHelper.hasGroupBy(sql)
|| SqlParserSelectFunctionHelper.hasFunction(sql, "count") || SqlParserSelectFunctionHelper.hasFunction(sql, "count")
@@ -131,7 +131,7 @@ public class QueryReqConverter {
return AggOption.DEFAULT; return AggOption.DEFAULT;
} }
private void convertNameToBizName(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) { private void convertNameToBizName(QueryS2SQLReq databaseReq, ModelSchemaResp modelSchemaResp) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp); Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp);
String sql = databaseReq.getSql(); String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql); log.info("convert name to bizName before:{}", sql);
@@ -158,7 +158,7 @@ public class QueryReqConverter {
return metrics; return metrics;
} }
private void functionNameCorrector(QueryS2QLReq databaseReq) { private void functionNameCorrector(QueryS2SQLReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId()); DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId());
if (Objects.isNull(database) || Objects.isNull(database.getType())) { if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return; return;
@@ -208,7 +208,7 @@ public class QueryReqConverter {
return elements.stream(); return elements.stream();
} }
public void correctTableName(QueryS2QLReq databaseReq) { public void correctTableName(QueryS2SQLReq databaseReq) {
String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(), String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(),
Constants.TABLE_PREFIX + databaseReq.getModelId()); Constants.TABLE_PREFIX + databaseReq.getModelId());
databaseReq.setSql(sql); databaseReq.setSql(sql);

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
@@ -42,11 +42,11 @@ public class QueryController {
@PostMapping("/sql") @PostMapping("/sql")
public Object queryBySql(@RequestBody QueryS2QLReq queryS2QLReq, public Object queryBySql(@RequestBody QueryS2SQLReq queryS2SQLReq,
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response) throws Exception { HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response); User user = UserHolder.findUser(request, response);
Object queryBySql = queryService.queryBySql(queryS2QLReq, user); Object queryBySql = queryService.queryBySql(queryS2SQLReq, user);
log.info("queryBySql:{},queryBySql"); log.info("queryBySql:{},queryBySql");
return queryBySql; return queryBySql;
} }
@@ -124,9 +124,9 @@ public class QueryController {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum(); QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
if (QueryTypeEnum.SQL.equals(queryTypeEnum)) { if (QueryTypeEnum.SQL.equals(queryTypeEnum)) {
QueryS2QLReq queryS2QLReq = JsonUtil.toObject(queryReqJson, QueryS2QLReq.class); QueryS2SQLReq queryS2SQLReq = JsonUtil.toObject(queryReqJson, QueryS2SQLReq.class);
ExplainSqlReq<QueryS2QLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2QLReq>builder() ExplainSqlReq<QueryS2SQLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2SQLReq>builder()
.queryReq(queryS2QLReq) .queryReq(queryS2SQLReq)
.queryTypeEnum(queryTypeEnum).build(); .queryTypeEnum(queryTypeEnum).build();
return queryService.explain(explainSqlReqNew, user); return queryService.explain(explainSqlReqNew, user);
} }

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
@@ -18,7 +18,7 @@ import java.util.List;
public interface QueryService { public interface QueryService {
Object queryBySql(QueryS2QLReq querySqlCmd, User user) throws Exception; Object queryBySql(QueryS2SQLReq querySqlCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception; QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;

14
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -24,11 +24,11 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation; import com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor; import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter; import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
@@ -89,9 +89,9 @@ public class QueryServiceImpl implements QueryService {
} }
@Override @Override
@S2QLPermissionAnnotation @S2SQLPermissionAnnotation
@SneakyThrows @SneakyThrows
public Object queryBySql(QueryS2QLReq querySqlCmd, User user) { public Object queryBySql(QueryS2SQLReq querySqlCmd, User user) {
statUtils.initStatInfo(querySqlCmd, user); statUtils.initStatInfo(querySqlCmd, user);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
try { try {
@@ -109,7 +109,7 @@ public class QueryServiceImpl implements QueryService {
return semanticQueryEngine.execute(queryStatement); return semanticQueryEngine.execute(queryStatement);
} }
private QueryStatement convertToQueryStatement(QueryS2QLReq querySqlCmd, User user) throws Exception { private QueryStatement convertToQueryStatement(QueryS2SQLReq querySqlCmd, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = new ArrayList<>(); List<Long> modelIds = new ArrayList<>();
modelIds.add(querySqlCmd.getModelId()); modelIds.add(querySqlCmd.getModelId());
@@ -294,8 +294,8 @@ public class QueryServiceImpl implements QueryService {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum(); QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
T queryReq = explainSqlReq.getQueryReq(); T queryReq = explainSqlReq.getQueryReq();
if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2QLReq) { if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2SQLReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryS2QLReq) queryReq, user); QueryStatement queryStatement = convertToQueryStatement((QueryS2SQLReq) queryReq, user);
return getExplainResp(queryStatement); return getExplainResp(queryStatement);
} }
if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) { if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DimensionService;
import java.util.ArrayList; import java.util.ArrayList;
@@ -55,9 +55,9 @@ public class DimValueAspect {
return queryResultWithColumns; return queryResultWithColumns;
} }
Object[] args = joinPoint.getArgs(); Object[] args = joinPoint.getArgs();
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0]; QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2QLReq.getModelId())); MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelId()));
String sql = queryS2QLReq.getSql(); String sql = queryS2SQLReq.getSql();
log.info("correctorSql before replacing:{}", sql); log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue. // if dimensionvalue is alias,consider the true dimensionvalue.
List<FilterExpression> filterExpressionList = SqlParserSelectHelper.getWhereExpressions(sql); List<FilterExpression> filterExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
@@ -88,7 +88,7 @@ public class DimValueAspect {
log.info("filedNameToValueMap:{}", filedNameToValueMap); log.info("filedNameToValueMap:{}", filedNameToValueMap);
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap); sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql); log.info("correctorSql after replacing:{}", sql);
queryS2QLReq.setSql(sql); queryS2SQLReq.setSql(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions); Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -27,7 +27,7 @@ import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum; import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
@@ -199,8 +199,8 @@ public class QueryStructUtils {
return resNameEnSet; return resNameEnSet;
} }
public Set<String> getResName(QueryS2QLReq queryS2QLReq) { public Set<String> getResName(QueryS2SQLReq queryS2SQLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql()) Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql())
.stream().collect(Collectors.toSet()); .stream().collect(Collectors.toSet());
return resNameSet; return resNameSet;
} }
@@ -210,11 +210,11 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }
public Set<String> getResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq, User user) { public Set<String> getResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq, User user) {
Set<String> resNameSet = getResName(queryS2QLReq); Set<String> resNameSet = getResName(queryS2SQLReq);
Set<String> resNameEnSet = new HashSet<>(); Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(queryS2QLReq.getModelId()); List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelId());
filter.setModelIds(modelIds); filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user); List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) { if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -245,8 +245,8 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }
public Set<String> getFilterResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq) { public Set<String> getFilterResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq) {
String sql = queryS2QLReq.getSql(); String sql = queryS2SQLReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }

40
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2SQLDataAspect.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter; import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
@@ -42,7 +42,7 @@ import org.springframework.util.CollectionUtils;
@Aspect @Aspect
@Order(1) @Order(1)
@Slf4j @Slf4j
public class S2QLDataAspect { public class S2SQLDataAspect {
@Autowired @Autowired
private QueryStructUtils queryStructUtils; private QueryStructUtils queryStructUtils;
@@ -55,24 +55,24 @@ public class S2QLDataAspect {
@Value("${permission.data.enable:true}") @Value("${permission.data.enable:true}")
private Boolean permissionDataEnable; private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation)") @Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation)")
private void s2QLPermissionCheck() { private void s2SQLPermissionCheck() {
} }
@Around("s2QLPermissionCheck()") @Around("s2SQLPermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
log.info("s2ql permission check!"); log.info("s2SQL permission check!");
Object[] objects = joinPoint.getArgs(); Object[] objects = joinPoint.getArgs();
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) objects[0]; QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) objects[0];
User user = (User) objects[1]; User user = (User) objects[1];
if (!permissionDataEnable) { if (!permissionDataEnable) {
log.info("not to check s2QL permission!"); log.info("not to check s2SQL permission!");
return joinPoint.proceed(); return joinPoint.proceed();
} }
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) { if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information"); throw new RuntimeException("please provide user information");
} }
Long modelId = queryS2QLReq.getModelId(); Long modelId = queryS2SQLReq.getModelId();
//1. determine whether admin of the model //1. determine whether admin of the model
if (authCommonService.doModelAdmin(user, modelId)) { if (authCommonService.doModelAdmin(user, modelId)) {
@@ -82,7 +82,7 @@ public class S2QLDataAspect {
// 2. determine whether the subject field is visible // 2. determine whether the subject field is visible
authCommonService.doModelVisible(user, modelId); authCommonService.doModelVisible(user, modelId);
// 3. fetch data permission meta information // 3. fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2QLReq, user); Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2SQLReq, user);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege); log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId); Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId);
@@ -97,10 +97,10 @@ public class S2QLDataAspect {
Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId); Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId);
// 4.if sensitive fields without permission are involved in filter, thrown an exception // 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryS2QLReq, resAuthSet, sensitiveResReq); doFilterCheckLogic(queryS2SQLReq, resAuthSet, sensitiveResReq);
// 5.row permission pre-filter // 5.row permission pre-filter
doRowPermission(queryS2QLReq, authorizedResource); doRowPermission(queryS2SQLReq, authorizedResource);
// 6.proceed // 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
@@ -123,7 +123,7 @@ public class S2QLDataAspect {
return queryResultAfterDesensitization; return queryResultAfterDesensitization;
} }
private void doRowPermission(QueryS2QLReq queryS2QLReq, AuthorizedResourceResp authorizedResource) { private void doRowPermission(QueryS2SQLReq queryS2SQLReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic"); log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR "); StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>(); List<String> dimensionFilters = new ArrayList<>();
@@ -145,10 +145,10 @@ public class S2QLDataAspect {
try { try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) "); Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) { if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(queryS2QLReq.getSql(), expression); String sql = SqlParserAddHelper.addWhere(queryS2SQLReq.getSql(), expression);
log.info("before doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql()); log.info("before doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
queryS2QLReq.setSql(sql); queryS2SQLReq.setSql(sql);
log.info("after doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql()); log.info("after doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
} }
} catch (JSQLParserException jsqlParserException) { } catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
@@ -156,15 +156,15 @@ public class S2QLDataAspect {
} }
private void doFilterCheckLogic(QueryS2QLReq queryS2QLReq, Set<String> resAuthName, private void doFilterCheckLogic(QueryS2SQLReq queryS2SQLReq, Set<String> resAuthName,
Set<String> sensitiveResReq) { Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2QLReq); Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2SQLReq);
Set<String> need2Apply = resFilterSet.stream() Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>(); Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = new ArrayList<>(); List<Long> modelIds = new ArrayList<>();
modelIds.add(queryS2QLReq.getModelId()); modelIds.add(queryS2SQLReq.getModelId());
List<ModelResp> modelInfos = modelService.getModelList(modelIds); List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = Constants.EMPTY; String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) { if (!CollectionUtils.isEmpty(modelInfos)) {

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLPermissionAnnotation.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2SQLPermissionAnnotation.java
View File

@@ -9,6 +9,6 @@ import java.lang.annotation.Documented;
@Target(ElementType.METHOD) @Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME) @Retention(RetentionPolicy.RUNTIME)
@Documented @Documented
public @interface S2QLPermissionAnnotation { public @interface S2SQLPermissionAnnotation {
} }

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java
View File

@@ -13,7 +13,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.ModelService;
@@ -88,11 +88,11 @@ public class StatUtils {
} }
public void initStatInfo(QueryS2QLReq queryS2QLReq, User facadeUser) { public void initStatInfo(QueryS2SQLReq queryS2SQLReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat(); QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql()); List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
queryStatInfo.setModelId(queryS2QLReq.getModelId()); queryStatInfo.setModelId(queryS2SQLReq.getModelId());
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2QLReq.getModelId()); ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelId());
List<String> dimensions = new ArrayList<>(); List<String> dimensions = new ArrayList<>();
if (Objects.nonNull(modelSchemaResp)) { if (Objects.nonNull(modelSchemaResp)) {
@@ -107,12 +107,12 @@ public class StatUtils {
String userName = getUserName(facadeUser); String userName = getUserName(facadeUser);
try { try {
queryStatInfo.setTraceId("") queryStatInfo.setTraceId("")
.setModelId(queryS2QLReq.getModelId()) .setModelId(queryS2SQLReq.getModelId())
.setUser(userName) .setUser(userName)
.setQueryType(QueryTypeEnum.SQL.getValue()) .setQueryType(QueryTypeEnum.SQL.getValue())
.setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState()) .setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())
.setQuerySqlCmd(queryS2QLReq.toString()) .setQuerySqlCmd(queryS2SQLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2QLReq.toString())) .setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2SQLReq.toString()))
.setStartTime(System.currentTimeMillis()) .setStartTime(System.currentTimeMillis())
.setUseResultCache(true) .setUseResultCache(true)
.setUseSqlCache(true) .setUseSqlCache(true)