Modify s2ql to s2sql and add LLMSemanticQuery containing S2SQLQuery and MetricInterpretQuery. (#350)

This commit is contained in:
lexluo09
2023-11-09 21:59:27 +08:00
committed by GitHub
parent acee0a36da
commit 7d33c49db8
90 changed files with 379 additions and 357 deletions

View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
@@ -38,7 +38,7 @@ public interface SemanticInterpreter {
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user);
QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user);
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);

View File

@@ -5,7 +5,7 @@ import lombok.Data;
@Data
public class SqlInfo {
private String s2QL;
private String logicSql;
private String querySql;
private String s2SQL;
private String correctS2SQL;
private String querySQL;
}

View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.agent.tool;
public enum AgentToolType {
RULE,
LLM_S2QL,
LLM_S2SQL,
PLUGIN,
INTERPRET
}

View File

@@ -39,8 +39,8 @@ public class OptimizationConfig {
@Value("${candidate.threshold}")
private Double candidateThreshold;
@Value("${user.s2ql.switch:false}")
private boolean useS2qlSwitch;
@Value("${user.s2SQL.switch:false}")
private boolean useS2SqlSwitch;
@Value("${embedding.mapper.word.min:4}")
private int embeddingMapperWordMin;

View File

@@ -27,7 +27,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
public void correct(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
try {
if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getLogicSql())) {
if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getCorrectS2SQL())) {
return;
}
work(queryReq, semanticParseInfo);
@@ -83,12 +83,12 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
needAddFields.removeAll(selectFields);
needAddFields.remove(TimeDimensionEnum.DAY.getChName());
String replaceFields = SqlParserAddHelper.addFieldsToSelect(logicSql, new ArrayList<>(needAddFields));
semanticParseInfo.getSqlInfo().setLogicSql(replaceFields);
semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceFields);
}
protected void addAggregateToMetric(SemanticParseInfo semanticParseInfo) {
//add aggregate to all metric
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql();
String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
Long modelId = semanticParseInfo.getModel().getModel();
List<SchemaElement> metrics = getMetricElements(modelId);
@@ -105,7 +105,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
return;
}
String aggregateSql = SqlParserAddHelper.addAggregateToField(logicSql, metricToAggregate);
semanticParseInfo.getSqlInfo().setLogicSql(aggregateSql);
semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql);
}
protected List<SchemaElement> getMetricElements(Long modelId) {

View File

@@ -15,14 +15,14 @@ public class GlobalAfterCorrector extends BaseSemanticCorrector {
@Override
public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql();
String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
if (!SqlParserSelectFunctionHelper.hasAggregateFunction(logicSql)) {
return;
}
Expression havingExpression = SqlParserSelectHelper.getHavingExpression(logicSql);
if (Objects.nonNull(havingExpression)) {
String replaceSql = SqlParserAddHelper.addFunctionToSelect(logicSql, havingExpression);
semanticParseInfo.getSqlInfo().setLogicSql(replaceSql);
semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceSql);
}
return;
}

View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue;
import com.tencent.supersonic.chat.parser.llm.s2sql.ParseResult;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -34,15 +34,15 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
private void replaceAlias(SemanticParseInfo semanticParseInfo) {
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String replaceAlias = SqlParserReplaceHelper.replaceAlias(sqlInfo.getLogicSql());
sqlInfo.setLogicSql(replaceAlias);
String replaceAlias = SqlParserReplaceHelper.replaceAlias(sqlInfo.getCorrectS2SQL());
sqlInfo.setCorrectS2SQL(replaceAlias);
}
private void correctFieldName(SemanticParseInfo semanticParseInfo) {
Map<String, String> fieldNameMap = getFieldNameMap(semanticParseInfo.getModelId());
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getLogicSql(), fieldNameMap);
sqlInfo.setLogicSql(sql);
String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap);
sqlInfo.setCorrectS2SQL(sql);
}
private void updateFieldNameByLinkingValue(SemanticParseInfo semanticParseInfo) {
@@ -57,8 +57,8 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String sql = SqlParserReplaceHelper.replaceFieldNameByValue(sqlInfo.getLogicSql(), fieldValueToFieldNames);
sqlInfo.setLogicSql(sql);
String sql = SqlParserReplaceHelper.replaceFieldNameByValue(sqlInfo.getCorrectS2SQL(), fieldValueToFieldNames);
sqlInfo.setCorrectS2SQL(sql);
}
private List<ElementValue> getLinkingValues(SemanticParseInfo semanticParseInfo) {
@@ -91,7 +91,7 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
)));
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String sql = SqlParserReplaceHelper.replaceValue(sqlInfo.getLogicSql(), filedNameToValueMap, false);
sqlInfo.setLogicSql(sql);
String sql = SqlParserReplaceHelper.replaceValue(sqlInfo.getCorrectS2SQL(), filedNameToValueMap, false);
sqlInfo.setCorrectS2SQL(sql);
}
}

View File

@@ -31,7 +31,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
//add dimension group by
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
String logicSql = sqlInfo.getLogicSql();
String logicSql = sqlInfo.getCorrectS2SQL();
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
//add alias field name
Set<String> dimensions = semanticSchema.getDimensions(modelId).stream()
@@ -71,14 +71,14 @@ public class GroupByCorrector extends BaseSemanticCorrector {
return true;
})
.collect(Collectors.toSet());
semanticParseInfo.getSqlInfo().setLogicSql(SqlParserAddHelper.addGroupBy(logicSql, groupByFields));
semanticParseInfo.getSqlInfo().setCorrectS2SQL(SqlParserAddHelper.addGroupBy(logicSql, groupByFields));
addAggregate(semanticParseInfo);
}
private void addAggregate(SemanticParseInfo semanticParseInfo) {
List<String> sqlGroupByFields = SqlParserSelectHelper.getGroupByFields(
semanticParseInfo.getSqlInfo().getLogicSql());
semanticParseInfo.getSqlInfo().getCorrectS2SQL());
if (CollectionUtils.isEmpty(sqlGroupByFields)) {
return;
}

View File

@@ -28,8 +28,8 @@ public class HavingCorrector extends BaseSemanticCorrector {
if (CollectionUtils.isEmpty(metrics)) {
return;
}
String havingSql = SqlParserAddHelper.addHaving(semanticParseInfo.getSqlInfo().getLogicSql(), metrics);
semanticParseInfo.getSqlInfo().setLogicSql(havingSql);
String havingSql = SqlParserAddHelper.addHaving(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), metrics);
semanticParseInfo.getSqlInfo().setCorrectS2SQL(havingSql);
}
}

View File

@@ -12,7 +12,7 @@ public class SelectCorrector extends BaseSemanticCorrector {
@Override
public void work(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql();
String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
List<String> aggregateFields = SqlParserSelectHelper.getAggregateFields(logicSql);
List<String> selectFields = SqlParserSelectHelper.getSelectFields(logicSql);
// If the number of aggregated fields is equal to the number of queried fields, do not add fields to select.

View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.llm.s2ql.S2QLDateHelper;
import com.tencent.supersonic.chat.parser.llm.s2sql.S2SQLDateHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
@@ -46,7 +46,7 @@ public class WhereCorrector extends BaseSemanticCorrector {
private void addQueryFilter(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
String queryFilter = getQueryFilter(queryReq.getQueryFilters());
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql();
String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
if (StringUtils.isNotEmpty(queryFilter)) {
log.info("add queryFilter to logicSql :{}", queryFilter);
@@ -57,27 +57,27 @@ public class WhereCorrector extends BaseSemanticCorrector {
log.error("parseCondExpression", e);
}
logicSql = SqlParserAddHelper.addWhere(logicSql, expression);
semanticParseInfo.getSqlInfo().setLogicSql(logicSql);
semanticParseInfo.getSqlInfo().setCorrectS2SQL(logicSql);
}
}
private void parserDateDiffFunction(SemanticParseInfo semanticParseInfo) {
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql();
String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
logicSql = SqlParserReplaceHelper.replaceFunction(logicSql);
semanticParseInfo.getSqlInfo().setLogicSql(logicSql);
semanticParseInfo.getSqlInfo().setCorrectS2SQL(logicSql);
}
private void addDateIfNotExist(SemanticParseInfo semanticParseInfo) {
String logicSql = semanticParseInfo.getSqlInfo().getLogicSql();
String logicSql = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
List<String> whereFields = SqlParserSelectHelper.getWhereFields(logicSql);
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(TimeDimensionEnum.DAY.getChName())) {
String currentDate = S2QLDateHelper.getReferenceDate(semanticParseInfo.getModelId());
String currentDate = S2SQLDateHelper.getReferenceDate(semanticParseInfo.getModelId());
if (StringUtils.isNotBlank(currentDate)) {
logicSql = SqlParserAddHelper.addParenthesisToWhere(logicSql);
logicSql = SqlParserAddHelper.addWhere(logicSql, TimeDimensionEnum.DAY.getChName(), currentDate);
}
}
semanticParseInfo.getSqlInfo().setLogicSql(logicSql);
semanticParseInfo.getSqlInfo().setCorrectS2SQL(logicSql);
}
private String getQueryFilter(QueryFilters queryFilters) {
@@ -106,9 +106,9 @@ public class WhereCorrector extends BaseSemanticCorrector {
}
Map<String, Map<String, String>> aliasAndBizNameToTechName = getAliasAndBizNameToTechName(dimensions);
String logicSql = SqlParserReplaceHelper.replaceValue(semanticParseInfo.getSqlInfo().getLogicSql(),
String logicSql = SqlParserReplaceHelper.replaceValue(semanticParseInfo.getSqlInfo().getCorrectS2SQL(),
aliasAndBizNameToTechName);
semanticParseInfo.getSqlInfo().setLogicSql(logicSql);
semanticParseInfo.getSqlInfo().setCorrectS2SQL(logicSql);
return;
}

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
@@ -20,7 +20,7 @@ public class SatisfactionChecker {
// check all the parse info in candidate
public static boolean check(QueryContext queryContext) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
if (query.getQueryMode().equals(S2QLQuery.QUERY_MODE)) {
if (query.getQueryMode().equals(S2SQLQuery.QUERY_MODE)) {
continue;
}
if (checkThreshold(queryContext.getRequest().getQueryText(), query.getParseInfo())) {

View File

@@ -7,33 +7,32 @@ import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.MetricInterpretTool;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashMap;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@Slf4j
public class MetricInterpretParser implements SemanticParser {
@@ -71,7 +70,7 @@ public class MetricInterpretParser implements SemanticParser {
private void buildQuery(Long modelId, QueryContext queryContext,
List<Long> metricIds, List<SchemaElementMatch> schemaElementMatches, String toolName) {
PluginSemanticQuery metricInterpretQuery = QueryManager.createPluginQuery(MetricInterpretQuery.QUERY_MODE);
LLMSemanticQuery metricInterpretQuery = QueryManager.createLLMQuery(MetricInterpretQuery.QUERY_MODE);
Set<SchemaElement> metrics = getMetrics(metricIds, modelId);
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, queryContext.getRequest(),
metrics, schemaElementMatches, toolName);

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
@@ -12,9 +12,9 @@ import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.config.LLMParserConfig;
import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
@@ -66,18 +66,18 @@ public class LLMRequestService {
public boolean check(QueryContext queryCtx) {
QueryReq request = queryCtx.getRequest();
if (StringUtils.isEmpty(llmParserConfig.getUrl())) {
log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMS2QLParser.class, llmParserConfig);
log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMS2SQLParser.class, llmParserConfig);
return true;
}
if (SatisfactionChecker.check(queryCtx)) {
log.info("skip {}, queryText:{}", LLMS2QLParser.class, request.getQueryText());
log.info("skip {}, queryText:{}", LLMS2SQLParser.class, request.getQueryText());
return true;
}
return false;
}
public Long getModelId(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) {
Set<Long> distinctModelIds = agentService.getModelIds(agentId, AgentToolType.LLM_S2QL);
Set<Long> distinctModelIds = agentService.getModelIds(agentId, AgentToolType.LLM_S2SQL);
if (agentService.containsAllModel(distinctModelIds)) {
distinctModelIds = new HashSet<>();
}
@@ -89,7 +89,7 @@ public class LLMRequestService {
public CommonAgentTool getParserTool(QueryReq request, Long modelId) {
List<CommonAgentTool> commonAgentTools = agentService.getParserTools(request.getAgentId(),
AgentToolType.LLM_S2QL);
AgentToolType.LLM_S2SQL);
Optional<CommonAgentTool> llmParserTool = commonAgentTools.stream()
.filter(tool -> {
List<Long> modelIds = tool.getModelIds();
@@ -131,7 +131,7 @@ public class LLMRequestService {
linking.addAll(getValueList(queryCtx, modelId, semanticSchema));
llmReq.setLinking(linking);
String currentDate = S2QLDateHelper.getReferenceDate(modelId);
String currentDate = S2SQLDateHelper.getReferenceDate(modelId);
if (StringUtils.isEmpty(currentDate)) {
currentDate = DateUtils.getBeforeDate(0);
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
@@ -6,8 +6,8 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.service.SchemaService;
@@ -20,11 +20,11 @@ import org.springframework.stereotype.Service;
@Slf4j
@Service
public class LLMResponseService {
public SemanticParseInfo addParseInfo(QueryContext queryCtx, ParseResult parseResult, String s2ql, Double weight) {
public SemanticParseInfo addParseInfo(QueryContext queryCtx, ParseResult parseResult, String s2SQL, Double weight) {
if (Objects.isNull(weight)) {
weight = 0D;
}
PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(S2QLQuery.QUERY_MODE);
LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(S2SQLQuery.QUERY_MODE);
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
Long modelId = parseResult.getModelId();
CommonAgentTool commonAgentTool = parseResult.getCommonAgentTool();
@@ -38,7 +38,7 @@ public class LLMResponseService {
parseInfo.setProperties(properties);
parseInfo.setScore(queryCtx.getRequest().getQueryText().length() * (1 + weight));
parseInfo.setQueryMode(semanticQuery.getQueryMode());
parseInfo.getSqlInfo().setS2QL(s2ql);
parseInfo.getSqlInfo().setS2SQL(s2SQL);
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
Map<Long, String> modelIdToName = semanticSchema.getModelIdToName();

View File

@@ -1,19 +1,19 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LLMS2QLParser implements SemanticParser {
public class LLMS2SQLParser implements SemanticParser {
@Override
public void parse(QueryContext queryCtx, ChatContext chatCtx) {
@@ -32,7 +32,7 @@ public class LLMS2QLParser implements SemanticParser {
//3.get agent tool and determine whether to skip this parser.
CommonAgentTool commonAgentTool = requestService.getParserTool(request, modelId);
if (Objects.isNull(commonAgentTool)) {
log.info("no tool in this agent, skip {}", LLMS2QLParser.class);
log.info("no tool in this agent, skip {}", LLMS2SQLParser.class);
return;
}
//4.construct a request, call the API for the large model, and retrieve the results.
@@ -62,7 +62,7 @@ public class LLMS2QLParser implements SemanticParser {
}
} catch (Exception e) {
log.error("LLMS2QLParser error", e);
log.error("parse", e);
}
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.api.pojo.ChatContext;

View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2sql.LLMResp;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
@@ -11,7 +11,7 @@ import java.util.List;
import java.util.Objects;
import org.apache.commons.collections.CollectionUtils;
public class S2QLDateHelper {
public class S2SQLDateHelper {
public static String getReferenceDate(Long modelId) {
String defaultDate = DateUtils.getBeforeDate(0);

View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.chat.service.PluginService;
import com.tencent.supersonic.common.util.ContextUtils;
import java.net.URI;
@@ -101,7 +101,7 @@ public class FunctionBasedParser extends PluginParser {
log.info("user decide Model:{}", modelId);
List<Plugin> plugins = getPluginList(queryContext);
List<PluginParseConfig> functionDOList = plugins.stream().filter(plugin -> {
if (S2QLQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
if (S2SQLQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
return false;
}
if (plugin.getParseModeConfig() == null) {

View File

@@ -16,7 +16,7 @@ import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.io.Serializable;
import java.util.List;
@@ -40,10 +40,10 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
ExplainSqlReq explainSqlReq = null;
SqlInfo sqlInfo = parseInfo.getSqlInfo();
try {
QueryS2QLReq queryS2QLReq = QueryReqBuilder.buildS2QLReq(sqlInfo.getLogicSql(), parseInfo.getModelId());
QueryS2SQLReq queryS2SQLReq = QueryReqBuilder.buildS2SQLReq(sqlInfo.getCorrectS2SQL(), parseInfo.getModelId());
explainSqlReq = ExplainSqlReq.builder()
.queryTypeEnum(QueryTypeEnum.SQL)
.queryReq(queryS2QLReq)
.queryReq(queryS2SQLReq)
.build();
ExplainResp explain = semanticInterpreter.explain(explainSqlReq, user);
if (Objects.nonNull(explain)) {
@@ -105,9 +105,9 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
protected void initS2SqlByStruct() {
QueryStructReq queryStructReq = convertQueryStruct();
convertBizNameToName(queryStructReq);
QueryS2QLReq queryS2QLReq = queryStructReq.convert(queryStructReq);
parseInfo.getSqlInfo().setS2QL(queryS2QLReq.getSql());
parseInfo.getSqlInfo().setLogicSql(queryS2QLReq.getSql());
QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq);
parseInfo.getSqlInfo().setS2SQL(queryS2SQLReq.getSql());
parseInfo.getSqlInfo().setCorrectS2SQL(queryS2SQLReq.getSql());
}
}

View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.chat.query;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.query.rule.entity.EntitySemanticQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -16,25 +16,45 @@ public class QueryManager {
private static Map<String, RuleSemanticQuery> ruleQueryMap = new ConcurrentHashMap<>();
private static Map<String, PluginSemanticQuery> pluginQueryMap = new ConcurrentHashMap<>();
private static Map<String, LLMSemanticQuery> llmQueryMap = new ConcurrentHashMap<>();
public static void register(SemanticQuery query) {
if (query instanceof RuleSemanticQuery) {
ruleQueryMap.put(query.getQueryMode(), (RuleSemanticQuery) query);
} else if (query instanceof PluginSemanticQuery) {
pluginQueryMap.put(query.getQueryMode(), (PluginSemanticQuery) query);
} else if (query instanceof LLMSemanticQuery) {
llmQueryMap.put(query.getQueryMode(), (LLMSemanticQuery) query);
}
}
public static SemanticQuery createQuery(String queryMode) {
if (containsRuleQuery(queryMode)) {
return createRuleQuery(queryMode);
} else {
}
if (containsPluginQuery(queryMode)) {
return createPluginQuery(queryMode);
}
return createLLMQuery(queryMode);
}
public static RuleSemanticQuery createRuleQuery(String queryMode) {
RuleSemanticQuery semanticQuery = ruleQueryMap.get(queryMode);
return (RuleSemanticQuery) getSemanticQuery(queryMode, semanticQuery);
}
public static PluginSemanticQuery createPluginQuery(String queryMode) {
PluginSemanticQuery semanticQuery = pluginQueryMap.get(queryMode);
return (PluginSemanticQuery) getSemanticQuery(queryMode, semanticQuery);
}
public static LLMSemanticQuery createLLMQuery(String queryMode) {
LLMSemanticQuery semanticQuery = llmQueryMap.get(queryMode);
return (LLMSemanticQuery) getSemanticQuery(queryMode, semanticQuery);
}
private static SemanticQuery getSemanticQuery(String queryMode, SemanticQuery semanticQuery) {
if (Objects.isNull(semanticQuery)) {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
@@ -45,17 +65,6 @@ public class QueryManager {
}
}
public static PluginSemanticQuery createPluginQuery(String queryMode) {
PluginSemanticQuery semanticQuery = pluginQueryMap.get(queryMode);
if (Objects.isNull(semanticQuery)) {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
try {
return semanticQuery.getClass().getDeclaredConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException("no supported queryMode :" + queryMode);
}
}
public static boolean containsRuleQuery(String queryMode) {
if (queryMode == null) {
return false;
@@ -77,7 +86,7 @@ public class QueryManager {
return ruleQueryMap.get(queryMode) instanceof EntitySemanticQuery;
}
public static boolean isPluginQuery(String queryMode) {
public static boolean containsPluginQuery(String queryMode) {
return queryMode != null && pluginQueryMap.containsKey(queryMode);
}

View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.chat.query.llm;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.query.BaseSemanticQuery;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class LLMSemanticQuery extends BaseSemanticQuery {
@Override
public void initS2Sql(User user) {
}
}

View File

@@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.query.llm.interpret;
import lombok.Data;
@Data
public class LLmAnswerReq {
public class LLMAnswerReq {
private String queryText;

View File

@@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.query.llm.interpret;
import lombok.Data;
@Data
public class LLmAnswerResp {
public class LLMAnswerResp {
private String assistantMessage;
}

View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.Aggregator;
@@ -35,7 +35,7 @@ import org.springframework.util.CollectionUtils;
@Slf4j
@Component
public class MetricInterpretQuery extends PluginSemanticQuery {
public class MetricInterpretQuery extends LLMSemanticQuery {
public static final String QUERY_MODE = "METRIC_INTERPRET";
@@ -56,9 +56,9 @@ public class MetricInterpretQuery extends PluginSemanticQuery {
SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (optimizationConfig.isUseS2qlSwitch()) {
queryStructReq.setS2QL(parseInfo.getSqlInfo().getS2QL());
queryStructReq.setS2QL(parseInfo.getSqlInfo().getQuerySql());
if (optimizationConfig.isUseS2SqlSwitch()) {
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getQuerySQL());
}
QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user);
@@ -151,12 +151,12 @@ public class MetricInterpretQuery extends PluginSemanticQuery {
public String fetchInterpret(String queryText, String dataText) {
PluginManager pluginManager = ContextUtils.getBean(PluginManager.class);
LLmAnswerReq lLmAnswerReq = new LLmAnswerReq();
LLMAnswerReq lLmAnswerReq = new LLMAnswerReq();
lLmAnswerReq.setQueryText(queryText);
lLmAnswerReq.setPluginOutput(dataText);
ResponseEntity<String> responseEntity = pluginManager.doRequest("answer_with_plugin_call",
JSONObject.toJSONString(lLmAnswerReq));
LLmAnswerResp lLmAnswerResp = JSONObject.parseObject(responseEntity.getBody(), LLmAnswerResp.class);
LLMAnswerResp lLmAnswerResp = JSONObject.parseObject(responseEntity.getBody(), LLMAnswerResp.class);
if (lLmAnswerResp != null) {
return lLmAnswerResp.getAssistantMessage();
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.s2ql;
package com.tencent.supersonic.chat.query.llm.s2sql;
import java.util.List;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.s2ql;
package com.tencent.supersonic.chat.query.llm.s2sql;
import java.util.List;
import java.util.Map;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.s2ql;
package com.tencent.supersonic.chat.query.llm.s2sql;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
@@ -6,12 +6,12 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -21,12 +21,12 @@ import org.springframework.stereotype.Component;
@Slf4j
@Component
public class S2QLQuery extends PluginSemanticQuery {
public class S2SQLQuery extends LLMSemanticQuery {
public static final String QUERY_MODE = "LLM_S2QL";
public static final String QUERY_MODE = "LLM_S2SQL";
protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
public S2QLQuery() {
public S2SQLQuery() {
QueryManager.register(this);
}
@@ -39,11 +39,11 @@ public class S2QLQuery extends PluginSemanticQuery {
public QueryResult execute(User user) {
long startTime = System.currentTimeMillis();
String querySql = parseInfo.getSqlInfo().getLogicSql();
QueryS2QLReq queryS2QLReq = QueryReqBuilder.buildS2QLReq(querySql, parseInfo.getModelId());
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2QL(queryS2QLReq, user);
String querySql = parseInfo.getSqlInfo().getCorrectS2SQL();
QueryS2SQLReq queryS2SQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModelId());
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2SQL(queryS2SQLReq, user);
log.info("queryByS2QL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql);
log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql);
QueryResult queryResult = new QueryResult();
if (Objects.nonNull(queryResp)) {
@@ -65,6 +65,6 @@ public class S2QLQuery extends PluginSemanticQuery {
@Override
public void initS2Sql(User user) {
SqlInfo sqlInfo = parseInfo.getSqlInfo();
sqlInfo.setLogicSql(sqlInfo.getS2QL());
sqlInfo.setCorrectS2SQL(sqlInfo.getS2SQL());
}
}

View File

@@ -200,9 +200,9 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
QueryStructReq queryStructReq = convertQueryStruct();
OptimizationConfig optimizationConfig = ContextUtils.getBean(OptimizationConfig.class);
if (optimizationConfig.isUseS2qlSwitch()) {
queryStructReq.setS2QL(parseInfo.getSqlInfo().getS2QL());
queryStructReq.setLogicSql(parseInfo.getSqlInfo().getLogicSql());
if (optimizationConfig.isUseS2SqlSwitch()) {
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setCorrectS2SQL(parseInfo.getSqlInfo().getCorrectS2SQL());
}
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);

View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List;
@@ -24,7 +24,7 @@ public class EntityInfoExecuteResponder implements ExecuteResponder {
return;
}
String queryMode = semanticParseInfo.getQueryMode();
if (QueryManager.isPluginQuery(queryMode) && !S2QLQuery.QUERY_MODE.equals(queryMode)) {
if (QueryManager.containsPluginQuery(queryMode) || MetricInterpretQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) {
return;
}
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);

View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List;
@@ -18,22 +18,22 @@ public class EntityInfoParseResponder implements ParseResponder {
@Override
public void fillResponse(ParseResp parseResp, QueryContext queryContext,
List<ChatParseDO> chatParseDOS) {
List<ChatParseDO> chatParseDOS) {
List<SemanticParseInfo> selectedParses = parseResp.getSelectedParses();
if (CollectionUtils.isEmpty(selectedParses)) {
return;
}
QueryReq queryReq = queryContext.getRequest();
selectedParses.forEach(parseInfo -> {
if (QueryManager.isPluginQuery(parseInfo.getQueryMode())
&& !S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
String queryMode = parseInfo.getQueryMode();
if (QueryManager.containsPluginQuery(queryMode) || MetricInterpretQuery.QUERY_MODE.equalsIgnoreCase(queryMode)) {
return;
}
//1. set entity info
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryReq.getUser());
if (QueryManager.isEntityQuery(parseInfo.getQueryMode())
|| QueryManager.isMetricQuery(parseInfo.getQueryMode())) {
if (QueryManager.isEntityQuery(queryMode)
|| QueryManager.isMetricQuery(queryMode)) {
parseInfo.setEntityInfo(entityInfo);
}
//2. set native value

View File

@@ -68,7 +68,7 @@ public class SqlInfoParseResponder implements ParseResponder {
if (StringUtils.isBlank(explainSql)) {
return;
}
parseInfo.getSqlInfo().setQuerySql(explainSql);
parseInfo.getSqlInfo().setQuerySQL(explainSql);
}
}

View File

@@ -73,7 +73,7 @@ public class ParserInfoServiceImpl implements ParseInfoService {
public void updateParseInfo(SemanticParseInfo parseInfo) {
SqlInfo sqlInfo = parseInfo.getSqlInfo();
String logicSql = sqlInfo.getLogicSql();
String logicSql = sqlInfo.getCorrectS2SQL();
if (StringUtils.isBlank(logicSql)) {
return;
}
@@ -103,20 +103,20 @@ public class ParserInfoServiceImpl implements ParseInfoService {
if (Objects.isNull(semanticSchema)) {
return;
}
List<String> allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getLogicSql()));
List<String> allFields = getFieldsExceptDate(SqlParserSelectHelper.getAllFields(sqlInfo.getCorrectS2SQL()));
Set<SchemaElement> metrics = getElements(parseInfo.getModelId(), allFields, semanticSchema.getMetrics());
parseInfo.setMetrics(metrics);
if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getLogicSql())) {
if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectS2SQL())) {
parseInfo.setNativeQuery(false);
List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getLogicSql());
List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL());
List<String> groupByDimensions = getFieldsExceptDate(groupByFields);
parseInfo.setDimensions(
getElements(parseInfo.getModelId(), groupByDimensions, semanticSchema.getDimensions()));
} else {
parseInfo.setNativeQuery(true);
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getLogicSql());
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL());
List<String> selectDimensions = getFieldsExceptDate(selectFields);
parseInfo.setDimensions(
getElements(parseInfo.getModelId(), selectDimensions, semanticSchema.getDimensions()));

View File

@@ -27,7 +27,7 @@ import com.tencent.supersonic.chat.persistence.dataobject.CostType;
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.llm.s2sql.S2SQLQuery;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder;
import com.tencent.supersonic.chat.service.ChatService;
@@ -291,11 +291,11 @@ public class QueryServiceImpl implements QueryService {
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
if (S2SQLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
Map<String, Map<String, String>> havingFiledNameToValueMap = new HashMap<>();
String correctorSql = parseInfo.getSqlInfo().getLogicSql();
String correctorSql = parseInfo.getSqlInfo().getCorrectS2SQL();
log.info("correctorSql before replacing:{}", correctorSql);
// get where filter and having filter
List<FilterExpression> whereExpressionList = SqlParserSelectHelper.getWhereExpressions(correctorSql);
@@ -321,11 +321,11 @@ public class QueryServiceImpl implements QueryService {
correctorSql = SqlParserAddHelper.addHaving(correctorSql, addHavingConditions);
log.info("correctorSql after replacing:{}", correctorSql);
parseInfo.getSqlInfo().setLogicSql(correctorSql);
parseInfo.getSqlInfo().setCorrectS2SQL(correctorSql);
semanticQuery.setParseInfo(parseInfo);
String explainSql = semanticQuery.explain(user);
if (StringUtils.isNotBlank(explainSql)) {
parseInfo.getSqlInfo().setQuerySql(explainSql);
parseInfo.getSqlInfo().setQuerySQL(explainSql);
}
}
semanticQuery.setParseInfo(parseInfo);
@@ -522,7 +522,7 @@ public class QueryServiceImpl implements QueryService {
private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) {
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
if (S2SQLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
return parseInfo;
}
if (CollectionUtils.isNotEmpty(queryData.getDimensions())) {

View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver;
import com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver;
import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder;
@@ -20,7 +20,7 @@ public class ComponentFactory {
private static List<SchemaMapper> schemaMappers = new ArrayList<>();
private static List<SemanticParser> semanticParsers = new ArrayList<>();
private static List<SemanticCorrector> s2QLCorrections = new ArrayList<>();
private static List<SemanticCorrector> s2SQLCorrections = new ArrayList<>();
private static SemanticInterpreter semanticInterpreter;
private static List<ParseResponder> parseResponders = new ArrayList<>();
private static List<ExecuteResponder> executeResponders = new ArrayList<>();
@@ -35,8 +35,8 @@ public class ComponentFactory {
}
public static List<SemanticCorrector> getSqlCorrections() {
return CollectionUtils.isEmpty(s2QLCorrections) ? init(SemanticCorrector.class,
s2QLCorrections) : s2QLCorrections;
return CollectionUtils.isEmpty(s2SQLCorrections) ? init(SemanticCorrector.class,
s2SQLCorrections) : s2SQLCorrections;
}
public static List<ParseResponder> getParseResponders() {

View File

@@ -13,7 +13,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.time.LocalDate;
import java.util.ArrayList;
@@ -124,19 +124,19 @@ public class QueryReqBuilder {
}
/**
* convert to QueryS2QLReq
* convert to QueryS2SQLReq
*
* @param querySql
* @param modelId
* @return
*/
public static QueryS2QLReq buildS2QLReq(String querySql, Long modelId) {
QueryS2QLReq queryS2QLReq = new QueryS2QLReq();
public static QueryS2SQLReq buildS2SQLReq(String querySql, Long modelId) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
if (Objects.nonNull(querySql)) {
queryS2QLReq.setSql(querySql);
queryS2SQLReq.setSql(querySql);
}
queryS2QLReq.setModelId(modelId);
return queryS2QLReq;
queryS2SQLReq.setModelId(modelId);
return queryS2SQLReq;
}
private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.s2ql;
package com.tencent.supersonic.chat.parser.llm.s2sql;
import static org.mockito.Mockito.when;
@@ -14,7 +14,7 @@ import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
class LLMS2QLParserTest {
class LLMS2SQLParserTest {
@Test
void setFilter() {

View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,7 +25,7 @@ import org.mockito.Mockito;
class QueryReqBuilderTest {
@Test
void buildS2QLReq() {
void buildS2SQLReq() {
init();
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setModelId(1L);
@@ -50,17 +50,17 @@ class QueryReqBuilderTest {
orders.add(order);
queryStructReq.setOrders(orders);
QueryS2QLReq queryS2QLReq = queryStructReq.convert(queryStructReq);
QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals(
"SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "GROUP BY department ORDER BY uv LIMIT 2000", queryS2QLReq.getSql());
+ "GROUP BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
queryStructReq.setNativeQuery(true);
queryS2QLReq = queryStructReq.convert(queryStructReq);
queryS2SQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals(
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "ORDER BY uv LIMIT 2000",
queryS2QLReq.getSql());
queryS2SQLReq.getSql());
}

View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
@@ -41,12 +41,12 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@SneakyThrows
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getLogicSql())) {
QueryS2QLReq queryS2QLReq = new QueryS2QLReq();
queryS2QLReq.setSql(queryStructReq.getLogicSql());
queryS2QLReq.setModelId(queryStructReq.getModelId());
queryS2QLReq.setVariables(new HashMap<>());
return queryByS2QL(queryS2QLReq, user);
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
queryS2SQLReq.setModelId(queryStructReq.getModelId());
queryS2SQLReq.setVariables(new HashMap<>());
return queryByS2SQL(queryS2SQLReq, user);
}
queryService = ContextUtils.getBean(QueryService.class);
return queryService.queryByStructWithAuth(queryStructReq, user);
@@ -65,9 +65,9 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@Override
@SneakyThrows
public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) {
public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) {
queryService = ContextUtils.getBean(QueryService.class);
Object object = queryService.queryBySql(queryS2QLReq, user);
Object object = queryService.queryBySql(queryS2SQLReq, user);
return JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp.class);
}

View File

@@ -31,7 +31,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.net.URI;
@@ -70,12 +70,12 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getLogicSql())) {
QueryS2QLReq queryS2QLReq = new QueryS2QLReq();
queryS2QLReq.setSql(queryStructReq.getLogicSql());
queryS2QLReq.setModelId(queryStructReq.getModelId());
queryS2QLReq.setVariables(new HashMap<>());
return queryByS2QL(queryS2QLReq, user);
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
queryS2SQLReq.setModelId(queryStructReq.getModelId());
queryS2SQLReq.setVariables(new HashMap<>());
return queryByS2SQL(queryS2SQLReq, user);
}
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
@@ -93,10 +93,10 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
}
@Override
public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) {
public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
new Gson().toJson(queryS2QLReq));
new Gson().toJson(queryS2SQLReq));
}
public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) {

View File

@@ -7,7 +7,7 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\
com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \
com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \
com.tencent.supersonic.chat.parser.llm.s2sql.LLMS2SQLParser, \
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser
com.tencent.supersonic.chat.api.component.SemanticCorrector=\
@@ -24,8 +24,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver
com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.s2sql.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

View File

@@ -230,7 +230,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.LLM_S2QL);
llmParserTool.setType(AgentToolType.LLM_S2SQL);
llmParserTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(llmParserTool);
@@ -257,7 +257,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.LLM_S2QL);
llmParserTool.setType(AgentToolType.LLM_S2SQL);
llmParserTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(llmParserTool);
@@ -279,7 +279,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.LLM_S2QL);
llmParserTool.setType(AgentToolType.LLM_S2SQL);
llmParserTool.setModelIds(Lists.newArrayList(3L));
agentConfig.getTools().add(llmParserTool);

View File

@@ -7,7 +7,7 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\
com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \
com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \
com.tencent.supersonic.chat.parser.llm.s2sql.LLMS2SQLParser, \
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser
@@ -25,8 +25,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver
com.tencent.supersonic.chat.parser.llm.s2sql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.s2sql.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

View File

@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.query.llm.interpret.LLmAnswerResp;
import com.tencent.supersonic.chat.query.llm.interpret.LLMAnswerResp;
import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.service.QueryService;
import com.tencent.supersonic.util.DataUtils;
@@ -45,7 +45,7 @@ public class MetricInterpretTest {
public void testMetricInterpret() throws Exception {
MockConfiguration.mockAgent(agentService);
MockConfiguration.mockEmbeddingUrl(embeddingConfig);
LLmAnswerResp lLmAnswerResp = new LLmAnswerResp();
LLMAnswerResp lLmAnswerResp = new LLMAnswerResp();
lLmAnswerResp.setAssistantMessage("alice最近在超音数的访问情况有增多");
MockConfiguration.mockPluginManagerDoRequest(pluginManager, "answer_with_plugin_call",
ResponseEntity.ok(JSONObject.toJSONString(lLmAnswerResp)));

View File

@@ -6,7 +6,7 @@ import lombok.ToString;
@Data
@ToString
public class QueryS2QLReq {
public class QueryS2SQLReq {
private Long modelId;

View File

@@ -62,11 +62,11 @@ public class QueryStructReq {
/**
* Later deleted for compatibility only
*/
private String s2QL;
private String s2SQL;
/**
* Later deleted for compatibility only
*/
private String logicSql;
private String correctS2SQL;
public List<String> getGroups() {
if (!CollectionUtils.isEmpty(this.groups)) {
@@ -171,7 +171,7 @@ public class QueryStructReq {
* @param queryStructReq
* @return
*/
public QueryS2QLReq convert(QueryStructReq queryStructReq) {
public QueryS2SQLReq convert(QueryStructReq queryStructReq) {
String sql = null;
try {
sql = buildSql(queryStructReq);
@@ -179,7 +179,7 @@ public class QueryStructReq {
log.error("buildSql error", e);
}
QueryS2QLReq result = new QueryS2QLReq();
QueryS2SQLReq result = new QueryS2SQLReq();
result.setSql(sql);
result.setModelId(queryStructReq.getModelId());
result.setVariables(new HashMap<>());

View File

@@ -20,16 +20,16 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.MaterializationPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SqlParser;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import org.springframework.stereotype.Component;

View File

@@ -11,16 +11,16 @@ import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MetricTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.HashMap;

View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;

View File

@@ -4,15 +4,15 @@ import com.tencent.supersonic.common.util.calcite.SqlParseUtils;
import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
public class Constants {

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import java.util.List;
import lombok.Builder;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.Arrays;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import lombok.AllArgsConstructor;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.ArrayList;
import java.util.List;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.List;
import lombok.Builder;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import lombok.AllArgsConstructor;
import lombok.Builder;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.List;
import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import java.util.ArrayList;
import java.util.HashMap;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
package com.tencent.supersonic.semantic.query.parser.calcite.s2sql;
import lombok.Builder;
import lombok.Data;

View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.S2QLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.S2SQLSqlValidatorImpl;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -35,9 +35,9 @@ public class SchemaBuilder {
Configuration.typeFactory,
Configuration.config
);
S2QLSqlValidatorImpl s2QLSqlValidator = new S2QLSqlValidatorImpl(Configuration.operatorTable, catalogReader,
S2SQLSqlValidatorImpl s2SQLSqlValidator = new S2SQLSqlValidatorImpl(Configuration.operatorTable, catalogReader,
Configuration.typeFactory, Configuration.validatorConfig);
return new ParameterScope(s2QLSqlValidator, nameToTypeMap);
return new ParameterScope(s2SQLSqlValidator, nameToTypeMap);
}
public static CalciteSchema getMaterializationSchema() {

View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.SemanticModel;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

View File

@@ -5,11 +5,11 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.HashSet;
import java.util.List;

View File

@@ -5,9 +5,9 @@ import org.apache.calcite.sql.SqlOperatorTable;
import org.apache.calcite.sql.validate.SqlValidatorCatalogReader;
import org.apache.calcite.sql.validate.SqlValidatorImpl;
public class S2QLSqlValidatorImpl extends SqlValidatorImpl {
public class S2SQLSqlValidatorImpl extends SqlValidatorImpl {
public S2QLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader,
public S2SQLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader,
RelDataTypeFactory typeFactory, Config config) {
super(opTab, catalogReader, typeFactory, config);
}

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.extend.LateralViewExplodeNode;
import java.util.ArrayList;

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify.Type;
import java.util.List;
import java.util.Optional;
import java.util.Set;

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;

View File

@@ -6,9 +6,9 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;

View File

@@ -1,15 +1,15 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;

View File

@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;

View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.ModelService;
@@ -56,7 +56,7 @@ public class QueryReqConverter {
@Autowired
private Catalog catalog;
public QueryStatement convert(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception {
public QueryStatement convert(QueryS2SQLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception {
if (Objects.isNull(modelSchemaResp)) {
return new QueryStatement();
@@ -119,9 +119,9 @@ public class QueryReqConverter {
return queryStatement;
}
private AggOption getAggOption(QueryS2QLReq databaseReq) {
// if there is no group by in S2QL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2QL,set MetricTable's aggOption to "NATIVE"
private AggOption getAggOption(QueryS2SQLReq databaseReq) {
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql();
if (!SqlParserSelectHelper.hasGroupBy(sql)
|| SqlParserSelectFunctionHelper.hasFunction(sql, "count")
@@ -131,7 +131,7 @@ public class QueryReqConverter {
return AggOption.DEFAULT;
}
private void convertNameToBizName(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) {
private void convertNameToBizName(QueryS2SQLReq databaseReq, ModelSchemaResp modelSchemaResp) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
@@ -158,7 +158,7 @@ public class QueryReqConverter {
return metrics;
}
private void functionNameCorrector(QueryS2QLReq databaseReq) {
private void functionNameCorrector(QueryS2SQLReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId());
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
@@ -208,7 +208,7 @@ public class QueryReqConverter {
return elements.stream();
}
public void correctTableName(QueryS2QLReq databaseReq) {
public void correctTableName(QueryS2SQLReq databaseReq) {
String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(),
Constants.TABLE_PREFIX + databaseReq.getModelId());
databaseReq.setSql(sql);

View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
@@ -42,11 +42,11 @@ public class QueryController {
@PostMapping("/sql")
public Object queryBySql(@RequestBody QueryS2QLReq queryS2QLReq,
public Object queryBySql(@RequestBody QueryS2SQLReq queryS2SQLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
Object queryBySql = queryService.queryBySql(queryS2QLReq, user);
Object queryBySql = queryService.queryBySql(queryS2SQLReq, user);
log.info("queryBySql:{},queryBySql");
return queryBySql;
}
@@ -124,9 +124,9 @@ public class QueryController {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
if (QueryTypeEnum.SQL.equals(queryTypeEnum)) {
QueryS2QLReq queryS2QLReq = JsonUtil.toObject(queryReqJson, QueryS2QLReq.class);
ExplainSqlReq<QueryS2QLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2QLReq>builder()
.queryReq(queryS2QLReq)
QueryS2SQLReq queryS2SQLReq = JsonUtil.toObject(queryReqJson, QueryS2SQLReq.class);
ExplainSqlReq<QueryS2SQLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2SQLReq>builder()
.queryReq(queryS2SQLReq)
.queryTypeEnum(queryTypeEnum).build();
return queryService.explain(explainSqlReqNew, user);
}

View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
@@ -18,7 +18,7 @@ import java.util.List;
public interface QueryService {
Object queryBySql(QueryS2QLReq querySqlCmd, User user) throws Exception;
Object queryBySql(QueryS2SQLReq querySqlCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;

View File

@@ -24,11 +24,11 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation;
import com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
@@ -89,9 +89,9 @@ public class QueryServiceImpl implements QueryService {
}
@Override
@S2QLPermissionAnnotation
@S2SQLPermissionAnnotation
@SneakyThrows
public Object queryBySql(QueryS2QLReq querySqlCmd, User user) {
public Object queryBySql(QueryS2SQLReq querySqlCmd, User user) {
statUtils.initStatInfo(querySqlCmd, user);
QueryStatement queryStatement = new QueryStatement();
try {
@@ -109,7 +109,7 @@ public class QueryServiceImpl implements QueryService {
return semanticQueryEngine.execute(queryStatement);
}
private QueryStatement convertToQueryStatement(QueryS2QLReq querySqlCmd, User user) throws Exception {
private QueryStatement convertToQueryStatement(QueryS2SQLReq querySqlCmd, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = new ArrayList<>();
modelIds.add(querySqlCmd.getModelId());
@@ -294,8 +294,8 @@ public class QueryServiceImpl implements QueryService {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
T queryReq = explainSqlReq.getQueryReq();
if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2QLReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryS2QLReq) queryReq, user);
if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2SQLReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryS2SQLReq) queryReq, user);
return getExplainResp(queryStatement);
}
if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {

View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import java.util.ArrayList;
@@ -55,9 +55,9 @@ public class DimValueAspect {
return queryResultWithColumns;
}
Object[] args = joinPoint.getArgs();
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2QLReq.getModelId()));
String sql = queryS2QLReq.getSql();
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelId()));
String sql = queryS2SQLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue.
List<FilterExpression> filterExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
@@ -88,7 +88,7 @@ public class DimValueAspect {
log.info("filedNameToValueMap:{}", filedNameToValueMap);
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql);
queryS2QLReq.setSql(sql);
queryS2SQLReq.setSql(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();

View File

@@ -27,7 +27,7 @@ import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
@@ -199,8 +199,8 @@ public class QueryStructUtils {
return resNameEnSet;
}
public Set<String> getResName(QueryS2QLReq queryS2QLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql())
public Set<String> getResName(QueryS2SQLReq queryS2SQLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql())
.stream().collect(Collectors.toSet());
return resNameSet;
}
@@ -210,11 +210,11 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq, User user) {
Set<String> resNameSet = getResName(queryS2QLReq);
public Set<String> getResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq, User user) {
Set<String> resNameSet = getResName(queryS2SQLReq);
Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(queryS2QLReq.getModelId());
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelId());
filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -245,8 +245,8 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getFilterResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq) {
String sql = queryS2QLReq.getSql();
public Set<String> getFilterResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq) {
String sql = queryS2SQLReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}

View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.model.domain.pojo.MetaFilter;
@@ -42,7 +42,7 @@ import org.springframework.util.CollectionUtils;
@Aspect
@Order(1)
@Slf4j
public class S2QLDataAspect {
public class S2SQLDataAspect {
@Autowired
private QueryStructUtils queryStructUtils;
@@ -55,24 +55,24 @@ public class S2QLDataAspect {
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation)")
private void s2QLPermissionCheck() {
@Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.S2SQLPermissionAnnotation)")
private void s2SQLPermissionCheck() {
}
@Around("s2QLPermissionCheck()")
@Around("s2SQLPermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
log.info("s2ql permission check!");
log.info("s2SQL permission check!");
Object[] objects = joinPoint.getArgs();
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) objects[0];
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) objects[0];
User user = (User) objects[1];
if (!permissionDataEnable) {
log.info("not to check s2QL permission!");
log.info("not to check s2SQL permission!");
return joinPoint.proceed();
}
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
Long modelId = queryS2QLReq.getModelId();
Long modelId = queryS2SQLReq.getModelId();
//1. determine whether admin of the model
if (authCommonService.doModelAdmin(user, modelId)) {
@@ -82,7 +82,7 @@ public class S2QLDataAspect {
// 2. determine whether the subject field is visible
authCommonService.doModelVisible(user, modelId);
// 3. fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2QLReq, user);
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2SQLReq, user);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId);
@@ -97,10 +97,10 @@ public class S2QLDataAspect {
Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId);
// 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryS2QLReq, resAuthSet, sensitiveResReq);
doFilterCheckLogic(queryS2SQLReq, resAuthSet, sensitiveResReq);
// 5.row permission pre-filter
doRowPermission(queryS2QLReq, authorizedResource);
doRowPermission(queryS2SQLReq, authorizedResource);
// 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
@@ -123,7 +123,7 @@ public class S2QLDataAspect {
return queryResultAfterDesensitization;
}
private void doRowPermission(QueryS2QLReq queryS2QLReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(QueryS2SQLReq queryS2SQLReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -145,10 +145,10 @@ public class S2QLDataAspect {
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(queryS2QLReq.getSql(), expression);
log.info("before doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql());
queryS2QLReq.setSql(sql);
log.info("after doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql());
String sql = SqlParserAddHelper.addWhere(queryS2SQLReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
queryS2SQLReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
}
} catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
@@ -156,15 +156,15 @@ public class S2QLDataAspect {
}
private void doFilterCheckLogic(QueryS2QLReq queryS2QLReq, Set<String> resAuthName,
private void doFilterCheckLogic(QueryS2SQLReq queryS2SQLReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2QLReq);
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2SQLReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = new ArrayList<>();
modelIds.add(queryS2QLReq.getModelId());
modelIds.add(queryS2SQLReq.getModelId());
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {

View File

@@ -9,6 +9,6 @@ import java.lang.annotation.Documented;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface S2QLPermissionAnnotation {
public @interface S2SQLPermissionAnnotation {
}

View File

@@ -13,7 +13,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.model.domain.ModelService;
@@ -88,11 +88,11 @@ public class StatUtils {
}
public void initStatInfo(QueryS2QLReq queryS2QLReq, User facadeUser) {
public void initStatInfo(QueryS2SQLReq queryS2SQLReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql());
queryStatInfo.setModelId(queryS2QLReq.getModelId());
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2QLReq.getModelId());
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
queryStatInfo.setModelId(queryS2SQLReq.getModelId());
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelId());
List<String> dimensions = new ArrayList<>();
if (Objects.nonNull(modelSchemaResp)) {
@@ -107,12 +107,12 @@ public class StatUtils {
String userName = getUserName(facadeUser);
try {
queryStatInfo.setTraceId("")
.setModelId(queryS2QLReq.getModelId())
.setModelId(queryS2SQLReq.getModelId())
.setUser(userName)
.setQueryType(QueryTypeEnum.SQL.getValue())
.setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())
.setQuerySqlCmd(queryS2QLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2QLReq.toString()))
.setQuerySqlCmd(queryS2SQLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2SQLReq.toString()))
.setStartTime(System.currentTimeMillis())
.setUseResultCache(true)
.setUseSqlCache(true)