(feature)(project) change dsl to s2ql in project and queryMode to llmParser (#250)

This commit is contained in:
lexluo09
2023-10-18 09:53:01 +08:00
committed by GitHub
parent bf5be11549
commit 8d81f63e08
77 changed files with 320 additions and 305 deletions

View File

@@ -14,7 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
@@ -37,7 +37,7 @@ public interface SemanticInterpreter {
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user); QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user);
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);

View File

@@ -5,7 +5,7 @@ import lombok.Data;
@Data @Data
public class SqlInfo { public class SqlInfo {
private String llmParseSql; private String s2QL;
private String logicSql; private String logicSql;
private String querySql; private String querySql;
} }

View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.agent.tool;
public enum AgentToolType { public enum AgentToolType {
RULE, RULE,
DSL, LLM_S2QL,
PLUGIN, PLUGIN,
INTERPRET INTERPRET
} }

View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.chat.agent.tool;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class CommonAgentTool extends AgentTool {
protected List<Long> modelIds;
}

View File

@@ -5,9 +5,7 @@ import lombok.Data;
import java.util.List; import java.util.List;
@Data @Data
public class DslTool extends AgentTool { public class LLMParserTool extends CommonAgentTool {
private List<Long> modelIds;
private List<String> exampleQuestions; private List<String> exampleQuestions;

View File

@@ -7,9 +7,8 @@ import org.apache.commons.collections.CollectionUtils;
import java.util.List; import java.util.List;
@Data @Data
public class RuleQueryTool extends AgentTool { public class RuleQueryTool extends CommonAgentTool {
private List<Long> modelIds;
private List<String> queryModes; private List<String> queryModes;

View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.corrector; package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq; import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue; import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -67,11 +67,11 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
return null; return null;
} }
DSLParseResult dslParseResult = JsonUtil.toObject(JsonUtil.toString(context), DSLParseResult.class); ParseResult parseResult = JsonUtil.toObject(JsonUtil.toString(context), ParseResult.class);
if (Objects.isNull(dslParseResult) || Objects.isNull(dslParseResult.getLlmReq())) { if (Objects.isNull(parseResult) || Objects.isNull(parseResult.getLlmReq())) {
return null; return null;
} }
LLMReq llmReq = dslParseResult.getLlmReq(); LLMReq llmReq = parseResult.getLlmReq();
return llmReq.getLinking(); return llmReq.getLinking();
} }

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo; import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLDateHelper; import com.tencent.supersonic.chat.parser.llm.s2ql.S2QLDateHelper;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.DateUtils;
@@ -72,7 +72,7 @@ public class WhereCorrector extends BaseSemanticCorrector {
String sql = semanticCorrectInfo.getSql(); String sql = semanticCorrectInfo.getSql();
List<String> whereFields = SqlParserSelectHelper.getWhereFields(sql); List<String> whereFields = SqlParserSelectHelper.getWhereFields(sql);
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(DateUtils.DATE_FIELD)) { if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(DateUtils.DATE_FIELD)) {
String currentDate = DSLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId()); String currentDate = S2QLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId());
if (StringUtils.isNotBlank(currentDate)) { if (StringUtils.isNotBlank(currentDate)) {
sql = SqlParserAddHelper.addParenthesisToWhere(sql); sql = SqlParserAddHelper.addParenthesisToWhere(sql);
sql = SqlParserAddHelper.addWhere(sql, DateUtils.DATE_FIELD, currentDate); sql = SqlParserAddHelper.addWhere(sql, DateUtils.DATE_FIELD, currentDate);

View File

@@ -88,7 +88,7 @@ public class MapperHelper {
AgentService agentService = ContextUtils.getBean(AgentService.class); AgentService agentService = ContextUtils.getBean(AgentService.class);
Set<Long> detectModelIds = agentService.getDslToolsModelIds(request.getAgentId(), null); Set<Long> detectModelIds = agentService.getModelIds(request.getAgentId(), null);
//contains all //contains all
if (agentService.containsAllModel(detectModelIds)) { if (agentService.containsAllModel(detectModelIds)) {
if (Objects.nonNull(modelId) && modelId > 0) { if (Objects.nonNull(modelId) && modelId > 0) {

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.OptimizationConfig; import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -20,7 +20,7 @@ public class SatisfactionChecker {
// check all the parse info in candidate // check all the parse info in candidate
public static boolean check(QueryContext queryContext) { public static boolean check(QueryContext queryContext) {
for (SemanticQuery query : queryContext.getCandidateQueries()) { for (SemanticQuery query : queryContext.getCandidateQueries()) {
if (query.getQueryMode().equals(DslQuery.QUERY_MODE)) { if (query.getQueryMode().equals(S2QLQuery.QUERY_MODE)) {
continue; continue;
} }
if (checkThreshold(queryContext.getRequest().getQueryText(), query.getParseInfo())) { if (checkThreshold(queryContext.getRequest().getQueryText(), query.getParseInfo())) {

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl; package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.parser.llm.dsl; package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.agent.tool.AgentToolType; import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.DslTool; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.component.SemanticCorrector; import com.tencent.supersonic.chat.api.component.SemanticCorrector;
import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -18,10 +18,10 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.config.LLMParserConfig; import com.tencent.supersonic.chat.config.LLMParserConfig;
import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq; import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue; import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import com.tencent.supersonic.chat.query.llm.dsl.LLMResp; import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.ComponentFactory;
@@ -59,18 +59,18 @@ import org.springframework.util.CollectionUtils;
import org.springframework.web.client.RestTemplate; import org.springframework.web.client.RestTemplate;
@Slf4j @Slf4j
public class LLMDslParser implements SemanticParser { public class LLMS2QLParser implements SemanticParser {
@Override @Override
public void parse(QueryContext queryCtx, ChatContext chatCtx) { public void parse(QueryContext queryCtx, ChatContext chatCtx) {
QueryReq request = queryCtx.getRequest(); QueryReq request = queryCtx.getRequest();
LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class); LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class);
if (StringUtils.isEmpty(llmParserConfig.getUrl())) { if (StringUtils.isEmpty(llmParserConfig.getUrl())) {
log.info("llm parser url is empty, skip dsl parser, llmParserConfig:{}", llmParserConfig); log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMS2QLParser.class, llmParserConfig);
return; return;
} }
if (SatisfactionChecker.check(queryCtx)) { if (SatisfactionChecker.check(queryCtx)) {
log.info("skip dsl parser, queryText:{}", request.getQueryText()); log.info("skip {}, queryText:{}", LLMS2QLParser.class, request.getQueryText());
return; return;
} }
try { try {
@@ -79,9 +79,9 @@ public class LLMDslParser implements SemanticParser {
return; return;
} }
DslTool dslTool = getDslTool(request, modelId); CommonAgentTool commonAgentTool = getParserTool(request, modelId);
if (Objects.isNull(dslTool)) { if (Objects.isNull(commonAgentTool)) {
log.info("no dsl tool in this agent, skip dsl parser"); log.info("no tool in this agent, skip {}", LLMS2QLParser.class);
return; return;
} }
@@ -91,10 +91,10 @@ public class LLMDslParser implements SemanticParser {
if (Objects.isNull(llmResp)) { if (Objects.isNull(llmResp)) {
return; return;
} }
DSLParseResult dslParseResult = DSLParseResult.builder().request(request) ParseResult parseResult = ParseResult.builder().request(request)
.dslTool(dslTool).llmReq(llmReq).llmResp(llmResp).build(); .commonAgentTool(commonAgentTool).llmReq(llmReq).llmResp(llmResp).build();
SemanticParseInfo parseInfo = getParseInfo(queryCtx, modelId, dslTool, dslParseResult); SemanticParseInfo parseInfo = getParseInfo(queryCtx, modelId, commonAgentTool, parseResult);
SemanticCorrectInfo semanticCorrectInfo = getCorrectorSql(queryCtx, parseInfo, llmResp.getSqlOutput()); SemanticCorrectInfo semanticCorrectInfo = getCorrectorSql(queryCtx, parseInfo, llmResp.getSqlOutput());
@@ -103,7 +103,7 @@ public class LLMDslParser implements SemanticParser {
updateParseInfo(semanticCorrectInfo, modelId, parseInfo); updateParseInfo(semanticCorrectInfo, modelId, parseInfo);
} catch (Exception e) { } catch (Exception e) {
log.error("LLMDSLParser error", e); log.error("LLMS2QLParser error", e);
} }
} }
@@ -243,12 +243,12 @@ public class LLMDslParser implements SemanticParser {
.queryFilters(queryCtx.getRequest().getQueryFilters()).sql(sql) .queryFilters(queryCtx.getRequest().getQueryFilters()).sql(sql)
.parseInfo(parseInfo).build(); .parseInfo(parseInfo).build();
List<SemanticCorrector> dslCorrections = ComponentFactory.getSqlCorrections(); List<SemanticCorrector> corrections = ComponentFactory.getSqlCorrections();
dslCorrections.forEach(dslCorrection -> { corrections.forEach(correction -> {
try { try {
dslCorrection.correct(correctInfo); correction.correct(correctInfo);
log.info("sqlCorrection:{} sql:{}", dslCorrection.getClass().getSimpleName(), correctInfo.getSql()); log.info("sqlCorrection:{} sql:{}", correction.getClass().getSimpleName(), correctInfo.getSql());
} catch (Exception e) { } catch (Exception e) {
log.error(String.format("correct error,correctInfo:%s", correctInfo), e); log.error(String.format("correct error,correctInfo:%s", correctInfo), e);
} }
@@ -256,21 +256,21 @@ public class LLMDslParser implements SemanticParser {
return correctInfo; return correctInfo;
} }
private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, DslTool dslTool, private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, CommonAgentTool commonAgentTool,
DSLParseResult dslParseResult) { ParseResult parseResult) {
PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(DslQuery.QUERY_MODE); PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(S2QLQuery.QUERY_MODE);
SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(modelId)); parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(modelId));
Map<String, Object> properties = new HashMap<>(); Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, dslParseResult); properties.put(Constants.CONTEXT, parseResult);
properties.put("type", "internal"); properties.put("type", "internal");
properties.put("name", dslTool.getName()); properties.put("name", commonAgentTool.getName());
parseInfo.setProperties(properties); parseInfo.setProperties(properties);
parseInfo.setScore(queryCtx.getRequest().getQueryText().length()); parseInfo.setScore(queryCtx.getRequest().getQueryText().length());
parseInfo.setQueryMode(semanticQuery.getQueryMode()); parseInfo.setQueryMode(semanticQuery.getQueryMode());
parseInfo.getSqlInfo().setLlmParseSql(dslParseResult.getLlmResp().getSqlOutput()); parseInfo.getSqlInfo().setS2QL(parseResult.getLlmResp().getSqlOutput());
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema(); SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
Map<Long, String> modelIdToName = semanticSchema.getModelIdToName(); Map<Long, String> modelIdToName = semanticSchema.getModelIdToName();
@@ -284,10 +284,11 @@ public class LLMDslParser implements SemanticParser {
return parseInfo; return parseInfo;
} }
private DslTool getDslTool(QueryReq request, Long modelId) { private CommonAgentTool getParserTool(QueryReq request, Long modelId) {
AgentService agentService = ContextUtils.getBean(AgentService.class); AgentService agentService = ContextUtils.getBean(AgentService.class);
List<DslTool> dslTools = agentService.getDslTools(request.getAgentId(), AgentToolType.DSL); List<CommonAgentTool> commonAgentTools = agentService.getParserTools(request.getAgentId(),
Optional<DslTool> dslToolOptional = dslTools.stream() AgentToolType.LLM_S2QL);
Optional<CommonAgentTool> llmParserTool = commonAgentTools.stream()
.filter(tool -> { .filter(tool -> {
List<Long> modelIds = tool.getModelIds(); List<Long> modelIds = tool.getModelIds();
if (agentService.containsAllModel(new HashSet<>(modelIds))) { if (agentService.containsAllModel(new HashSet<>(modelIds))) {
@@ -296,18 +297,18 @@ public class LLMDslParser implements SemanticParser {
return modelIds.contains(modelId); return modelIds.contains(modelId);
}) })
.findFirst(); .findFirst();
return dslToolOptional.orElse(null); return llmParserTool.orElse(null);
} }
private Long getModelId(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) { private Long getModelId(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) {
AgentService agentService = ContextUtils.getBean(AgentService.class); AgentService agentService = ContextUtils.getBean(AgentService.class);
Set<Long> distinctModelIds = agentService.getDslToolsModelIds(agentId, AgentToolType.DSL); Set<Long> distinctModelIds = agentService.getModelIds(agentId, AgentToolType.LLM_S2QL);
if (agentService.containsAllModel(distinctModelIds)) { if (agentService.containsAllModel(distinctModelIds)) {
distinctModelIds = new HashSet<>(); distinctModelIds = new HashSet<>();
} }
ModelResolver modelResolver = ComponentFactory.getModelResolver(); ModelResolver modelResolver = ComponentFactory.getModelResolver();
Long modelId = modelResolver.resolve(queryCtx, chatCtx, distinctModelIds); Long modelId = modelResolver.resolve(queryCtx, chatCtx, distinctModelIds);
log.info("resolve modelId:{},dslModels:{}", modelId, distinctModelIds); log.info("resolve modelId:{},llmParser Models:{}", modelId, distinctModelIds);
return modelId; return modelId;
} }
@@ -354,7 +355,7 @@ public class LLMDslParser implements SemanticParser {
linking.addAll(getValueList(queryCtx, modelId, semanticSchema)); linking.addAll(getValueList(queryCtx, modelId, semanticSchema));
llmReq.setLinking(linking); llmReq.setLinking(linking);
String currentDate = DSLDateHelper.getReferenceDate(modelId); String currentDate = S2QLDateHelper.getReferenceDate(modelId);
llmReq.setCurrentDate(currentDate); llmReq.setCurrentDate(currentDate);
return llmReq; return llmReq;
} }

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl; package com.tencent.supersonic.chat.parser.llm.s2ql;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl; package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;

View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.parser.llm.dsl; package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.tencent.supersonic.chat.agent.tool.DslTool; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq; import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMResp; import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
@@ -13,7 +13,7 @@ import lombok.NoArgsConstructor;
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@NoArgsConstructor @NoArgsConstructor
public class DSLParseResult { public class ParseResult {
private LLMReq llmReq; private LLMReq llmReq;
@@ -21,5 +21,5 @@ public class DSLParseResult {
private QueryReq request; private QueryReq request;
private DslTool dslTool; private CommonAgentTool commonAgentTool;
} }

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl; package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
@@ -11,7 +11,7 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
public class DSLDateHelper { public class S2QLDateHelper {
public static String getReferenceDate(Long modelId) { public static String getReferenceDate(Long modelId) {
String defaultDate = DateUtils.getBeforeDate(0); String defaultDate = DateUtils.getBeforeDate(0);

View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseConfig; import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.plugin.PluginRecallResult; import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.service.PluginService; import com.tencent.supersonic.chat.service.PluginService;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.net.URI; import java.net.URI;
@@ -102,7 +102,7 @@ public class FunctionBasedParser extends PluginParser {
log.info("user decide Model:{}", modelId); log.info("user decide Model:{}", modelId);
List<Plugin> plugins = getPluginList(queryContext); List<Plugin> plugins = getPluginList(queryContext);
List<PluginParseConfig> functionDOList = plugins.stream().filter(plugin -> { List<PluginParseConfig> functionDOList = plugins.stream().filter(plugin -> {
if (DslQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) { if (S2QLQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
return false; return false;
} }
if (plugin.getParseModeConfig() == null) { if (plugin.getParseModeConfig() == null) {

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.dsl; package com.tencent.supersonic.chat.query.llm.s2ql;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.dsl; package com.tencent.supersonic.chat.query.llm.s2ql;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;

View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.chat.query.llm.dsl; package com.tencent.supersonic.chat.query.llm.s2ql;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.ComponentFactory;
@@ -16,7 +16,7 @@ import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -26,12 +26,12 @@ import org.springframework.stereotype.Component;
@Slf4j @Slf4j
@Component @Component
public class DslQuery extends PluginSemanticQuery { public class S2QLQuery extends PluginSemanticQuery {
public static final String QUERY_MODE = "DSL"; public static final String QUERY_MODE = "LLM_S2QL";
protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
public DslQuery() { public S2QLQuery() {
QueryManager.register(this); QueryManager.register(this);
} }
@@ -45,10 +45,10 @@ public class DslQuery extends PluginSemanticQuery {
LLMResp llmResp = getLlmResp(); LLMResp llmResp = getLlmResp();
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
QueryDslReq queryDslReq = getQueryDslReq(llmResp); QueryS2QLReq queryS2QLReq = getQueryS2QLReq(llmResp);
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByDsl(queryDslReq, user); QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2QL(queryS2QLReq, user);
log.info("queryByDsl cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput()); log.info("queryByS2QL cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput());
QueryResult queryResult = new QueryResult(); QueryResult queryResult = new QueryResult();
if (Objects.nonNull(queryResp)) { if (Objects.nonNull(queryResp)) {
@@ -69,13 +69,12 @@ public class DslQuery extends PluginSemanticQuery {
private LLMResp getLlmResp() { private LLMResp getLlmResp() {
String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)); String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class); ParseResult parseResult = JsonUtil.toObject(json, ParseResult.class);
return dslParseResult.getLlmResp(); return parseResult.getLlmResp();
} }
private QueryDslReq getQueryDslReq(LLMResp llmResp) { private QueryS2QLReq getQueryS2QLReq(LLMResp llmResp) {
QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId()); return QueryReqBuilder.buildS2QLReq(llmResp.getCorrectorSql(), parseInfo.getModelId());
return queryDslReq;
} }
@Override @Override
@@ -84,7 +83,7 @@ public class DslQuery extends PluginSemanticQuery {
try { try {
explainSqlReq = ExplainSqlReq.builder() explainSqlReq = ExplainSqlReq.builder()
.queryTypeEnum(QueryTypeEnum.SQL) .queryTypeEnum(QueryTypeEnum.SQL)
.queryReq(getQueryDslReq(getLlmResp())) .queryReq(getQueryS2QLReq(getLlmResp()))
.build(); .build();
return semanticInterpreter.explain(explainSqlReq, user); return semanticInterpreter.explain(explainSqlReq, user);
} catch (Exception e) { } catch (Exception e) {

View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List; import java.util.List;
@@ -24,7 +24,7 @@ public class EntityInfoExecuteResponder implements ExecuteResponder {
return; return;
} }
String queryMode = semanticParseInfo.getQueryMode(); String queryMode = semanticParseInfo.getQueryMode();
if (QueryManager.isPluginQuery(queryMode) && !DslQuery.QUERY_MODE.equals(queryMode)) { if (QueryManager.isPluginQuery(queryMode) && !S2QLQuery.QUERY_MODE.equals(queryMode)) {
return; return;
} }
SemanticService semanticService = ContextUtils.getBean(SemanticService.class); SemanticService semanticService = ContextUtils.getBean(SemanticService.class);

View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List; import java.util.List;
@@ -24,7 +24,7 @@ public class EntityInfoParseResponder implements ParseResponder {
QueryReq queryReq = queryContext.getRequest(); QueryReq queryReq = queryContext.getRequest();
selectedParses.forEach(parseInfo -> { selectedParses.forEach(parseInfo -> {
if (QueryManager.isPluginQuery(parseInfo.getQueryMode()) if (QueryManager.isPluginQuery(parseInfo.getQueryMode())
&& !DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { && !S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
return; return;
} }
//1. set entity info //1. set entity info

View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.agent.Agent;
import com.tencent.supersonic.chat.agent.tool.AgentToolType; import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.DslTool; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@@ -19,9 +19,9 @@ public interface AgentService {
void deleteAgent(Integer id); void deleteAgent(Integer id);
List<DslTool> getDslTools(Integer agentId, AgentToolType agentToolType); List<CommonAgentTool> getParserTools(Integer agentId, AgentToolType agentToolType);
Set<Long> getDslToolsModelIds(Integer agentId, AgentToolType agentToolType); Set<Long> getModelIds(Integer agentId, AgentToolType agentToolType);
boolean containsAllModel(Set<Long> detectModelIds); boolean containsAllModel(Set<Long> detectModelIds);
} }

View File

@@ -5,7 +5,7 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.agent.Agent;
import com.tencent.supersonic.chat.agent.tool.AgentToolType; import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.DslTool; import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.persistence.dataobject.AgentDO; import com.tencent.supersonic.chat.persistence.dataobject.AgentDO;
import com.tencent.supersonic.chat.persistence.repository.AgentRepository; import com.tencent.supersonic.chat.persistence.repository.AgentRepository;
import com.tencent.supersonic.chat.service.AgentService; import com.tencent.supersonic.chat.service.AgentService;
@@ -87,7 +87,7 @@ public class AgentServiceImpl implements AgentService {
return agentDO; return agentDO;
} }
public List<DslTool> getDslTools(Integer agentId, AgentToolType agentToolType) { public List<CommonAgentTool> getParserTools(Integer agentId, AgentToolType agentToolType) {
Agent agent = getAgent(agentId); Agent agent = getAgent(agentId);
if (agent == null) { if (agent == null) {
return Lists.newArrayList(); return Lists.newArrayList();
@@ -96,15 +96,16 @@ public class AgentServiceImpl implements AgentService {
if (CollectionUtils.isEmpty(tools)) { if (CollectionUtils.isEmpty(tools)) {
return Lists.newArrayList(); return Lists.newArrayList();
} }
return tools.stream().map(tool -> JSONObject.parseObject(tool, DslTool.class)).collect(Collectors.toList()); return tools.stream().map(tool -> JSONObject.parseObject(tool, CommonAgentTool.class))
.collect(Collectors.toList());
} }
public Set<Long> getDslToolsModelIds(Integer agentId, AgentToolType agentToolType) { public Set<Long> getModelIds(Integer agentId, AgentToolType agentToolType) {
List<DslTool> dslTools = getDslTools(agentId, agentToolType); List<CommonAgentTool> commonAgentTools = getParserTools(agentId, agentToolType);
if (CollectionUtils.isEmpty(dslTools)) { if (CollectionUtils.isEmpty(commonAgentTools)) {
return new HashSet<>(); return new HashSet<>();
} }
return dslTools.stream().map(DslTool::getModelIds) return commonAgentTools.stream().map(CommonAgentTool::getModelIds)
.filter(modelIds -> !CollectionUtils.isEmpty(modelIds)) .filter(modelIds -> !CollectionUtils.isEmpty(modelIds))
.flatMap(Collection::stream) .flatMap(Collection::stream)
.collect(Collectors.toSet()); .collect(Collectors.toSet());

View File

@@ -19,15 +19,15 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.persistence.dataobject.CostType; import com.tencent.supersonic.chat.persistence.dataobject.CostType;
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO; import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.QuerySelector; import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.llm.dsl.LLMResp; import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder;
import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.service.ChatService;
@@ -305,12 +305,12 @@ public class QueryServiceImpl implements QueryService {
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
if (DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>(); Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
Map<String, Map<String, String>> havingFiledNameToValueMap = new HashMap<>(); Map<String, Map<String, String>> havingFiledNameToValueMap = new HashMap<>();
String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)); String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class); ParseResult parseResult = JsonUtil.toObject(json, ParseResult.class);
LLMResp llmResp = dslParseResult.getLlmResp(); LLMResp llmResp = parseResult.getLlmResp();
String correctorSql = llmResp.getCorrectorSql(); String correctorSql = llmResp.getCorrectorSql();
log.info("correctorSql before replacing:{}", correctorSql); log.info("correctorSql before replacing:{}", correctorSql);
@@ -330,9 +330,9 @@ public class QueryServiceImpl implements QueryService {
correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap); correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap);
log.info("correctorSql after replacing:{}", correctorSql); log.info("correctorSql after replacing:{}", correctorSql);
llmResp.setCorrectorSql(correctorSql); llmResp.setCorrectorSql(correctorSql);
dslParseResult.setLlmResp(llmResp); parseResult.setLlmResp(llmResp);
Map<String, Object> properties = new HashMap<>(); Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, dslParseResult); properties.put(Constants.CONTEXT, parseResult);
parseInfo.setProperties(properties); parseInfo.setProperties(properties);
parseInfo.getSqlInfo().setLogicSql(correctorSql); parseInfo.getSqlInfo().setLogicSql(correctorSql);
semanticQuery.setParseInfo(parseInfo); semanticQuery.setParseInfo(parseInfo);
@@ -399,29 +399,29 @@ public class QueryServiceImpl implements QueryService {
if (CollectionUtils.isEmpty(metricFilters)) { if (CollectionUtils.isEmpty(metricFilters)) {
return; return;
} }
for (QueryFilter dslQueryFilter : metricFilters) { for (QueryFilter queryFilter : metricFilters) {
Map<String, String> map = new HashMap<>(); Map<String, String> map = new HashMap<>();
for (FilterExpression filterExpression : filterExpressionList) { for (FilterExpression filterExpression : filterExpressionList) {
if (filterExpression.getFieldName() != null if (filterExpression.getFieldName() != null
&& filterExpression.getFieldName().contains(dslQueryFilter.getName()) && filterExpression.getFieldName().contains(queryFilter.getName())
&& dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) { && queryFilter.getOperator().getValue().equals(filterExpression.getOperator())) {
map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString()); map.put(filterExpression.getFieldValue().toString(), queryFilter.getValue().toString());
contextMetricFilters.stream().forEach(o -> { contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) { if (o.getName().equals(queryFilter.getName())) {
o.setValue(dslQueryFilter.getValue()); o.setValue(queryFilter.getValue());
} }
}); });
break; break;
} }
} }
filedNameToValueMap.put(dslQueryFilter.getName(), map); filedNameToValueMap.put(queryFilter.getName(), map);
} }
} }
private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) { private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) {
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
if (DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
return parseInfo; return parseInfo;
} }
if (CollectionUtils.isNotEmpty(queryData.getDimensions())) { if (CollectionUtils.isNotEmpty(queryData.getDimensions())) {

View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver; import com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver;
import com.tencent.supersonic.chat.query.QuerySelector; import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder;
@@ -20,7 +20,7 @@ public class ComponentFactory {
private static List<SchemaMapper> schemaMappers = new ArrayList<>(); private static List<SchemaMapper> schemaMappers = new ArrayList<>();
private static List<SemanticParser> semanticParsers = new ArrayList<>(); private static List<SemanticParser> semanticParsers = new ArrayList<>();
private static List<SemanticCorrector> dslCorrections = new ArrayList<>(); private static List<SemanticCorrector> s2QLCorrections = new ArrayList<>();
private static SemanticInterpreter semanticInterpreter; private static SemanticInterpreter semanticInterpreter;
private static List<ParseResponder> parseResponders = new ArrayList<>(); private static List<ParseResponder> parseResponders = new ArrayList<>();
private static List<ExecuteResponder> executeResponders = new ArrayList<>(); private static List<ExecuteResponder> executeResponders = new ArrayList<>();
@@ -35,7 +35,8 @@ public class ComponentFactory {
} }
public static List<SemanticCorrector> getSqlCorrections() { public static List<SemanticCorrector> getSqlCorrections() {
return CollectionUtils.isEmpty(dslCorrections) ? init(SemanticCorrector.class, dslCorrections) : dslCorrections; return CollectionUtils.isEmpty(s2QLCorrections) ? init(SemanticCorrector.class,
s2QLCorrections) : s2QLCorrections;
} }
public static List<ParseResponder> getParseResponders() { public static List<ParseResponder> getParseResponders() {

View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum; import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.time.LocalDate; import java.time.LocalDate;
@@ -123,19 +123,19 @@ public class QueryReqBuilder {
} }
/** /**
* convert to QueryDslReq * convert to QueryS2QLReq
* *
* @param querySql * @param querySql
* @param modelId * @param modelId
* @return * @return
*/ */
public static QueryDslReq buildDslReq(String querySql, Long modelId) { public static QueryS2QLReq buildS2QLReq(String querySql, Long modelId) {
QueryDslReq queryDslReq = new QueryDslReq(); QueryS2QLReq queryS2QLReq = new QueryS2QLReq();
if (Objects.nonNull(querySql)) { if (Objects.nonNull(querySql)) {
queryDslReq.setSql(querySql); queryS2QLReq.setSql(querySql);
} }
queryDslReq.setModelId(modelId); queryS2QLReq.setModelId(modelId);
return queryDslReq; return queryS2QLReq;
} }

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl; package com.tencent.supersonic.chat.parser.llm.s2ql;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@@ -16,7 +16,7 @@ import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic; import org.mockito.MockedStatic;
import org.mockito.Mockito; import org.mockito.Mockito;
class LLMDslParserTest { class LLMS2QLParserTest {
@Test @Test
void setFilter() { void setFilter() {
@@ -72,9 +72,9 @@ class LLMDslParserTest {
.parseInfo(parseInfo) .parseInfo(parseInfo)
.build(); .build();
LLMDslParser llmDslParser = new LLMDslParser(); LLMS2QLParser llms2QLParser = new LLMS2QLParser();
llmDslParser.updateParseInfo(semanticCorrectInfo, 2L, parseInfo); llms2QLParser.updateParseInfo(semanticCorrectInfo, 2L, parseInfo);
} }
} }

View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DimensionService;
@@ -56,9 +56,9 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@Override @Override
@SneakyThrows @SneakyThrows
public QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user) { public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) {
queryService = ContextUtils.getBean(QueryService.class); queryService = ContextUtils.getBean(QueryService.class);
Object object = queryService.queryBySql(queryDslReq, user); Object object = queryService.queryBySql(queryS2QLReq, user);
QueryResultWithSchemaResp queryResultWithSchemaResp = JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp queryResultWithSchemaResp = JsonUtil.toObject(JsonUtil.toString(object),
QueryResultWithSchemaResp.class); QueryResultWithSchemaResp.class);
return queryResultWithSchemaResp; return queryResultWithSchemaResp;

View File

@@ -31,7 +31,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.net.URI; import java.net.URI;
@@ -83,10 +83,10 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
} }
@Override @Override
public QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user) { public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(), return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
new Gson().toJson(queryDslReq)); new Gson().toJson(queryS2QLReq));
} }
public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) { public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) {

View File

@@ -11,7 +11,7 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \ com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \
com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \ com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \
com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \ com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \
com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser, \ com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser
@@ -21,8 +21,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver=\ com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.dsl.HeuristicModelResolver com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.agent.Agent; import com.tencent.supersonic.chat.agent.Agent;
import com.tencent.supersonic.chat.agent.AgentConfig; import com.tencent.supersonic.chat.agent.AgentConfig;
import com.tencent.supersonic.chat.agent.tool.AgentToolType; import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.DslTool; import com.tencent.supersonic.chat.agent.tool.LLMParserTool;
import com.tencent.supersonic.chat.agent.tool.RuleQueryTool; import com.tencent.supersonic.chat.agent.tool.RuleQueryTool;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
@@ -228,11 +228,11 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
)); ));
agentConfig.getTools().add(ruleQueryTool); agentConfig.getTools().add(ruleQueryTool);
DslTool dslTool = new DslTool(); LLMParserTool llmParserTool = new LLMParserTool();
dslTool.setId("1"); llmParserTool.setId("1");
dslTool.setType(AgentToolType.DSL); llmParserTool.setType(AgentToolType.LLM_S2QL);
dslTool.setModelIds(Lists.newArrayList(-1L)); llmParserTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(dslTool); agentConfig.getTools().add(llmParserTool);
agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
agentService.createAgent(agent, User.getFakeUser()); agentService.createAgent(agent, User.getFakeUser());
@@ -255,11 +255,11 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
"ENTITY_DETAIL", "ENTITY_LIST_FILTER", "ENTITY_ID")); "ENTITY_DETAIL", "ENTITY_LIST_FILTER", "ENTITY_ID"));
agentConfig.getTools().add(ruleQueryTool); agentConfig.getTools().add(ruleQueryTool);
DslTool dslTool = new DslTool(); LLMParserTool llmParserTool = new LLMParserTool();
dslTool.setId("1"); llmParserTool.setId("1");
dslTool.setType(AgentToolType.DSL); llmParserTool.setType(AgentToolType.LLM_S2QL);
dslTool.setModelIds(Lists.newArrayList(-1L)); llmParserTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(dslTool); agentConfig.getTools().add(llmParserTool);
agent.setAgentConfig(JSONObject.toJSONString(agentConfig)); agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
agentService.createAgent(agent, User.getFakeUser()); agentService.createAgent(agent, User.getFakeUser());

View File

@@ -11,7 +11,7 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \ com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \
com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \ com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \
com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \ com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \
com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser, \ com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \ com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser
@@ -22,8 +22,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver=\ com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.dsl.HeuristicModelResolver com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

View File

@@ -66,3 +66,5 @@ alter table s2_dimension add column data_type varchar(50) not null DEFAULT 'va
alter table s2_query_stat_info add column `query_opt_mode` varchar(20) DEFAULT NULL COMMENT '优化模式'; alter table s2_query_stat_info add column `query_opt_mode` varchar(20) DEFAULT NULL COMMENT '优化模式';
alter table s2_datasource add column depends text COMMENT '上游依赖标识' after datasource_detail; alter table s2_datasource add column depends text COMMENT '上游依赖标识' after datasource_detail;
--20231018
UPDATE `s2_agent` SET `config` = replace (`config`,'DSL','LLM_S2QL') WHERE `config` LIKE '%DSL%';

View File

@@ -64,7 +64,7 @@
<xk.time.version>3.2.4</xk.time.version> <xk.time.version>3.2.4</xk.time.version>
<mockito-inline.version>4.5.1</mockito-inline.version> <mockito-inline.version>4.5.1</mockito-inline.version>
<jsqlparser.version>4.5</jsqlparser.version> <jsqlparser.version>4.5</jsqlparser.version>
<revision>0.7.5-SNAPSHOT</revision> <revision>0.7.6-SNAPSHOT</revision>
<!-- Do not bump spotless plugin version since 2.30.0 is the latest version supports Java 8--> <!-- Do not bump spotless plugin version since 2.30.0 is the latest version supports Java 8-->
<maven.plugin.spotless.version>2.30.0</maven.plugin.spotless.version> <maven.plugin.spotless.version>2.30.0</maven.plugin.spotless.version>
<spotless.python.includes></spotless.python.includes> <spotless.python.includes></spotless.python.includes>

View File

@@ -6,7 +6,7 @@ import lombok.ToString;
@Data @Data
@ToString @ToString
public class QueryDslReq { public class QueryS2QLReq {
private Long modelId; private Long modelId;

View File

@@ -18,16 +18,16 @@ import com.tencent.supersonic.semantic.materialization.domain.MaterializationCon
import com.tencent.supersonic.semantic.materialization.domain.MaterializationRecordService; import com.tencent.supersonic.semantic.materialization.domain.MaterializationRecordService;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager; import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.MaterializationPlanner; import com.tencent.supersonic.semantic.query.parser.calcite.planner.MaterializationPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils; import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SqlParser; import com.tencent.supersonic.semantic.query.parser.SqlParser;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner; import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;

View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.semantic.query.parser.calcite;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@@ -33,7 +32,6 @@ import org.apache.calcite.sql2rel.SqlToRelConverter;
public class Configuration { public class Configuration {
public static Properties configProperties = new Properties(); public static Properties configProperties = new Properties();
public static DSLSqlValidatorImpl dslSqlValidator;
public static RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); public static RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
public static SqlOperatorTable operatorTable = SqlStdOperatorTable.instance(); public static SqlOperatorTable operatorTable = SqlStdOperatorTable.instance();
public static CalciteConnectionConfig config = new CalciteConnectionConfigImpl(configProperties); public static CalciteConnectionConfig config = new CalciteConnectionConfigImpl(configProperties);
@@ -71,7 +69,7 @@ public class Configuration {
public static SqlValidator getSqlValidator(CalciteSchema rootSchema) { public static SqlValidator getSqlValidator(CalciteSchema rootSchema) {
List<SqlOperatorTable> tables = new ArrayList<>(); List<SqlOperatorTable> tables = new ArrayList<>();
tables.add(SqlStdOperatorTable.instance()); tables.add(SqlStdOperatorTable.instance());
SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables); //.of(SqlStdOperatorTable.instance()); SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables);
//operatorTable. //operatorTable.
SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT
.withLenientOperatorLookup(config.lenientOperatorLookup()) .withLenientOperatorLookup(config.lenientOperatorLookup())

View File

@@ -11,15 +11,15 @@ import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl; import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DimensionTimeTypeParams; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MetricTypeParams; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MetricTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;

View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;

View File

@@ -4,15 +4,15 @@ import com.tencent.supersonic.common.util.calcite.SqlParseUtils;
import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MaterializationElement; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.TimeRange; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
public class Constants { public class Constants {

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import java.util.List; import java.util.List;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import java.util.List; import java.util.List;
import lombok.Builder; import lombok.Builder;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl; package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;

View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl; import com.tencent.supersonic.semantic.query.parser.calcite.sql.S2QLSqlValidatorImpl;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@@ -35,9 +35,9 @@ public class SchemaBuilder {
Configuration.typeFactory, Configuration.typeFactory,
Configuration.config Configuration.config
); );
DSLSqlValidatorImpl dslSqlValidator = new DSLSqlValidatorImpl(Configuration.operatorTable, catalogReader, S2QLSqlValidatorImpl s2QLSqlValidator = new S2QLSqlValidatorImpl(Configuration.operatorTable, catalogReader,
Configuration.typeFactory, Configuration.validatorConfig); Configuration.typeFactory, Configuration.validatorConfig);
return new ParameterScope(dslSqlValidator, nameToTypeMap); return new ParameterScope(s2QLSqlValidator, nameToTypeMap);
} }
public static CalciteSchema getMaterializationSchema() { public static CalciteSchema getMaterializationSchema() {

View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.semantic.query.parser.calcite.schema; package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;

View File

@@ -5,11 +5,11 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;

View File

@@ -5,9 +5,9 @@ import org.apache.calcite.sql.SqlOperatorTable;
import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; import org.apache.calcite.sql.validate.SqlValidatorCatalogReader;
import org.apache.calcite.sql.validate.SqlValidatorImpl; import org.apache.calcite.sql.validate.SqlValidatorImpl;
public class DSLSqlValidatorImpl extends SqlValidatorImpl { public class S2QLSqlValidatorImpl extends SqlValidatorImpl {
public DSLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader, public S2QLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader,
RelDataTypeFactory typeFactory, Config config) { RelDataTypeFactory typeFactory, Config config) {
super(opTab, catalogReader, typeFactory, config); super(opTab, catalogReader, typeFactory, config);
} }

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql; package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;

View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration; import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList; import java.util.ArrayList;

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import java.util.List; import java.util.List;
import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorScope;

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorScope;

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node; package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import lombok.Data; import lombok.Data;

View File

@@ -6,9 +6,9 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList; import java.util.ArrayList;

View File

@@ -1,15 +1,15 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render; package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList; import java.util.ArrayList;

View File

@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;

View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.AggOption; import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable; import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.ModelService;
@@ -54,7 +54,7 @@ public class QueryReqConverter {
@Autowired @Autowired
private Catalog catalog; private Catalog catalog;
public QueryStatement convert(QueryDslReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception { public QueryStatement convert(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception {
if (Objects.isNull(modelSchemaResp)) { if (Objects.isNull(modelSchemaResp)) {
return new QueryStatement(); return new QueryStatement();
@@ -117,9 +117,9 @@ public class QueryReqConverter {
return queryStatement; return queryStatement;
} }
private AggOption getAggOption(QueryDslReq databaseReq) { private AggOption getAggOption(QueryS2QLReq databaseReq) {
// if there is no group by in dsl,set MetricTable's aggOption to "NATIVE" // if there is no group by in S2QL,set MetricTable's aggOption to "NATIVE"
// if there is count() in dsl,set MetricTable's aggOption to "NATIVE" // if there is count() in S2QL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql(); String sql = databaseReq.getSql();
if (!SqlParserSelectHelper.hasGroupBy(sql) if (!SqlParserSelectHelper.hasGroupBy(sql)
|| SqlParserSelectFunctionHelper.hasFunction(sql, "count")) { || SqlParserSelectFunctionHelper.hasFunction(sql, "count")) {
@@ -128,7 +128,7 @@ public class QueryReqConverter {
return AggOption.DEFAULT; return AggOption.DEFAULT;
} }
private void convertNameToBizName(QueryDslReq databaseReq, ModelSchemaResp modelSchemaResp) { private void convertNameToBizName(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp); Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp);
String sql = databaseReq.getSql(); String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql); log.info("convert name to bizName before:{}", sql);
@@ -155,7 +155,7 @@ public class QueryReqConverter {
return metrics; return metrics;
} }
private void functionNameCorrector(QueryDslReq databaseReq) { private void functionNameCorrector(QueryS2QLReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId()); DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId());
if (Objects.isNull(database) || Objects.isNull(database.getType())) { if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return; return;
@@ -182,7 +182,7 @@ public class QueryReqConverter {
return result; return result;
} }
public void correctTableName(QueryDslReq databaseReq) { public void correctTableName(QueryS2QLReq databaseReq) {
String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(), TABLE_PREFIX + databaseReq.getModelId()); String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(), TABLE_PREFIX + databaseReq.getModelId());
databaseReq.setSql(sql); databaseReq.setSql(sql);
} }

View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
@@ -42,11 +42,11 @@ public class QueryController {
@PostMapping("/sql") @PostMapping("/sql")
public Object queryBySql(@RequestBody QueryDslReq queryDslReq, public Object queryBySql(@RequestBody QueryS2QLReq queryS2QLReq,
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response) throws Exception { HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response); User user = UserHolder.findUser(request, response);
Object queryBySql = queryService.queryBySql(queryDslReq, user); Object queryBySql = queryService.queryBySql(queryS2QLReq, user);
log.info("queryBySql:{},queryBySql"); log.info("queryBySql:{},queryBySql");
return queryBySql; return queryBySql;
} }
@@ -124,9 +124,9 @@ public class QueryController {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum(); QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
if (QueryTypeEnum.SQL.equals(queryTypeEnum)) { if (QueryTypeEnum.SQL.equals(queryTypeEnum)) {
QueryDslReq queryDslReq = JsonUtil.toObject(queryReqJson, QueryDslReq.class); QueryS2QLReq queryS2QLReq = JsonUtil.toObject(queryReqJson, QueryS2QLReq.class);
ExplainSqlReq<QueryDslReq> explainSqlReqNew = ExplainSqlReq.<QueryDslReq>builder() ExplainSqlReq<QueryS2QLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2QLReq>builder()
.queryReq(queryDslReq) .queryReq(queryS2QLReq)
.queryTypeEnum(queryTypeEnum).build(); .queryTypeEnum(queryTypeEnum).build();
return queryService.explain(explainSqlReqNew, user); return queryService.explain(explainSqlReqNew, user);
} }

View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
@@ -18,7 +18,7 @@ import java.util.List;
public interface QueryService { public interface QueryService {
Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception; Object queryBySql(QueryS2QLReq querySqlCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception; QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;

View File

@@ -24,11 +24,11 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq; import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.utils.DslPermissionAnnotation; import com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor; import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter; import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
@@ -89,9 +89,9 @@ public class QueryServiceImpl implements QueryService {
} }
@Override @Override
@DslPermissionAnnotation @S2QLPermissionAnnotation
@SneakyThrows @SneakyThrows
public Object queryBySql(QueryDslReq querySqlCmd, User user) { public Object queryBySql(QueryS2QLReq querySqlCmd, User user) {
statUtils.initStatInfo(querySqlCmd, user); statUtils.initStatInfo(querySqlCmd, user);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
try { try {
@@ -109,7 +109,7 @@ public class QueryServiceImpl implements QueryService {
return semanticQueryEngine.execute(queryStatement); return semanticQueryEngine.execute(queryStatement);
} }
private QueryStatement convertToQueryStatement(QueryDslReq querySqlCmd, User user) throws Exception { private QueryStatement convertToQueryStatement(QueryS2QLReq querySqlCmd, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = new ArrayList<>(); List<Long> modelIds = new ArrayList<>();
modelIds.add(querySqlCmd.getModelId()); modelIds.add(querySqlCmd.getModelId());
@@ -294,8 +294,8 @@ public class QueryServiceImpl implements QueryService {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum(); QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
T queryReq = explainSqlReq.getQueryReq(); T queryReq = explainSqlReq.getQueryReq();
if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryDslReq) { if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2QLReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryDslReq) queryReq, user); QueryStatement queryStatement = convertToQueryStatement((QueryS2QLReq) queryReq, user);
return getExplainResp(queryStatement); return getExplainResp(queryStatement);
} }
if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) { if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DimensionService;
import java.util.ArrayList; import java.util.ArrayList;
@@ -45,9 +45,9 @@ public class DimValueAspect {
return queryResultWithColumns; return queryResultWithColumns;
} }
Object[] args = joinPoint.getArgs(); Object[] args = joinPoint.getArgs();
QueryDslReq queryDslReq = (QueryDslReq) args[0]; QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0];
List<DimensionResp> dimensions = dimensionService.getDimensions(queryDslReq.getModelId()); List<DimensionResp> dimensions = dimensionService.getDimensions(queryS2QLReq.getModelId());
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions); Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();

View File

@@ -24,7 +24,7 @@ import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum; import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
@@ -188,8 +188,8 @@ public class QueryStructUtils {
return resNameEnSet; return resNameEnSet;
} }
public Set<String> getResName(QueryDslReq queryDslReq) { public Set<String> getResName(QueryS2QLReq queryS2QLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryDslReq.getSql()) Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql())
.stream().collect(Collectors.toSet()); .stream().collect(Collectors.toSet());
return resNameSet; return resNameSet;
} }
@@ -199,11 +199,11 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }
public Set<String> getResNameEnExceptInternalCol(QueryDslReq queryDslReq, User user) { public Set<String> getResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq, User user) {
Set<String> resNameSet = getResName(queryDslReq); Set<String> resNameSet = getResName(queryS2QLReq);
Set<String> resNameEnSet = new HashSet<>(); Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(queryDslReq.getModelId()); List<Long> modelIds = Lists.newArrayList(queryS2QLReq.getModelId());
filter.setModelIds(modelIds); filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user); List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) { if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -234,8 +234,8 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }
public Set<String> getFilterResNameEnExceptInternalCol(QueryDslReq queryDslReq) { public Set<String> getFilterResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq) {
String sql = queryDslReq.getSql(); String sql = queryS2QLReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }

View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService; import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.query.service.AuthCommonService; import com.tencent.supersonic.semantic.query.service.AuthCommonService;
@@ -41,7 +41,7 @@ import org.springframework.util.CollectionUtils;
@Aspect @Aspect
@Order(1) @Order(1)
@Slf4j @Slf4j
public class DslDataAspect { public class S2QLDataAspect {
@Autowired @Autowired
private QueryStructUtils queryStructUtils; private QueryStructUtils queryStructUtils;
@@ -54,24 +54,24 @@ public class DslDataAspect {
@Value("${permission.data.enable:true}") @Value("${permission.data.enable:true}")
private Boolean permissionDataEnable; private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.DslPermissionAnnotation)") @Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation)")
private void dslPermissionCheck() { private void s2QLPermissionCheck() {
} }
@Around("dslPermissionCheck()") @Around("s2QLPermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
log.info("dsl permission check!"); log.info("s2ql permission check!");
Object[] objects = joinPoint.getArgs(); Object[] objects = joinPoint.getArgs();
QueryDslReq queryDslReq = (QueryDslReq) objects[0]; QueryS2QLReq queryS2QLReq = (QueryS2QLReq) objects[0];
User user = (User) objects[1]; User user = (User) objects[1];
if (!permissionDataEnable) { if (!permissionDataEnable) {
log.info("not to check dsl permission!"); log.info("not to check s2QL permission!");
return joinPoint.proceed(); return joinPoint.proceed();
} }
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) { if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information"); throw new RuntimeException("please provide user information");
} }
Long modelId = queryDslReq.getModelId(); Long modelId = queryS2QLReq.getModelId();
//1. determine whether admin of the model //1. determine whether admin of the model
if (authCommonService.doModelAdmin(user, modelId)) { if (authCommonService.doModelAdmin(user, modelId)) {
@@ -82,7 +82,7 @@ public class DslDataAspect {
authCommonService.doModelVisible(user, modelId); authCommonService.doModelVisible(user, modelId);
// 3. fetch data permission meta information // 3. fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryDslReq, user); Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2QLReq, user);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege); log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId); Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId);
@@ -97,10 +97,10 @@ public class DslDataAspect {
Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId); Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId);
// 4.if sensitive fields without permission are involved in filter, thrown an exception // 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryDslReq, resAuthSet, sensitiveResReq); doFilterCheckLogic(queryS2QLReq, resAuthSet, sensitiveResReq);
// 5.row permission pre-filter // 5.row permission pre-filter
doRowPermission(queryDslReq, authorizedResource); doRowPermission(queryS2QLReq, authorizedResource);
// 6.proceed // 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
@@ -123,7 +123,7 @@ public class DslDataAspect {
return queryResultAfterDesensitization; return queryResultAfterDesensitization;
} }
private void doRowPermission(QueryDslReq queryDslReq, AuthorizedResourceResp authorizedResource) { private void doRowPermission(QueryS2QLReq queryS2QLReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic"); log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR "); StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>(); List<String> dimensionFilters = new ArrayList<>();
@@ -143,12 +143,12 @@ public class DslDataAspect {
} }
}); });
try { try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner.toString() + " ) "); Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) { if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(queryDslReq.getSql(), expression); String sql = SqlParserAddHelper.addWhere(queryS2QLReq.getSql(), expression);
log.info("before doRowPermission, queryDslReq:{}", queryDslReq.getSql()); log.info("before doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql());
queryDslReq.setSql(sql); queryS2QLReq.setSql(sql);
log.info("after doRowPermission, queryDslReq:{}", queryDslReq.getSql()); log.info("after doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql());
} }
} catch (JSQLParserException jsqlParserException) { } catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
@@ -156,22 +156,22 @@ public class DslDataAspect {
} }
private void doFilterCheckLogic(QueryDslReq queryDslReq, Set<String> resAuthName, private void doFilterCheckLogic(QueryS2QLReq queryS2QLReq, Set<String> resAuthName,
Set<String> sensitiveResReq) { Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryDslReq); Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2QLReq);
Set<String> need2Apply = resFilterSet.stream() Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>(); Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = new ArrayList<>(); List<Long> modelIds = new ArrayList<>();
modelIds.add(queryDslReq.getModelId()); modelIds.add(queryS2QLReq.getModelId());
List<ModelResp> modelInfos = modelService.getModelList(modelIds); List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = Constants.EMPTY; String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) { if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName(); modelNameCn = modelInfos.get(0).getName();
} }
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryDslReq.getModelId()); List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryS2QLReq.getModelId());
String finalDomainNameCn = modelNameCn; String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName())) dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName())); .forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));

View File

@@ -9,6 +9,6 @@ import java.lang.annotation.Documented;
@Target(ElementType.METHOD) @Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME) @Retention(RetentionPolicy.RUNTIME)
@Documented @Documented
public @interface DslPermissionAnnotation { public @interface S2QLPermissionAnnotation {
} }

View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem; import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.model.domain.ModelService; import com.tencent.supersonic.semantic.model.domain.ModelService;
@@ -87,11 +87,11 @@ public class StatUtils {
} }
public void initStatInfo(QueryDslReq queryDslReq, User facadeUser) { public void initStatInfo(QueryS2QLReq queryS2QLReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat(); QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(queryDslReq.getSql()); List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql());
queryStatInfo.setModelId(queryDslReq.getModelId()); queryStatInfo.setModelId(queryS2QLReq.getModelId());
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryDslReq.getModelId()); ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2QLReq.getModelId());
List<String> dimensions = new ArrayList<>(); List<String> dimensions = new ArrayList<>();
if (Objects.nonNull(modelSchemaResp)) { if (Objects.nonNull(modelSchemaResp)) {
@@ -106,12 +106,12 @@ public class StatUtils {
String userName = getUserName(facadeUser); String userName = getUserName(facadeUser);
try { try {
queryStatInfo.setTraceId("") queryStatInfo.setTraceId("")
.setModelId(queryDslReq.getModelId()) .setModelId(queryS2QLReq.getModelId())
.setUser(userName) .setUser(userName)
.setQueryType(QueryTypeEnum.SQL.getValue()) .setQueryType(QueryTypeEnum.SQL.getValue())
.setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState()) .setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())
.setQuerySqlCmd(queryDslReq.toString()) .setQuerySqlCmd(queryS2QLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryDslReq.toString())) .setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2QLReq.toString()))
.setStartTime(System.currentTimeMillis()) .setStartTime(System.currentTimeMillis())
.setUseResultCache(true) .setUseResultCache(true)
.setUseSqlCache(true) .setUseSqlCache(true)