topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(feature)(project) change dsl to s2ql in project and queryMode to llmParser (#250)

Browse Source
This commit is contained in:
lexluo09
2023-10-18 09:53:01 +08:00
committed by GitHub
parent bf5be11549
commit 8d81f63e08
77 changed files with 320 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticInterpreter.java
View File

@@ -14,7 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
@@ -37,7 +37,7 @@ public interface SemanticInterpreter {
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user);
QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user);
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);

2
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SqlInfo.java
View File

@@ -5,7 +5,7 @@ import lombok.Data;
@Data
public class SqlInfo {
private String llmParseSql;
private String s2QL;
private String logicSql;
private String querySql;
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/AgentToolType.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.agent.tool;
public enum AgentToolType {
RULE,
DSL,
LLM_S2QL,
PLUGIN,
INTERPRET
}

16
chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/CommonAgentTool.java Normal file
View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.chat.agent.tool;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class CommonAgentTool extends AgentTool {
protected List<Long> modelIds;
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/DslTool.java → chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/LLMParserTool.java
View File

@@ -5,9 +5,7 @@ import lombok.Data;
import java.util.List;
@Data
public class DslTool extends AgentTool {
private List<Long> modelIds;
public class LLMParserTool extends CommonAgentTool {
private List<String> exampleQuestions;

3
chat/core/src/main/java/com/tencent/supersonic/chat/agent/tool/RuleQueryTool.java
View File

@@ -7,9 +7,8 @@ import org.apache.commons.collections.CollectionUtils;
import java.util.List;
@Data
public class RuleQueryTool extends AgentTool {
public class RuleQueryTool extends CommonAgentTool {
private List<Long> modelIds;
private List<String> queryModes;

12
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/GlobalBeforeCorrector.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue;
import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -67,11 +67,11 @@ public class GlobalBeforeCorrector extends BaseSemanticCorrector {
return null;
}
DSLParseResult dslParseResult = JsonUtil.toObject(JsonUtil.toString(context), DSLParseResult.class);
if (Objects.isNull(dslParseResult) || Objects.isNull(dslParseResult.getLlmReq())) {
ParseResult parseResult = JsonUtil.toObject(JsonUtil.toString(context), ParseResult.class);
if (Objects.isNull(parseResult) || Objects.isNull(parseResult.getLlmReq())) {
return null;
}
LLMReq llmReq = dslParseResult.getLlmReq();
LLMReq llmReq = parseResult.getLlmReq();
return llmReq.getLinking();
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/corrector/WhereCorrector.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
import com.tencent.supersonic.chat.api.pojo.SemanticCorrectInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLDateHelper;
import com.tencent.supersonic.chat.parser.llm.s2ql.S2QLDateHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils;
@@ -72,7 +72,7 @@ public class WhereCorrector extends BaseSemanticCorrector {
String sql = semanticCorrectInfo.getSql();
List<String> whereFields = SqlParserSelectHelper.getWhereFields(sql);
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(DateUtils.DATE_FIELD)) {
String currentDate = DSLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId());
String currentDate = S2QLDateHelper.getReferenceDate(semanticCorrectInfo.getParseInfo().getModelId());
if (StringUtils.isNotBlank(currentDate)) {
sql = SqlParserAddHelper.addParenthesisToWhere(sql);
sql = SqlParserAddHelper.addWhere(sql, DateUtils.DATE_FIELD, currentDate);

2
chat/core/src/main/java/com/tencent/supersonic/chat/mapper/MapperHelper.java
View File

@@ -88,7 +88,7 @@ public class MapperHelper {
AgentService agentService = ContextUtils.getBean(AgentService.class);
Set<Long> detectModelIds = agentService.getDslToolsModelIds(request.getAgentId(), null);
Set<Long> detectModelIds = agentService.getModelIds(request.getAgentId(), null);
//contains all
if (agentService.containsAllModel(detectModelIds)) {
if (Objects.nonNull(modelId) && modelId > 0) {

4
chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.OptimizationConfig;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
@@ -20,7 +20,7 @@ public class SatisfactionChecker {
// check all the parse info in candidate
public static boolean check(QueryContext queryContext) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
if (query.getQueryMode().equals(DslQuery.QUERY_MODE)) {
if (query.getQueryMode().equals(S2QLQuery.QUERY_MODE)) {
continue;
}
if (checkThreshold(queryContext.getRequest().getQueryText(), query.getParseInfo())) {

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/HeuristicModelResolver.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/HeuristicModelResolver.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl;
package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;

67
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParser.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.chat.parser.llm.dsl;
package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.DslTool;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.component.SemanticCorrector;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
@@ -18,10 +18,10 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.config.LLMParserConfig;
import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq.ElementValue;
import com.tencent.supersonic.chat.query.llm.dsl.LLMResp;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq.ElementValue;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
@@ -59,18 +59,18 @@ import org.springframework.util.CollectionUtils;
import org.springframework.web.client.RestTemplate;
@Slf4j
public class LLMDslParser implements SemanticParser {
public class LLMS2QLParser implements SemanticParser {
@Override
public void parse(QueryContext queryCtx, ChatContext chatCtx) {
QueryReq request = queryCtx.getRequest();
LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class);
if (StringUtils.isEmpty(llmParserConfig.getUrl())) {
log.info("llm parser url is empty, skip dsl parser, llmParserConfig:{}", llmParserConfig);
log.info("llm parser url is empty, skip {} , llmParserConfig:{}", LLMS2QLParser.class, llmParserConfig);
return;
}
if (SatisfactionChecker.check(queryCtx)) {
log.info("skip dsl parser, queryText:{}", request.getQueryText());
log.info("skip {}, queryText:{}", LLMS2QLParser.class, request.getQueryText());
return;
}
try {
@@ -79,9 +79,9 @@ public class LLMDslParser implements SemanticParser {
return;
}
DslTool dslTool = getDslTool(request, modelId);
if (Objects.isNull(dslTool)) {
log.info("no dsl tool in this agent, skip dsl parser");
CommonAgentTool commonAgentTool = getParserTool(request, modelId);
if (Objects.isNull(commonAgentTool)) {
log.info("no tool in this agent, skip {}", LLMS2QLParser.class);
return;
}
@@ -91,10 +91,10 @@ public class LLMDslParser implements SemanticParser {
if (Objects.isNull(llmResp)) {
return;
}
DSLParseResult dslParseResult = DSLParseResult.builder().request(request)
.dslTool(dslTool).llmReq(llmReq).llmResp(llmResp).build();
ParseResult parseResult = ParseResult.builder().request(request)
.commonAgentTool(commonAgentTool).llmReq(llmReq).llmResp(llmResp).build();
SemanticParseInfo parseInfo = getParseInfo(queryCtx, modelId, dslTool, dslParseResult);
SemanticParseInfo parseInfo = getParseInfo(queryCtx, modelId, commonAgentTool, parseResult);
SemanticCorrectInfo semanticCorrectInfo = getCorrectorSql(queryCtx, parseInfo, llmResp.getSqlOutput());
@@ -103,7 +103,7 @@ public class LLMDslParser implements SemanticParser {
updateParseInfo(semanticCorrectInfo, modelId, parseInfo);
} catch (Exception e) {
log.error("LLMDSLParser error", e);
log.error("LLMS2QLParser error", e);
}
}
@@ -243,12 +243,12 @@ public class LLMDslParser implements SemanticParser {
.queryFilters(queryCtx.getRequest().getQueryFilters()).sql(sql)
.parseInfo(parseInfo).build();
List<SemanticCorrector> dslCorrections = ComponentFactory.getSqlCorrections();
List<SemanticCorrector> corrections = ComponentFactory.getSqlCorrections();
dslCorrections.forEach(dslCorrection -> {
corrections.forEach(correction -> {
try {
dslCorrection.correct(correctInfo);
log.info("sqlCorrection:{} sql:{}", dslCorrection.getClass().getSimpleName(), correctInfo.getSql());
correction.correct(correctInfo);
log.info("sqlCorrection:{} sql:{}", correction.getClass().getSimpleName(), correctInfo.getSql());
} catch (Exception e) {
log.error(String.format("correct error,correctInfo:%s", correctInfo), e);
}
@@ -256,21 +256,21 @@ public class LLMDslParser implements SemanticParser {
return correctInfo;
}
private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, DslTool dslTool,
DSLParseResult dslParseResult) {
PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(DslQuery.QUERY_MODE);
private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, CommonAgentTool commonAgentTool,
ParseResult parseResult) {
PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(S2QLQuery.QUERY_MODE);
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(modelId));
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, dslParseResult);
properties.put(Constants.CONTEXT, parseResult);
properties.put("type", "internal");
properties.put("name", dslTool.getName());
properties.put("name", commonAgentTool.getName());
parseInfo.setProperties(properties);
parseInfo.setScore(queryCtx.getRequest().getQueryText().length());
parseInfo.setQueryMode(semanticQuery.getQueryMode());
parseInfo.getSqlInfo().setLlmParseSql(dslParseResult.getLlmResp().getSqlOutput());
parseInfo.getSqlInfo().setS2QL(parseResult.getLlmResp().getSqlOutput());
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
Map<Long, String> modelIdToName = semanticSchema.getModelIdToName();
@@ -284,10 +284,11 @@ public class LLMDslParser implements SemanticParser {
return parseInfo;
}
private DslTool getDslTool(QueryReq request, Long modelId) {
private CommonAgentTool getParserTool(QueryReq request, Long modelId) {
AgentService agentService = ContextUtils.getBean(AgentService.class);
List<DslTool> dslTools = agentService.getDslTools(request.getAgentId(), AgentToolType.DSL);
Optional<DslTool> dslToolOptional = dslTools.stream()
List<CommonAgentTool> commonAgentTools = agentService.getParserTools(request.getAgentId(),
AgentToolType.LLM_S2QL);
Optional<CommonAgentTool> llmParserTool = commonAgentTools.stream()
.filter(tool -> {
List<Long> modelIds = tool.getModelIds();
if (agentService.containsAllModel(new HashSet<>(modelIds))) {
@@ -296,18 +297,18 @@ public class LLMDslParser implements SemanticParser {
return modelIds.contains(modelId);
})
.findFirst();
return dslToolOptional.orElse(null);
return llmParserTool.orElse(null);
}
private Long getModelId(QueryContext queryCtx, ChatContext chatCtx, Integer agentId) {
AgentService agentService = ContextUtils.getBean(AgentService.class);
Set<Long> distinctModelIds = agentService.getDslToolsModelIds(agentId, AgentToolType.DSL);
Set<Long> distinctModelIds = agentService.getModelIds(agentId, AgentToolType.LLM_S2QL);
if (agentService.containsAllModel(distinctModelIds)) {
distinctModelIds = new HashSet<>();
}
ModelResolver modelResolver = ComponentFactory.getModelResolver();
Long modelId = modelResolver.resolve(queryCtx, chatCtx, distinctModelIds);
log.info("resolve modelId:{},dslModels:{}", modelId, distinctModelIds);
log.info("resolve modelId:{},llmParser Models:{}", modelId, distinctModelIds);
return modelId;
}
@@ -354,7 +355,7 @@ public class LLMDslParser implements SemanticParser {
linking.addAll(getValueList(queryCtx, modelId, semanticSchema));
llmReq.setLinking(linking);
String currentDate = DSLDateHelper.getReferenceDate(modelId);
String currentDate = S2QLDateHelper.getReferenceDate(modelId);
llmReq.setCurrentDate(currentDate);
return llmReq;
}

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/ModelMatchResult.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelMatchResult.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl;
package com.tencent.supersonic.chat.parser.llm.s2ql;
import lombok.Data;

2
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/ModelResolver.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ModelResolver.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl;
package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.tencent.supersonic.chat.api.pojo.ChatContext;

12
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLParseResult.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/ParseResult.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.parser.llm.dsl;
package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.tencent.supersonic.chat.agent.tool.DslTool;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMReq;
import com.tencent.supersonic.chat.query.llm.dsl.LLMResp;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMReq;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@@ -13,7 +13,7 @@ import lombok.NoArgsConstructor;
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class DSLParseResult {
public class ParseResult {
private LLMReq llmReq;
@@ -21,5 +21,5 @@ public class DSLParseResult {
private QueryReq request;
private DslTool dslTool;
private CommonAgentTool commonAgentTool;
}

4
chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/dsl/DSLDateHelper.java → chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/S2QLDateHelper.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl;
package com.tencent.supersonic.chat.parser.llm.s2ql;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
@@ -11,7 +11,7 @@ import java.util.List;
import java.util.Objects;
import org.apache.commons.collections.CollectionUtils;
public class DSLDateHelper {
public class S2QLDateHelper {
public static String getReferenceDate(Long modelId) {
String defaultDate = DateUtils.getBeforeDate(0);

4
chat/core/src/main/java/com/tencent/supersonic/chat/parser/plugin/function/FunctionBasedParser.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.service.PluginService;
import com.tencent.supersonic.common.util.ContextUtils;
import java.net.URI;
@@ -102,7 +102,7 @@ public class FunctionBasedParser extends PluginParser {
log.info("user decide Model:{}", modelId);
List<Plugin> plugins = getPluginList(queryContext);
List<PluginParseConfig> functionDOList = plugins.stream().filter(plugin -> {
if (DslQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
if (S2QLQuery.QUERY_MODE.equalsIgnoreCase(plugin.getType())) {
return false;
}
if (plugin.getParseModeConfig() == null) {

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/LLMReq.java → chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMReq.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.dsl;
package com.tencent.supersonic.chat.query.llm.s2ql;
import java.util.List;
import lombok.Data;

2
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/LLMResp.java → chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/LLMResp.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.query.llm.dsl;
package com.tencent.supersonic.chat.query.llm.s2ql;
import java.util.List;
import lombok.Data;

29
chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java → chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/s2ql/S2QLQuery.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.chat.query.llm.dsl;
package com.tencent.supersonic.chat.query.llm.s2ql;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.utils.ComponentFactory;
@@ -16,7 +16,7 @@ import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -26,12 +26,12 @@ import org.springframework.stereotype.Component;
@Slf4j
@Component
public class DslQuery extends PluginSemanticQuery {
public class S2QLQuery extends PluginSemanticQuery {
public static final String QUERY_MODE = "DSL";
public static final String QUERY_MODE = "LLM_S2QL";
protected SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
public DslQuery() {
public S2QLQuery() {
QueryManager.register(this);
}
@@ -45,10 +45,10 @@ public class DslQuery extends PluginSemanticQuery {
LLMResp llmResp = getLlmResp();
long startTime = System.currentTimeMillis();
QueryDslReq queryDslReq = getQueryDslReq(llmResp);
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByDsl(queryDslReq, user);
QueryS2QLReq queryS2QLReq = getQueryS2QLReq(llmResp);
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2QL(queryS2QLReq, user);
log.info("queryByDsl cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput());
log.info("queryByS2QL cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput());
QueryResult queryResult = new QueryResult();
if (Objects.nonNull(queryResp)) {
@@ -69,13 +69,12 @@ public class DslQuery extends PluginSemanticQuery {
private LLMResp getLlmResp() {
String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class);
return dslParseResult.getLlmResp();
ParseResult parseResult = JsonUtil.toObject(json, ParseResult.class);
return parseResult.getLlmResp();
}
private QueryDslReq getQueryDslReq(LLMResp llmResp) {
QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId());
return queryDslReq;
private QueryS2QLReq getQueryS2QLReq(LLMResp llmResp) {
return QueryReqBuilder.buildS2QLReq(llmResp.getCorrectorSql(), parseInfo.getModelId());
}
@Override
@@ -84,7 +83,7 @@ public class DslQuery extends PluginSemanticQuery {
try {
explainSqlReq = ExplainSqlReq.builder()
.queryTypeEnum(QueryTypeEnum.SQL)
.queryReq(getQueryDslReq(getLlmResp()))
.queryReq(getQueryS2QLReq(getLlmResp()))
.build();
return semanticInterpreter.explain(explainSqlReq, user);
} catch (Exception e) {

4
chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java
View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List;
@@ -24,7 +24,7 @@ public class EntityInfoExecuteResponder implements ExecuteResponder {
return;
}
String queryMode = semanticParseInfo.getQueryMode();
if (QueryManager.isPluginQuery(queryMode) && !DslQuery.QUERY_MODE.equals(queryMode)) {
if (QueryManager.isPluginQuery(queryMode) && !S2QLQuery.QUERY_MODE.equals(queryMode)) {
return;
}
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);

4
chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java
View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List;
@@ -24,7 +24,7 @@ public class EntityInfoParseResponder implements ParseResponder {
QueryReq queryReq = queryContext.getRequest();
selectedParses.forEach(parseInfo -> {
if (QueryManager.isPluginQuery(parseInfo.getQueryMode())
&& !DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
&& !S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
return;
}
//1. set entity info

6
chat/core/src/main/java/com/tencent/supersonic/chat/service/AgentService.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.agent.Agent;
import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.DslTool;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import java.util.List;
import java.util.Set;
@@ -19,9 +19,9 @@ public interface AgentService {
void deleteAgent(Integer id);
List<DslTool> getDslTools(Integer agentId, AgentToolType agentToolType);
List<CommonAgentTool> getParserTools(Integer agentId, AgentToolType agentToolType);
Set<Long> getDslToolsModelIds(Integer agentId, AgentToolType agentToolType);
Set<Long> getModelIds(Integer agentId, AgentToolType agentToolType);
boolean containsAllModel(Set<Long> detectModelIds);
}

15
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/AgentServiceImpl.java
View File

@@ -5,7 +5,7 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.agent.Agent;
import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.DslTool;
import com.tencent.supersonic.chat.agent.tool.CommonAgentTool;
import com.tencent.supersonic.chat.persistence.dataobject.AgentDO;
import com.tencent.supersonic.chat.persistence.repository.AgentRepository;
import com.tencent.supersonic.chat.service.AgentService;
@@ -87,7 +87,7 @@ public class AgentServiceImpl implements AgentService {
return agentDO;
}
public List<DslTool> getDslTools(Integer agentId, AgentToolType agentToolType) {
public List<CommonAgentTool> getParserTools(Integer agentId, AgentToolType agentToolType) {
Agent agent = getAgent(agentId);
if (agent == null) {
return Lists.newArrayList();
@@ -96,15 +96,16 @@ public class AgentServiceImpl implements AgentService {
if (CollectionUtils.isEmpty(tools)) {
return Lists.newArrayList();
}
return tools.stream().map(tool -> JSONObject.parseObject(tool, DslTool.class)).collect(Collectors.toList());
return tools.stream().map(tool -> JSONObject.parseObject(tool, CommonAgentTool.class))
.collect(Collectors.toList());
}
public Set<Long> getDslToolsModelIds(Integer agentId, AgentToolType agentToolType) {
List<DslTool> dslTools = getDslTools(agentId, agentToolType);
if (CollectionUtils.isEmpty(dslTools)) {
public Set<Long> getModelIds(Integer agentId, AgentToolType agentToolType) {
List<CommonAgentTool> commonAgentTools = getParserTools(agentId, agentToolType);
if (CollectionUtils.isEmpty(commonAgentTools)) {
return new HashSet<>();
}
return dslTools.stream().map(DslTool::getModelIds)
return commonAgentTools.stream().map(CommonAgentTool::getModelIds)
.filter(modelIds -> !CollectionUtils.isEmpty(modelIds))
.flatMap(Collection::stream)
.collect(Collectors.toSet());

34
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java
View File

@@ -19,15 +19,15 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult;
import com.tencent.supersonic.chat.parser.llm.s2ql.ParseResult;
import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.persistence.dataobject.CostType;
import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.query.llm.dsl.DslQuery;
import com.tencent.supersonic.chat.query.llm.dsl.LLMResp;
import com.tencent.supersonic.chat.query.llm.s2ql.S2QLQuery;
import com.tencent.supersonic.chat.query.llm.s2ql.LLMResp;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder;
import com.tencent.supersonic.chat.service.ChatService;
@@ -239,7 +239,7 @@ public class QueryServiceImpl implements QueryService {
}
private void saveSolvedQuery(ExecuteQueryReq queryReq, SemanticParseInfo parseInfo,
ChatQueryDO chatQueryDO, QueryResult queryResult) {
ChatQueryDO chatQueryDO, QueryResult queryResult) {
if (queryResult.getResponse() == null && CollectionUtils.isEmpty(queryResult.getQueryResults())) {
return;
}
@@ -305,12 +305,12 @@ public class QueryServiceImpl implements QueryService {
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
if (DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
Map<String, Map<String, String>> havingFiledNameToValueMap = new HashMap<>();
String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class);
LLMResp llmResp = dslParseResult.getLlmResp();
ParseResult parseResult = JsonUtil.toObject(json, ParseResult.class);
LLMResp llmResp = parseResult.getLlmResp();
String correctorSql = llmResp.getCorrectorSql();
log.info("correctorSql before replacing:{}", correctorSql);
@@ -330,9 +330,9 @@ public class QueryServiceImpl implements QueryService {
correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap);
log.info("correctorSql after replacing:{}", correctorSql);
llmResp.setCorrectorSql(correctorSql);
dslParseResult.setLlmResp(llmResp);
parseResult.setLlmResp(llmResp);
Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, dslParseResult);
properties.put(Constants.CONTEXT, parseResult);
parseInfo.setProperties(properties);
parseInfo.getSqlInfo().setLogicSql(correctorSql);
semanticQuery.setParseInfo(parseInfo);
@@ -399,29 +399,29 @@ public class QueryServiceImpl implements QueryService {
if (CollectionUtils.isEmpty(metricFilters)) {
return;
}
for (QueryFilter dslQueryFilter : metricFilters) {
for (QueryFilter queryFilter : metricFilters) {
Map<String, String> map = new HashMap<>();
for (FilterExpression filterExpression : filterExpressionList) {
if (filterExpression.getFieldName() != null
&& filterExpression.getFieldName().contains(dslQueryFilter.getName())
&& dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) {
map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString());
&& filterExpression.getFieldName().contains(queryFilter.getName())
&& queryFilter.getOperator().getValue().equals(filterExpression.getOperator())) {
map.put(filterExpression.getFieldValue().toString(), queryFilter.getValue().toString());
contextMetricFilters.stream().forEach(o -> {
if (o.getName().equals(dslQueryFilter.getName())) {
o.setValue(dslQueryFilter.getValue());
if (o.getName().equals(queryFilter.getName())) {
o.setValue(queryFilter.getValue());
}
});
break;
}
}
filedNameToValueMap.put(dslQueryFilter.getName(), map);
filedNameToValueMap.put(queryFilter.getName(), map);
}
}
private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) {
SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
if (DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
if (S2QLQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
return parseInfo;
}
if (CollectionUtils.isNotEmpty(queryData.getDimensions())) {

7
chat/core/src/main/java/com/tencent/supersonic/chat/utils/ComponentFactory.java
View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver;
import com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver;
import com.tencent.supersonic.chat.query.QuerySelector;
import com.tencent.supersonic.chat.responder.execute.ExecuteResponder;
import com.tencent.supersonic.chat.responder.parse.ParseResponder;
@@ -20,7 +20,7 @@ public class ComponentFactory {
private static List<SchemaMapper> schemaMappers = new ArrayList<>();
private static List<SemanticParser> semanticParsers = new ArrayList<>();
private static List<SemanticCorrector> dslCorrections = new ArrayList<>();
private static List<SemanticCorrector> s2QLCorrections = new ArrayList<>();
private static SemanticInterpreter semanticInterpreter;
private static List<ParseResponder> parseResponders = new ArrayList<>();
private static List<ExecuteResponder> executeResponders = new ArrayList<>();
@@ -35,7 +35,8 @@ public class ComponentFactory {
}
public static List<SemanticCorrector> getSqlCorrections() {
return CollectionUtils.isEmpty(dslCorrections) ? init(SemanticCorrector.class, dslCorrections) : dslCorrections;
return CollectionUtils.isEmpty(s2QLCorrections) ? init(SemanticCorrector.class,
s2QLCorrections) : s2QLCorrections;
}
public static List<ParseResponder> getParseResponders() {

14
chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java
View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.time.LocalDate;
@@ -123,19 +123,19 @@ public class QueryReqBuilder {
}
/**
* convert to QueryDslReq
* convert to QueryS2QLReq
*
* @param querySql
* @param modelId
* @return
*/
public static QueryDslReq buildDslReq(String querySql, Long modelId) {
QueryDslReq queryDslReq = new QueryDslReq();
public static QueryS2QLReq buildS2QLReq(String querySql, Long modelId) {
QueryS2QLReq queryS2QLReq = new QueryS2QLReq();
if (Objects.nonNull(querySql)) {
queryDslReq.setSql(querySql);
queryS2QLReq.setSql(querySql);
}
queryDslReq.setModelId(modelId);
return queryDslReq;
queryS2QLReq.setModelId(modelId);
return queryS2QLReq;
}

8
chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/dsl/LLMDslParserTest.java → chat/core/src/test/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParserTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.parser.llm.dsl;
package com.tencent.supersonic.chat.parser.llm.s2ql;
import static org.mockito.Mockito.when;
@@ -16,7 +16,7 @@ import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
class LLMDslParserTest {
class LLMS2QLParserTest {
@Test
void setFilter() {
@@ -72,9 +72,9 @@ class LLMDslParserTest {
.parseInfo(parseInfo)
.build();
LLMDslParser llmDslParser = new LLMDslParser();
LLMS2QLParser llms2QLParser = new LLMS2QLParser();
llmDslParser.updateParseInfo(semanticCorrectInfo, 2L, parseInfo);
llms2QLParser.updateParseInfo(semanticCorrectInfo, 2L, parseInfo);
}
}

6
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticInterpreter.java
View File

@@ -17,7 +17,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
@@ -56,9 +56,9 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@Override
@SneakyThrows
public QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user) {
public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) {
queryService = ContextUtils.getBean(QueryService.class);
Object object = queryService.queryBySql(queryDslReq, user);
Object object = queryService.queryBySql(queryS2QLReq, user);
QueryResultWithSchemaResp queryResultWithSchemaResp = JsonUtil.toObject(JsonUtil.toString(object),
QueryResultWithSchemaResp.class);
return queryResultWithSchemaResp;

6
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticInterpreter.java
View File

@@ -31,7 +31,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import java.net.URI;
@@ -83,10 +83,10 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
}
@Override
public QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user) {
public QueryResultWithSchemaResp queryByS2QL(QueryS2QLReq queryS2QLReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
new Gson().toJson(queryDslReq));
new Gson().toJson(queryS2QLReq));
}
public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) {

6
launchers/chat/src/main/resources/META-INF/spring.factories
View File

@@ -11,7 +11,7 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \
com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \
com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \
com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser, \
com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser
@@ -21,8 +21,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.dsl.HeuristicModelResolver
com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

22
launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java
View File

@@ -6,7 +6,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.agent.Agent;
import com.tencent.supersonic.chat.agent.AgentConfig;
import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.DslTool;
import com.tencent.supersonic.chat.agent.tool.LLMParserTool;
import com.tencent.supersonic.chat.agent.tool.RuleQueryTool;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
@@ -228,11 +228,11 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
));
agentConfig.getTools().add(ruleQueryTool);
DslTool dslTool = new DslTool();
dslTool.setId("1");
dslTool.setType(AgentToolType.DSL);
dslTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(dslTool);
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.LLM_S2QL);
llmParserTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(llmParserTool);
agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
agentService.createAgent(agent, User.getFakeUser());
@@ -255,11 +255,11 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
"ENTITY_DETAIL", "ENTITY_LIST_FILTER", "ENTITY_ID"));
agentConfig.getTools().add(ruleQueryTool);
DslTool dslTool = new DslTool();
dslTool.setId("1");
dslTool.setType(AgentToolType.DSL);
dslTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(dslTool);
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.LLM_S2QL);
llmParserTool.setModelIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(llmParserTool);
agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
agentService.createAgent(agent, User.getFakeUser());

6
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -11,7 +11,7 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.MetricCheckParser, \
com.tencent.supersonic.chat.parser.rule.TimeRangeParser, \
com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \
com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser, \
com.tencent.supersonic.chat.parser.llm.s2ql.LLMS2QLParser, \
com.tencent.supersonic.chat.parser.plugin.embedding.EmbeddingBasedParser, \
com.tencent.supersonic.chat.parser.plugin.function.FunctionBasedParser
@@ -22,8 +22,8 @@ com.tencent.supersonic.chat.api.component.SemanticInterpreter=\
com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.parser.llm.dsl.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.dsl.HeuristicModelResolver
com.tencent.supersonic.chat.parser.llm.s2ql.ModelResolver=\
com.tencent.supersonic.chat.parser.llm.s2ql.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

2
launchers/standalone/src/main/resources/db/sql-update.sql
View File

@@ -66,3 +66,5 @@ alter table s2_dimension add column data_type varchar(50) not null DEFAULT 'va
alter table s2_query_stat_info add column `query_opt_mode` varchar(20) DEFAULT NULL COMMENT '优化模式';
alter table s2_datasource add column depends text COMMENT '上游依赖标识' after datasource_detail;
--20231018
UPDATE `s2_agent` SET `config` = replace (`config`,'DSL','LLM_S2QL') WHERE `config` LIKE '%DSL%';

2
pom.xml
View File

@@ -64,7 +64,7 @@
<xk.time.version>3.2.4</xk.time.version>
<mockito-inline.version>4.5.1</mockito-inline.version>
<jsqlparser.version>4.5</jsqlparser.version>
<revision>0.7.5-SNAPSHOT</revision>
<revision>0.7.6-SNAPSHOT</revision>
<!-- Do not bump spotless plugin version since 2.30.0 is the latest version supports Java 8-->
<maven.plugin.spotless.version>2.30.0</maven.plugin.spotless.version>
<spotless.python.includes></spotless.python.includes>

2
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryDslReq.java → semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryS2QLReq.java
View File

@@ -6,7 +6,7 @@ import lombok.ToString;
@Data
@ToString
public class QueryDslReq {
public class QueryS2QLReq {
private Long modelId;

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java
View File

@@ -18,16 +18,16 @@ import com.tencent.supersonic.semantic.materialization.domain.MaterializationCon
import com.tencent.supersonic.semantic.materialization.domain.MaterializationRecordService;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.SemanticSchemaManager;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.MaterializationPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/CalciteSqlParser.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.SqlParser;
import com.tencent.supersonic.semantic.query.parser.calcite.planner.AggPlanner;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import org.springframework.stereotype.Component;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/Configuration.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.semantic.query.parser.calcite;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSqlDialect;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -33,7 +32,6 @@ import org.apache.calcite.sql2rel.SqlToRelConverter;
public class Configuration {
public static Properties configProperties = new Properties();
public static DSLSqlValidatorImpl dslSqlValidator;
public static RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
public static SqlOperatorTable operatorTable = SqlStdOperatorTable.instance();
public static CalciteConnectionConfig config = new CalciteConnectionConfigImpl(configProperties);
@@ -71,7 +69,7 @@ public class Configuration {
public static SqlValidator getSqlValidator(CalciteSchema rootSchema) {
List<SqlOperatorTable> tables = new ArrayList<>();
tables.add(SqlStdOperatorTable.instance());
SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables); //.of(SqlStdOperatorTable.instance());
SqlOperatorTable operatorTable = new ChainedSqlOperatorTable(tables);
//operatorTable.
SqlValidator.Config validatorConfig = SqlValidator.Config.DEFAULT
.withLenientOperatorLookup(config.lenientOperatorLookup())

18
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/SemanticSchemaManager.java
View File

@@ -11,15 +11,15 @@ import com.tencent.supersonic.semantic.api.model.yaml.MeasureYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MetricTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MetricTypeParams;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;
import java.util.HashMap;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/AggPlanner.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.semantic.query.parser.calcite.planner;
import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;

18
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/planner/MaterializationPlanner.java
View File

@@ -4,15 +4,15 @@ import com.tencent.supersonic.common.util.calcite.SqlParseUtils;
import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.MaterializationElement;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.TimeRange;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SchemaBuilder;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Constants.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Constants.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
public class Constants {

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DataSource.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DataSource.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import java.util.List;
import lombok.Builder;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Dimension.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Dimension.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/DimensionTimeTypeParams.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/DimensionTimeTypeParams.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Identify.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Identify.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import lombok.AllArgsConstructor;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Materialization.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Materialization.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import java.util.ArrayList;
import java.util.List;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MaterializationElement.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MaterializationElement.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import java.util.List;
import lombok.Builder;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Measure.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Measure.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import lombok.AllArgsConstructor;
import lombok.Builder;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/Metric.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/Metric.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticItem;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/MetricTypeParams.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/MetricTypeParams.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import java.util.List;
import lombok.Data;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/SemanticModel.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/SemanticModel.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import java.util.ArrayList;
import java.util.HashMap;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/dsl/TimeRange.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/s2ql/TimeRange.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.semantic.query.parser.calcite.dsl;
package com.tencent.supersonic.semantic.query.parser.calcite.s2ql;
import lombok.Builder;
import lombok.Data;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SchemaBuilder.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.DSLSqlValidatorImpl;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.S2QLSqlValidatorImpl;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -35,9 +35,9 @@ public class SchemaBuilder {
Configuration.typeFactory,
Configuration.config
);
DSLSqlValidatorImpl dslSqlValidator = new DSLSqlValidatorImpl(Configuration.operatorTable, catalogReader,
S2QLSqlValidatorImpl s2QLSqlValidator = new S2QLSqlValidatorImpl(Configuration.operatorTable, catalogReader,
Configuration.typeFactory, Configuration.validatorConfig);
return new ParameterScope(dslSqlValidator, nameToTypeMap);
return new ParameterScope(s2QLSqlValidator, nameToTypeMap);
}
public static CalciteSchema getMaterializationSchema() {

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/schema/SemanticSchema.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.semantic.query.parser.calcite.schema;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.SemanticModel;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.SemanticModel;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/Renderer.java
View File

@@ -5,11 +5,11 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MeasureNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.HashSet;
import java.util.List;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/DSLSqlValidatorImpl.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/S2QLSqlValidatorImpl.java
View File

@@ -5,9 +5,9 @@ import org.apache.calcite.sql.SqlOperatorTable;
import org.apache.calcite.sql.validate.SqlValidatorCatalogReader;
import org.apache.calcite.sql.validate.SqlValidatorImpl;
public class DSLSqlValidatorImpl extends SqlValidatorImpl {
public class S2QLSqlValidatorImpl extends SqlValidatorImpl {
public DSLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader,
public S2QLSqlValidatorImpl(SqlOperatorTable opTab, SqlValidatorCatalogReader catalogReader,
RelDataTypeFactory typeFactory, Config config) {
super(opTab, catalogReader, typeFactory, config);
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/TableView.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DataSourceNode.java
View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.Configuration;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import java.util.ArrayList;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/DimensionNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import java.util.List;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/IdentifyNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type;
import java.util.List;
import java.util.Optional;
import java.util.Set;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MeasureNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.validate.SqlValidatorScope;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/node/MetricNode.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.node;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/FilterRender.java
View File

@@ -6,9 +6,9 @@ import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;

14
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/JoinRender.java
View File

@@ -1,15 +1,15 @@
package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify.Type;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Materialization.TimePartType;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/OutputRender.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode;
import java.util.ArrayList;

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java
View File

@@ -2,12 +2,12 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Constants;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.DataSource;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Dimension;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Identify;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Measure;
import com.tencent.supersonic.semantic.query.parser.calcite.s2ql.Metric;
import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer;
import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView;

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.AggOption;
import com.tencent.supersonic.semantic.api.query.pojo.MetricTable;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.ModelService;
@@ -54,7 +54,7 @@ public class QueryReqConverter {
@Autowired
private Catalog catalog;
public QueryStatement convert(QueryDslReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception {
public QueryStatement convert(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) throws Exception {
if (Objects.isNull(modelSchemaResp)) {
return new QueryStatement();
@@ -117,9 +117,9 @@ public class QueryReqConverter {
return queryStatement;
}
private AggOption getAggOption(QueryDslReq databaseReq) {
// if there is no group by in dsl,set MetricTable's aggOption to "NATIVE"
// if there is count() in dsl,set MetricTable's aggOption to "NATIVE"
private AggOption getAggOption(QueryS2QLReq databaseReq) {
// if there is no group by in S2QL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2QL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql();
if (!SqlParserSelectHelper.hasGroupBy(sql)
|| SqlParserSelectFunctionHelper.hasFunction(sql, "count")) {
@@ -128,7 +128,7 @@ public class QueryReqConverter {
return AggOption.DEFAULT;
}
private void convertNameToBizName(QueryDslReq databaseReq, ModelSchemaResp modelSchemaResp) {
private void convertNameToBizName(QueryS2QLReq databaseReq, ModelSchemaResp modelSchemaResp) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
@@ -155,7 +155,7 @@ public class QueryReqConverter {
return metrics;
}
private void functionNameCorrector(QueryDslReq databaseReq) {
private void functionNameCorrector(QueryS2QLReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelId());
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
@@ -182,7 +182,7 @@ public class QueryReqConverter {
return result;
}
public void correctTableName(QueryDslReq databaseReq) {
public void correctTableName(QueryS2QLReq databaseReq) {
String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(), TABLE_PREFIX + databaseReq.getModelId());
databaseReq.setSql(sql);
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
@@ -42,11 +42,11 @@ public class QueryController {
@PostMapping("/sql")
public Object queryBySql(@RequestBody QueryDslReq queryDslReq,
public Object queryBySql(@RequestBody QueryS2QLReq queryS2QLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
Object queryBySql = queryService.queryBySql(queryDslReq, user);
Object queryBySql = queryService.queryBySql(queryS2QLReq, user);
log.info("queryBySql:{},queryBySql");
return queryBySql;
}
@@ -124,9 +124,9 @@ public class QueryController {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
if (QueryTypeEnum.SQL.equals(queryTypeEnum)) {
QueryDslReq queryDslReq = JsonUtil.toObject(queryReqJson, QueryDslReq.class);
ExplainSqlReq<QueryDslReq> explainSqlReqNew = ExplainSqlReq.<QueryDslReq>builder()
.queryReq(queryDslReq)
QueryS2QLReq queryS2QLReq = JsonUtil.toObject(queryReqJson, QueryS2QLReq.class);
ExplainSqlReq<QueryS2QLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2QLReq>builder()
.queryReq(queryS2QLReq)
.queryTypeEnum(queryTypeEnum).build();
return queryService.explain(explainSqlReqNew, user);
}

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
@@ -18,7 +18,7 @@ import java.util.List;
public interface QueryService {
Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception;
Object queryBySql(QueryS2QLReq querySqlCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;

14
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -24,11 +24,11 @@ import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.MetricReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.utils.DslPermissionAnnotation;
import com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation;
import com.tencent.supersonic.semantic.query.executor.QueryExecutor;
import com.tencent.supersonic.semantic.query.parser.convert.QueryReqConverter;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
@@ -89,9 +89,9 @@ public class QueryServiceImpl implements QueryService {
}
@Override
@DslPermissionAnnotation
@S2QLPermissionAnnotation
@SneakyThrows
public Object queryBySql(QueryDslReq querySqlCmd, User user) {
public Object queryBySql(QueryS2QLReq querySqlCmd, User user) {
statUtils.initStatInfo(querySqlCmd, user);
QueryStatement queryStatement = new QueryStatement();
try {
@@ -109,7 +109,7 @@ public class QueryServiceImpl implements QueryService {
return semanticQueryEngine.execute(queryStatement);
}
private QueryStatement convertToQueryStatement(QueryDslReq querySqlCmd, User user) throws Exception {
private QueryStatement convertToQueryStatement(QueryS2QLReq querySqlCmd, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = new ArrayList<>();
modelIds.add(querySqlCmd.getModelId());
@@ -294,8 +294,8 @@ public class QueryServiceImpl implements QueryService {
QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
T queryReq = explainSqlReq.getQueryReq();
if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryDslReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryDslReq) queryReq, user);
if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2QLReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryS2QLReq) queryReq, user);
return getExplainResp(queryStatement);
}
if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.pojo.Filter;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import java.util.ArrayList;
@@ -45,9 +45,9 @@ public class DimValueAspect {
return queryResultWithColumns;
}
Object[] args = joinPoint.getArgs();
QueryDslReq queryDslReq = (QueryDslReq) args[0];
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) args[0];
List<DimensionResp> dimensions = dimensionService.getDimensions(queryDslReq.getModelId());
List<DimensionResp> dimensions = dimensionService.getDimensions(queryS2QLReq.getModelId());
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -24,7 +24,7 @@ import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
@@ -188,8 +188,8 @@ public class QueryStructUtils {
return resNameEnSet;
}
public Set<String> getResName(QueryDslReq queryDslReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryDslReq.getSql())
public Set<String> getResName(QueryS2QLReq queryS2QLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql())
.stream().collect(Collectors.toSet());
return resNameSet;
}
@@ -199,11 +199,11 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getResNameEnExceptInternalCol(QueryDslReq queryDslReq, User user) {
Set<String> resNameSet = getResName(queryDslReq);
public Set<String> getResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq, User user) {
Set<String> resNameSet = getResName(queryS2QLReq);
Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(queryDslReq.getModelId());
List<Long> modelIds = Lists.newArrayList(queryS2QLReq.getModelId());
filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -234,8 +234,8 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getFilterResNameEnExceptInternalCol(QueryDslReq queryDslReq) {
String sql = queryDslReq.getSql();
public Set<String> getFilterResNameEnExceptInternalCol(QueryS2QLReq queryS2QLReq) {
String sql = queryS2QLReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}

46
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslDataAspect.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.query.service.AuthCommonService;
@@ -41,7 +41,7 @@ import org.springframework.util.CollectionUtils;
@Aspect
@Order(1)
@Slf4j
public class DslDataAspect {
public class S2QLDataAspect {
@Autowired
private QueryStructUtils queryStructUtils;
@@ -54,24 +54,24 @@ public class DslDataAspect {
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.DslPermissionAnnotation)")
private void dslPermissionCheck() {
@Pointcut("@annotation(com.tencent.supersonic.semantic.query.utils.S2QLPermissionAnnotation)")
private void s2QLPermissionCheck() {
}
@Around("dslPermissionCheck()")
@Around("s2QLPermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
log.info("dsl permission check!");
log.info("s2ql permission check!");
Object[] objects = joinPoint.getArgs();
QueryDslReq queryDslReq = (QueryDslReq) objects[0];
QueryS2QLReq queryS2QLReq = (QueryS2QLReq) objects[0];
User user = (User) objects[1];
if (!permissionDataEnable) {
log.info("not to check dsl permission!");
log.info("not to check s2QL permission!");
return joinPoint.proceed();
}
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
Long modelId = queryDslReq.getModelId();
Long modelId = queryS2QLReq.getModelId();
//1. determine whether admin of the model
if (authCommonService.doModelAdmin(user, modelId)) {
@@ -82,7 +82,7 @@ public class DslDataAspect {
authCommonService.doModelVisible(user, modelId);
// 3. fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryDslReq, user);
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2QLReq, user);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId);
@@ -97,10 +97,10 @@ public class DslDataAspect {
Set<String> resAuthSet = authCommonService.getAuthResNameSet(authorizedResource, modelId);
// 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryDslReq, resAuthSet, sensitiveResReq);
doFilterCheckLogic(queryS2QLReq, resAuthSet, sensitiveResReq);
// 5.row permission pre-filter
doRowPermission(queryDslReq, authorizedResource);
doRowPermission(queryS2QLReq, authorizedResource);
// 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
@@ -123,7 +123,7 @@ public class DslDataAspect {
return queryResultAfterDesensitization;
}
private void doRowPermission(QueryDslReq queryDslReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(QueryS2QLReq queryS2QLReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -143,12 +143,12 @@ public class DslDataAspect {
}
});
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner.toString() + " ) ");
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(queryDslReq.getSql(), expression);
log.info("before doRowPermission, queryDslReq:{}", queryDslReq.getSql());
queryDslReq.setSql(sql);
log.info("after doRowPermission, queryDslReq:{}", queryDslReq.getSql());
String sql = SqlParserAddHelper.addWhere(queryS2QLReq.getSql(), expression);
log.info("before doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql());
queryS2QLReq.setSql(sql);
log.info("after doRowPermission, queryS2QLReq:{}", queryS2QLReq.getSql());
}
} catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
@@ -156,22 +156,22 @@ public class DslDataAspect {
}
private void doFilterCheckLogic(QueryDslReq queryDslReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryDslReq);
private void doFilterCheckLogic(QueryS2QLReq queryS2QLReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2QLReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = new ArrayList<>();
modelIds.add(queryDslReq.getModelId());
modelIds.add(queryS2QLReq.getModelId());
List<ModelResp> modelInfos = modelService.getModelList(modelIds);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryDslReq.getModelId());
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(queryS2QLReq.getModelId());
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslPermissionAnnotation.java → semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLPermissionAnnotation.java
View File

@@ -9,6 +9,6 @@ import java.lang.annotation.Documented;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface DslPermissionAnnotation {
public @interface S2QLPermissionAnnotation {
}

16
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java
View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.model.domain.ModelService;
@@ -87,11 +87,11 @@ public class StatUtils {
}
public void initStatInfo(QueryDslReq queryDslReq, User facadeUser) {
public void initStatInfo(QueryS2QLReq queryS2QLReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(queryDslReq.getSql());
queryStatInfo.setModelId(queryDslReq.getModelId());
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryDslReq.getModelId());
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2QLReq.getSql());
queryStatInfo.setModelId(queryS2QLReq.getModelId());
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2QLReq.getModelId());
List<String> dimensions = new ArrayList<>();
if (Objects.nonNull(modelSchemaResp)) {
@@ -106,12 +106,12 @@ public class StatUtils {
String userName = getUserName(facadeUser);
try {
queryStatInfo.setTraceId("")
.setModelId(queryDslReq.getModelId())
.setModelId(queryS2QLReq.getModelId())
.setUser(userName)
.setQueryType(QueryTypeEnum.SQL.getValue())
.setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())
.setQuerySqlCmd(queryDslReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryDslReq.toString()))
.setQuerySqlCmd(queryS2QLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2QLReq.toString()))
.setStartTime(System.currentTimeMillis())
.setUseResultCache(true)
.setUseSqlCache(true)