diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java index 9261c7072..9f6f35b98 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java @@ -20,5 +20,8 @@ public class CacheCommonConfig { @Value("${cache.common.expire.after.write:10}") private Integer cacheCommonExpireAfterWrite; + @Value("${query.cache.enable:true}") + private Boolean cacheEnable; + } \ No newline at end of file diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java index 21c96520c..35fd89ce5 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java @@ -1,26 +1,21 @@ package com.tencent.supersonic.headless.core.cache; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component @Slf4j public class DefaultQueryCache implements QueryCache { - @Value("${query.cache.enable:true}") - private Boolean cacheEnable; - @Autowired - private CacheManager cacheManager; - public Object query(SemanticQueryReq semanticQueryReq, String cacheKey) { + CacheManager cacheManager = ContextUtils.getBean(CacheManager.class); if (isCache(semanticQueryReq)) { Object result = cacheManager.get(cacheKey); log.info("query from cache, key:{}", cacheKey); @@ -30,7 +25,9 @@ public class DefaultQueryCache implements QueryCache { } public Boolean put(String cacheKey, Object value) { - if (cacheEnable && Objects.nonNull(value)) { + CacheManager cacheManager = ContextUtils.getBean(CacheManager.class); + CacheCommonConfig cacheCommonConfig = ContextUtils.getBean(CacheCommonConfig.class); + if (cacheCommonConfig.getCacheEnable() && Objects.nonNull(value)) { CompletableFuture.supplyAsync(() -> cacheManager.put(cacheKey, value)) .exceptionally(exception -> { log.warn("exception:", exception); @@ -43,6 +40,7 @@ public class DefaultQueryCache implements QueryCache { } public String getCacheKey(SemanticQueryReq semanticQueryReq) { + CacheManager cacheManager = ContextUtils.getBean(CacheManager.class); String commandMd5 = semanticQueryReq.generateCommandMd5(); String keyByModelIds = getKeyByModelIds(semanticQueryReq.getModelIds()); return cacheManager.generateCacheKey(keyByModelIds, commandMd5); @@ -53,7 +51,8 @@ public class DefaultQueryCache implements QueryCache { } private boolean isCache(SemanticQueryReq semanticQueryReq) { - if (!cacheEnable) { + CacheCommonConfig cacheCommonConfig = ContextUtils.getBean(CacheCommonConfig.class); + if (!cacheCommonConfig.getCacheEnable()) { return false; } if (semanticQueryReq.getCacheInfo() != null) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java index 3f1127d38..4519643d5 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.core.executor; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; @@ -11,13 +12,6 @@ import org.springframework.stereotype.Component; @Component("JdbcExecutor") @Slf4j public class JdbcExecutor implements QueryExecutor { - - private final SqlUtils sqlUtils; - - public JdbcExecutor(SqlUtils sqlUtils) { - this.sqlUtils = sqlUtils; - } - @Override public boolean accept(QueryStatement queryStatement) { return true; @@ -25,6 +19,7 @@ public class JdbcExecutor implements QueryExecutor { @Override public SemanticQueryResp execute(QueryStatement queryStatement) { + SqlUtils sqlUtils = ContextUtils.getBean(SqlUtils.class); if (Strings.isEmpty(queryStatement.getSourceId())) { log.warn("data base id is empty"); return null; @@ -32,8 +27,8 @@ public class JdbcExecutor implements QueryExecutor { log.info("query SQL: {}", queryStatement.getSql()); Database database = queryStatement.getSemanticModel().getDatabase(); SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); - SqlUtils sqlUtils = this.sqlUtils.init(database); - sqlUtils.queryInternal(queryStatement.getSql(), queryResultWithColumns); + SqlUtils sqlUtil = sqlUtils.init(database); + sqlUtil.queryInternal(queryStatement.getSql(), queryResultWithColumns); queryResultWithColumns.setSql(queryStatement.getSql()); return queryResultWithColumns; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java index 49521ceb0..68d7d9b77 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java @@ -9,19 +9,18 @@ import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.EngineType; +import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; /** * supplement the QueryStatement when query with custom aggregation method @@ -30,12 +29,6 @@ import java.util.stream.Collectors; @Slf4j public class CalculateAggConverter implements HeadlessConverter { - private final SqlGenerateUtils sqlGenerateUtils; - - - public CalculateAggConverter(SqlGenerateUtils sqlGenerateUtils) { - this.sqlGenerateUtils = sqlGenerateUtils; - } public interface EngineSql { @@ -43,8 +36,9 @@ public class CalculateAggConverter implements HeadlessConverter { } public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement, - EngineType engineTypeEnum, String version) + EngineType engineTypeEnum, String version) throws Exception { + SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); QueryParam queryParam = queryStatement.getQueryParam(); // 同环比 if (isRatioAccept(queryParam)) { @@ -130,8 +124,9 @@ public class CalculateAggConverter implements HeadlessConverter { } public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum, - String version) + String version) throws Exception { + SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); QueryParam queryParam = queryStatement.getQueryParam(); check(queryParam); queryStatement.setEnableOptimize(false); @@ -412,6 +407,7 @@ public class CalculateAggConverter implements HeadlessConverter { } private String getSelectField(final Aggregator agg, String alias) { + SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); if (agg.getFunc().equals(AggOperatorEnum.RATIO_OVER) || agg.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { return alias + agg.getColumn(); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java index 8a77bf6a0..85bea7b19 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java @@ -1,20 +1,20 @@ package com.tencent.supersonic.headless.core.parser.converter; import com.tencent.supersonic.common.pojo.ColumnOrder; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - /** * HeadlessConverter default implement */ @@ -22,26 +22,18 @@ import java.util.stream.Collectors; @Slf4j public class ParserDefaultConverter implements HeadlessConverter { - private final SqlGenerateUtils sqlGenerateUtils; - - private final CalculateAggConverter calculateConverterAgg; - - public ParserDefaultConverter(CalculateAggConverter calculateConverterAgg, - SqlGenerateUtils sqlGenerateUtils) { - this.calculateConverterAgg = calculateConverterAgg; - this.sqlGenerateUtils = sqlGenerateUtils; - } - @Override public boolean accept(QueryStatement queryStatement) { if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) { return false; } + CalculateAggConverter calculateConverterAgg = ContextUtils.getBean(CalculateAggConverter.class); return !calculateConverterAgg.accept(queryStatement); } @Override public void convert(QueryStatement queryStatement) throws Exception { + SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); QueryParam queryParam = queryStatement.getQueryParam(); MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam(); MetricQueryParam metricReq = generateSqlCommand(queryStatement.getQueryParam(), queryStatement); @@ -50,6 +42,7 @@ public class ParserDefaultConverter implements HeadlessConverter { } public MetricQueryParam generateSqlCommand(QueryParam queryParam, QueryStatement queryStatement) { + SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); MetricQueryParam metricQueryParam = new MetricQueryParam(); metricQueryParam.setMetrics(queryParam.getMetrics()); metricQueryParam.setDimensions(queryParam.getGroups()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index 618ced7e3..29095e717 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -1,30 +1,23 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.chat.parser.JavaLLMProxy; import com.tencent.supersonic.headless.core.chat.parser.LLMProxy; import com.tencent.supersonic.headless.core.chat.parser.llm.DataSetResolver; -import com.tencent.supersonic.headless.core.executor.JdbcExecutor; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.parser.SqlParser; -import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser; -import com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter; -import com.tencent.supersonic.headless.core.parser.converter.DefaultDimValueConverter; import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter; -import com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter; -import com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter; -import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer; import com.tencent.supersonic.headless.core.planner.QueryOptimizer; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.core.io.support.SpringFactoriesLoader; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.io.support.SpringFactoriesLoader; /** * HeadlessConverter QueryOptimizer QueryExecutor object factory @@ -36,6 +29,7 @@ public class ComponentFactory { private static Map queryOptimizers = new HashMap<>(); private static List queryExecutors = new ArrayList<>(); private static SqlParser sqlParser; + private static QueryCache queryCache; private static LLMProxy llmProxy; private static DataSetResolver modelResolver; @@ -69,11 +63,18 @@ public class ComponentFactory { public static SqlParser getSqlParser() { if (sqlParser == null) { - sqlParser = ContextUtils.getContext().getBean("CalciteSqlParser", CalciteSqlParser.class); + initQueryParser(); } return sqlParser; } + public static QueryCache getQueryCache() { + if (queryCache == null) { + initQueryCache(); + } + return queryCache; + } + public static void setSqlParser(SqlParser parser) { sqlParser = parser; } @@ -82,23 +83,29 @@ public class ComponentFactory { queryOptimizers.put(name, queryOptimizer); } - public static T getBean(String name, Class tClass) { - return ContextUtils.getContext().getBean(name, tClass); - } - private static void initQueryOptimizer() { - queryOptimizers.put("DetailQueryOptimizer", getBean("DetailQueryOptimizer", DetailQueryOptimizer.class)); + List queryOptimizerList = new ArrayList<>(); + init(QueryOptimizer.class, queryOptimizerList); + if (!queryOptimizerList.isEmpty()) { + queryOptimizerList.stream().forEach(q -> addQueryOptimizer(q.getClass().getSimpleName(), q)); + } } private static void initQueryExecutors() { - queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", JdbcExecutor.class)); + //queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", JdbcExecutor.class)); + init(QueryExecutor.class, queryExecutors); } private static void initSemanticConverter() { - headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class)); - headlessConverters.add(getBean("SqlVariableParseConverter", SqlVariableParseConverter.class)); - headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class)); - headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class)); + init(HeadlessConverter.class, headlessConverters); + } + + private static void initQueryParser() { + sqlParser = init(SqlParser.class); + } + + private static void initQueryCache() { + queryCache = init(QueryCache.class); } public static LLMProxy getLLMProxy() { @@ -127,6 +134,10 @@ public class ComponentFactory { return modelResolver; } + public static T getBean(String name, Class tClass) { + return ContextUtils.getContext().getBean(name, tClass); + } + private static List init(Class factoryType, List list) { list.addAll(SpringFactoriesLoader.loadFactories(factoryType, Thread.currentThread().getContextClassLoader())); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index 4b8f0605c..9c260cd2a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -41,6 +41,7 @@ import com.tencent.supersonic.headless.core.parser.QueryParser; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.planner.QueryPlanner; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.server.annotation.S2DataPermission; import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; @@ -75,7 +76,6 @@ public class QueryServiceImpl implements QueryService { private final TagConverter tagConverter; private final Catalog catalog; private final AppService appService; - private final QueryCache queryCache; private final SemanticSchemaManager semanticSchemaManager; private final QueryParser queryParser; private final QueryPlanner queryPlanner; @@ -86,7 +86,6 @@ public class QueryServiceImpl implements QueryService { QueryReqConverter queryReqConverter, TagConverter tagConverter, Catalog catalog, AppService appService, - QueryCache queryCache, SemanticSchemaManager semanticSchemaManager, DefaultQueryParser queryParser, QueryPlanner queryPlanner) { @@ -96,7 +95,6 @@ public class QueryServiceImpl implements QueryService { this.tagConverter = tagConverter; this.catalog = catalog; this.appService = appService; - this.queryCache = queryCache; this.semanticSchemaManager = semanticSchemaManager; this.queryParser = queryParser; this.queryPlanner = queryPlanner; @@ -112,6 +110,7 @@ public class QueryServiceImpl implements QueryService { //1.initStatInfo statUtils.initStatInfo(queryReq, user); //2.query from cache + QueryCache queryCache = ComponentFactory.getQueryCache(); String cacheKey = queryCache.getCacheKey(queryReq); Object query = queryCache.query(queryReq, cacheKey); if (Objects.nonNull(query)) { diff --git a/launchers/headless/src/main/resources/META-INF/spring.factories b/launchers/headless/src/main/resources/META-INF/spring.factories index 79b69ddda..076b21d9e 100644 --- a/launchers/headless/src/main/resources/META-INF/spring.factories +++ b/launchers/headless/src/main/resources/META-INF/spring.factories @@ -5,4 +5,23 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\ - com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore \ No newline at end of file + com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore + + +com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter=\ + com.tencent.supersonic.headless.core.parser.converter.DefaultDimValueConverter,\ + com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter,\ + com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter,\ + com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter + +com.tencent.supersonic.headless.core.planner.QueryOptimizer=\ + com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer + +com.tencent.supersonic.headless.core.executor.QueryExecutor=\ + com.tencent.supersonic.headless.core.executor.JdbcExecutor + +com.tencent.supersonic.headless.core.parser.SqlParser=\ + com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser + +com.tencent.supersonic.headless.core.cache.QueryCache=\ + com.tencent.supersonic.headless.core.cache.DefaultQueryCache \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 3f781587a..b37edb18b 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -14,6 +14,24 @@ com.tencent.supersonic.headless.core.chat.corrector.SemanticCorrector=\ com.tencent.supersonic.headless.core.chat.corrector.TimeCorrector, \ com.tencent.supersonic.headless.core.chat.corrector.GrammarCorrector +com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter=\ + com.tencent.supersonic.headless.core.parser.converter.DefaultDimValueConverter,\ + com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter,\ + com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter,\ + com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter + +com.tencent.supersonic.headless.core.planner.QueryOptimizer=\ + com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer + +com.tencent.supersonic.headless.core.executor.QueryExecutor=\ + com.tencent.supersonic.headless.core.executor.JdbcExecutor + +com.tencent.supersonic.headless.core.parser.SqlParser=\ + com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser + +com.tencent.supersonic.headless.core.cache.QueryCache=\ + com.tencent.supersonic.headless.core.cache.DefaultQueryCache + com.tencent.supersonic.headless.server.processor.ResultProcessor=\ com.tencent.supersonic.headless.server.processor.ParseInfoProcessor, \ com.tencent.supersonic.headless.server.processor.QueryRankProcessor, \