topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][Headless] Add unit test cases and remove unnecessary logs. (#698)

Browse Source
This commit is contained in:
lexluo09
2024-01-30 16:46:06 +08:00
committed by GitHub
parent fdfad515dd
commit 954c67c947
33 changed files with 301 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java
View File

@@ -25,7 +25,7 @@ public abstract class BaseMapper implements SchemaMapper {
String simpleName = this.getClass().getSimpleName();
long startTime = System.currentTimeMillis();
log.info("before {},mapInfo:{}", simpleName, queryContext.getMapInfo().getModelElementMatches());
log.debug("before {},mapInfo:{}", simpleName, queryContext.getMapInfo().getModelElementMatches());
try {
doMap(queryContext);
@@ -34,7 +34,7 @@ public abstract class BaseMapper implements SchemaMapper {
}
long cost = System.currentTimeMillis() - startTime;
log.info("after {},cost:{},mapInfo:{}", simpleName, cost, queryContext.getMapInfo().getModelElementMatches());
log.debug("after {},cost:{},mapInfo:{}", simpleName, cost, queryContext.getMapInfo().getModelElementMatches());
}
public abstract void doMap(QueryContext queryContext);

6
chat/core/src/main/java/com/tencent/supersonic/chat/core/pojo/QueryContext.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.chat.core.pojo;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo;
@@ -37,10 +38,15 @@ public class QueryContext {
private List<SemanticQuery> candidateQueries = new ArrayList<>();
private SchemaMapInfo mapInfo = new SchemaMapInfo();
private SchemaModelClusterMapInfo modelClusterMapInfo = new SchemaModelClusterMapInfo();
@JsonIgnore
private SemanticSchema semanticSchema;
@JsonIgnore
private Agent agent;
@JsonIgnore
private Map<Long, ChatConfigRichResp> modelIdToChatRichConfig;
@JsonIgnore
private Map<String, Plugin> nameToPlugin;
@JsonIgnore
private List<Plugin> pluginList;
public List<SemanticQuery> getCandidateQueries() {

1
chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/ChatContextRepositoryImpl.java
View File

@@ -51,7 +51,6 @@ public class ChatContextRepositoryImpl implements ChatContextRepository {
chatContext.setUser(contextDO.getUser());
chatContext.setQueryText(contextDO.getQueryText());
if (contextDO.getSemanticParse() != null && !contextDO.getSemanticParse().isEmpty()) {
log.info("--->: {}", contextDO.getSemanticParse());
SemanticParseInfo semanticParseInfo = JsonUtil.toObject(contextDO.getSemanticParse(),
SemanticParseInfo.class);
chatContext.setParseInfo(semanticParseInfo);

4
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java
View File

@@ -153,7 +153,7 @@ public class QueryServiceImpl implements QueryService {
parser.parse(queryCtx, chatCtx);
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(parser.getClass().getSimpleName()).type(CostType.PARSER.getType()).build());
log.info("{} result:{}", parser.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
log.debug("{} result:{}", parser.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
});
// 3. corrector
@@ -177,7 +177,7 @@ public class QueryServiceImpl implements QueryService {
timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
.interfaceName(processor.getClass().getSimpleName())
.type(CostType.PROCESSOR.getType()).build());
log.info("{} result:{}", processor.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
log.debug("{} result:{}", processor.getClass().getSimpleName(), JsonUtil.toString(queryCtx));
});
if (Objects.nonNull(parseResult.getQueryId()) && timeCostDOList.size() > 0) {

9
common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java
View File

@@ -48,10 +48,10 @@ public class SysParameter {
//detect config
parameters.add(new Parameter("one.detection.size", "8",
"hanlp一次探测返回结果个数", "在每次探测后, 将前后缀匹配的结果合并, 并根据相似度阈值过滤后的结果个数",
"一次探测返回结果个数", "在每次探测后, 将前后缀匹配的结果合并, 并根据相似度阈值过滤后的结果个数",
"number", "Mapper相关配置"));
parameters.add(new Parameter("one.detection.max.size", "20",
"hanlp一次探测前后缀匹配结果返回个数", "单次前后缀匹配返回的结果个数", "number", "Mapper相关配置"));
"一次探测前后缀匹配结果返回个数", "单次前后缀匹配返回的结果个数", "number", "Mapper相关配置"));
//mapper config
parameters.add(new Parameter("metric.dimension.threshold", "0.3",
@@ -77,7 +77,7 @@ public class SysParameter {
parameters.add(new Parameter("embedding.mapper.distance.threshold",
"0.58", "向量召回相似度阈值", "相似度大于该阈值的则舍弃", "number", "Mapper相关配置"));
//llm config
//parser config
Parameter s2SQLParameter = new Parameter("s2SQL.generation", "TWO_PASS_AUTO_COT",
"S2SQL生成方式", "ONE_PASS_AUTO_COT: 通过思维链方式一步生成sql"
+ "\nONE_PASS_AUTO_COT_SELF_CONSISTENCY: 通过思维链且投票方式一步生成sql"
@@ -90,7 +90,6 @@ public class SysParameter {
parameters.add(new Parameter("s2SQL.linking.value.switch", "true",
"是否将Mapper探测识别到的维度值提供给大模型", "为了数据安全考虑, 这里可进行开关选择",
"bool", "Parser相关配置"));
parameters.add(new Parameter("query.text.length.threshold", "10",
"用户输入文本长短阈值", "文本超过该阈值为长文本", "number", "Parser相关配置"));
parameters.add(new Parameter("short.text.threshold", "0.5",
@@ -99,8 +98,6 @@ public class SysParameter {
parameters.add(new Parameter("long.text.threshold", "0.8",
"长文本匹配阈值", "如果是长文本, 若query得分/文本长度>该阈值, 则跳过当前parser",
"number", "Parser相关配置"));
//parse config
parameters.add(new Parameter("parse.show.count", "3",
"解析结果个数", "前端展示的解析个数",
"number", "Parser相关配置"));

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheManager.java vendored
View File

@@ -22,7 +22,7 @@ public class CaffeineCacheManager implements CacheManager {
@Override
public Boolean put(String key, Object value) {
log.info("[put caffeineCache] key:{}, value:{}", key, value);
log.debug("[put caffeineCache] key:{}, value:{}", key, value);
caffeineCache.put(key, value);
return true;
}
@@ -30,7 +30,7 @@ public class CaffeineCacheManager implements CacheManager {
@Override
public Object get(String key) {
Object value = caffeineCache.asMap().get(key);
log.info("[get caffeineCache] key:{}, value:{}", key, value);
log.debug("[get caffeineCache] key:{}, value:{}", key, value);
return value;
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java vendored
View File

@@ -23,7 +23,7 @@ public class DefaultQueryCache implements QueryCache {
public Object query(SemanticQueryReq semanticQueryReq, String cacheKey) {
if (isCache(semanticQueryReq)) {
Object result = cacheManager.get(cacheKey);
log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq);
log.info("query from cache, key:{}", cacheKey);
return result;
}
return null;
@@ -36,7 +36,7 @@ public class DefaultQueryCache implements QueryCache {
log.warn("exception:", exception);
return null;
});
log.info("add record to cache, key:{}", cacheKey);
log.info("put to cache, key:{}", cacheKey);
return true;
}
return false;

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java
View File

@@ -31,7 +31,6 @@ public class JdbcExecutor implements QueryExecutor {
}
log.info("query SQL: {}", queryStatement.getSql());
Database database = queryStatement.getSemanticModel().getDatabase();
log.info("database info:{}", database);
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
SqlUtils sqlUtils = this.sqlUtils.init(database);
sqlUtils.queryInternal(queryStatement.getSql(), queryResultWithColumns);

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/CalciteSqlParser.java
View File

@@ -42,7 +42,7 @@ public class CalciteSqlParser implements SqlParser {
getSqlByView(aggBuilder.getSql(engineType), queryStatement.getViewSql(),
queryStatement.getViewAlias()), engineType);
if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) {
log.info("simplifySql [{}]", simplifySql);
log.debug("simplifySql [{}]", simplifySql);
queryStatement.setViewSimplifySql(simplifySql);
}
}

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/planner/AggPlanner.java
View File

@@ -160,7 +160,6 @@ public class AggPlanner implements Planner {
private SqlNode optimizeSql(String sql, EngineType engineType) {
try {
log.info("before optimize:[{}]", sql);
SqlNode sqlNode = SqlParser.create(sql, Configuration.getParserConfig(engineType)).parseStmt();
if (Objects.nonNull(sqlNode)) {
return SemanticNode.optimize(scope, schema, sqlNode, engineType);

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java
View File

@@ -106,7 +106,6 @@ public class SqlUtils {
JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
jdbcTemplate.setDatabaseProductName(database.getName());
jdbcTemplate.setFetchSize(500);
log.info("jdbcTemplate:{}, dataSource:{}", jdbcTemplate, dataSource);
return jdbcTemplate;
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -221,7 +221,7 @@ public class QueryServiceImpl implements QueryService {
@Override
public <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception {
T queryReq = explainSqlReq.getQueryReq();
QueryStatement queryStatement = buildQueryStatement((QuerySqlReq) queryReq, user);
QueryStatement queryStatement = buildQueryStatement((SemanticQueryReq) queryReq, user);
queryStatement = plan(queryStatement);
return getExplainResp(queryStatement);
}
@@ -332,14 +332,12 @@ public class QueryServiceImpl implements QueryService {
private QueryStatement plan(QueryStatement queryStatement) throws Exception {
queryParser.parse(queryStatement);
log.info("queryStatement:{}", queryStatement);
queryPlanner.plan(queryStatement);
return queryStatement;
}
private SemanticQueryResp query(QueryStatement queryStatement) throws Exception {
SemanticQueryResp semanticQueryResp = null;
log.info("[QueryStatement:{}]", queryStatement);
try {
//1 parse
queryParser.parse(queryStatement);

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -188,10 +188,8 @@ public class QueryReqConverter {
}
String type = database.getType();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase());
log.info("type:{},engineAdaptor:{}", type, engineAdaptor);
if (Objects.nonNull(engineAdaptor)) {
String functionNameCorrector = engineAdaptor.functionNameCorrector(databaseReq.getSql());
log.info("sql:{} ,after corrector", databaseReq.getSql(), functionNameCorrector);
databaseReq.setSql(functionNameCorrector);
}
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -71,7 +71,6 @@ public class StatUtils {
QueryStat queryStatInfo = get();
queryStatInfo.setElapsedMs(System.currentTimeMillis() - queryStatInfo.getStartTime());
queryStatInfo.setQueryState(state.getStatus());
log.info("queryStatInfo: {}", queryStatInfo);
CompletableFuture.runAsync(() -> {
statRepository.createRecord(queryStatInfo);
}).exceptionally(exception -> {
@@ -186,16 +185,7 @@ public class StatUtils {
: "Admin";
}
public Boolean updateQueryOptMode(String mode) {
STATS.get().setQueryOptMode(mode);
return true;
}
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend) {
return statRepository.getStatInfo(itemUseCommend);
}
public List<QueryStat> getQueryStatInfoWithoutCache(ItemUseReq itemUseCommend) {
return statRepository.getQueryStatInfoWithoutCache(itemUseCommend);
}
}

4
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/BaseTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration;
package com.tencent.supersonic.chat;
import static org.junit.Assert.assertEquals;
@@ -11,7 +11,7 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
import com.tencent.supersonic.chat.integration.util.DataUtils;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.chat.server.service.ChatService;
import com.tencent.supersonic.chat.server.service.ConfigService;

2
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/MetricInterpretTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricInterpretTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration;
package com.tencent.supersonic.chat;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.chat.core.query.llm.analytics.LLMAnswerResp;

4
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/MetricTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration;
package com.tencent.supersonic.chat;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
@@ -11,7 +11,7 @@ import com.tencent.supersonic.chat.core.query.rule.metric.MetricFilterQuery;
import com.tencent.supersonic.chat.core.query.rule.metric.MetricGroupByQuery;
import com.tencent.supersonic.chat.core.query.rule.metric.MetricModelQuery;
import com.tencent.supersonic.chat.core.query.rule.metric.MetricTopNQuery;
import com.tencent.supersonic.chat.integration.util.DataUtils;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;

4
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/MockConfiguration.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.chat.integration;
package com.tencent.supersonic.chat;
import static org.mockito.Mockito.when;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.integration.util.DataUtils;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.chat.core.plugin.PluginManager;
import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.common.config.EmbeddingConfig;

4
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/MultiTurnsTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/MultiTurnsTest.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.chat.integration;
package com.tencent.supersonic.chat;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.core.query.rule.metric.MetricFilterQuery;
import com.tencent.supersonic.chat.integration.util.DataUtils;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;

4
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/TagTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration;
package com.tencent.supersonic.chat;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.core.query.rule.metric.MetricTagQuery;
import com.tencent.supersonic.chat.core.query.rule.tag.TagFilterQuery;
import com.tencent.supersonic.chat.integration.util.DataUtils;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;

6
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/mapper/MapperTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration.mapper;
package com.tencent.supersonic.chat.mapper;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
@@ -8,8 +8,8 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.core.query.rule.metric.MetricTagQuery;
import com.tencent.supersonic.chat.integration.BaseTest;
import com.tencent.supersonic.chat.integration.util.DataUtils;
import com.tencent.supersonic.chat.BaseTest;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;

2
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/model/MetricServiceImplTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/model/MetricServiceImplTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration.model;
package com.tencent.supersonic.chat.model;
import com.google.common.collect.Lists;
import com.tencent.supersonic.BaseApplication;

2
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/plugin/BasePluginTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/BasePluginTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration.plugin;
package com.tencent.supersonic.chat.plugin;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;

6
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/plugin/PluginRecognizeTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/chat/plugin/PluginRecognizeTest.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration.plugin;
package com.tencent.supersonic.chat.plugin;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
@@ -7,8 +7,8 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.core.plugin.PluginManager;
import com.tencent.supersonic.chat.integration.MockConfiguration;
import com.tencent.supersonic.chat.integration.util.DataUtils;
import com.tencent.supersonic.chat.MockConfiguration;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.chat.server.service.QueryService;
import com.tencent.supersonic.common.config.EmbeddingConfig;

75
launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java Normal file
View File

@@ -0,0 +1,75 @@
package com.tencent.supersonic.headless;
import static java.time.LocalDate.now;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.util.DataUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
public class BaseTest extends BaseApplication {
@Autowired
protected QueryService queryService;
protected SemanticQueryResp queryBySql(String sql) throws Exception {
return queryBySql(sql, User.getFakeUser());
}
protected SemanticQueryResp queryBySql(String sql, User user) throws Exception {
return queryService.queryByReq(buildQuerySqlReq(sql), user);
}
protected SemanticQueryReq buildQuerySqlReq(String sql) {
QuerySqlReq querySqlCmd = new QuerySqlReq();
querySqlCmd.setSql(sql);
querySqlCmd.setModelIds(DataUtils.getMetricAgentIModelIds());
return querySqlCmd;
}
protected QueryStructReq buildQueryStructReq(List<String> groups) {
QueryStructReq queryStructReq = new QueryStructReq();
for (Long modelId : DataUtils.getMetricAgentIModelIds()) {
queryStructReq.addModelId(modelId);
}
queryStructReq.setQueryType(QueryType.METRIC);
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("pv");
queryStructReq.setAggregators(Arrays.asList(aggregator));
if (CollectionUtils.isNotEmpty(groups)) {
queryStructReq.setGroups(groups);
queryStructReq.setGroups(Arrays.asList("department"));
}
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateMode.BETWEEN);
dateConf.setEndDate(now().plusDays(0).toString());
dateConf.setStartDate(now().plusDays(-365).toString());
queryStructReq.setDateInfo(dateConf);
List<Order> orders = new ArrayList<>();
Order order = new Order();
order.setColumn("pv");
orders.add(order);
queryStructReq.setOrders(orders);
return queryStructReq;
}
}

47
launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java Normal file
View File

@@ -0,0 +1,47 @@
package com.tencent.supersonic.headless;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
import com.tencent.supersonic.headless.api.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.util.DataUtils;
import java.util.Arrays;
import org.junit.jupiter.api.Test;
public class ExplainTest extends BaseTest {
@Test
public void testSqlExplain() throws Exception {
String sql = "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 ";
ExplainSqlReq<QuerySqlReq> explainSqlReq = ExplainSqlReq.<QuerySqlReq>builder()
.queryTypeEnum(QueryType.SQL)
.queryReq(QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentIModelIds()))
.build();
ExplainResp explain = queryService.explain(explainSqlReq, User.getFakeUser());
assertNotNull(explain);
assertNotNull(explain.getSql());
assertTrue(explain.getSql().contains("department"));
assertTrue(explain.getSql().contains("pv"));
}
@Test
public void testStructExplain() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"));
ExplainSqlReq<QueryStructReq> explainSqlReq = ExplainSqlReq.<QueryStructReq>builder()
.queryTypeEnum(QueryType.STRUCT)
.queryReq(queryStructReq)
.build();
ExplainResp explain = queryService.explain(explainSqlReq, User.getFakeUser());
assertNotNull(explain);
assertNotNull(explain.getSql());
assertTrue(explain.getSql().contains("department"));
assertTrue(explain.getSql().contains("pv"));
}
}

30
launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/QueryBySqlTest.java → launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.integration;
package com.tencent.supersonic.headless;
import static java.time.LocalDate.now;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
@@ -32,6 +33,31 @@ public class QueryBySqlTest extends BaseTest {
assertEquals(4, result.getResultList().size());
}
@Test
public void testFilterQuery() throws Exception {
SemanticQueryResp result = queryBySql(
"SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 WHERE 部门 ='HR' GROUP BY 部门 ");
assertEquals(2, result.getColumns().size());
QueryColumn firstColumn = result.getColumns().get(0);
QueryColumn secondColumn = result.getColumns().get(1);
assertEquals("部门", firstColumn.getName());
assertEquals("访问次数", secondColumn.getName());
assertEquals(1, result.getResultList().size());
assertEquals("HR", result.getResultList().get(0).get("department").toString());
}
@Test
public void testDateSumQuery() throws Exception {
String startDate = now().plusDays(-365).toString();
String endDate = now().plusDays(0).toString();
String sql = "SELECT SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 WHERE 数据日期 >= '%s' AND 数据日期 <= '%s' ";
SemanticQueryResp semanticQueryResp = queryBySql(String.format(sql, startDate, endDate));
assertEquals(1, semanticQueryResp.getColumns().size());
QueryColumn queryColumn = semanticQueryResp.getColumns().get(0);
assertEquals("访问次数", queryColumn.getName());
assertEquals(1, semanticQueryResp.getResultList().size());
}
@Test
public void testCacheQuery() throws Exception {
SemanticQueryResp result1 = queryBySql("SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 ");
@@ -50,7 +76,7 @@ public class QueryBySqlTest extends BaseTest {
}
@Test
public void testAuthorization() throws Exception {
public void testAuthorization() {
User alice = new User(2L, "alice", "alice", "alice@email", 0);
assertThrows(InvalidPermissionException.class,
() -> queryBySql("SELECT SUM(pv) FROM 超音数PVUV统计 WHERE department ='HR'", alice));

81
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java Normal file
View File

@@ -0,0 +1,81 @@
package com.tencent.supersonic.headless;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThrows;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.Test;
public class QueryByStructTest extends BaseTest {
@Test
public void testSumQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(null);
SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryStructReq, User.getFakeUser());
assertEquals(1, semanticQueryResp.getColumns().size());
QueryColumn queryColumn = semanticQueryResp.getColumns().get(0);
assertEquals("访问次数", queryColumn.getName());
assertEquals(1, semanticQueryResp.getResultList().size());
}
@Test
public void testGroupByQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"));
SemanticQueryResp result = queryService.queryByReq(queryStructReq, User.getFakeUser());
assertEquals(2, result.getColumns().size());
QueryColumn firstColumn = result.getColumns().get(0);
QueryColumn secondColumn = result.getColumns().get(1);
assertEquals("部门", firstColumn.getName());
assertEquals("访问次数", secondColumn.getName());
assertNotNull(result.getResultList().size());
}
@Test
public void testFilterQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"));
List<Filter> dimensionFilters = new ArrayList<>();
Filter filter = new Filter();
filter.setName("部门");
filter.setBizName("department");
filter.setOperator(FilterOperatorEnum.EQUALS);
filter.setValue("HR");
dimensionFilters.add(filter);
queryStructReq.setDimensionFilters(dimensionFilters);
SemanticQueryResp result = queryService.queryByReq(queryStructReq, User.getFakeUser());
assertEquals(2, result.getColumns().size());
QueryColumn firstColumn = result.getColumns().get(0);
QueryColumn secondColumn = result.getColumns().get(1);
assertEquals("部门", firstColumn.getName());
assertEquals("访问次数", secondColumn.getName());
assertEquals(1, result.getResultList().size());
assertEquals("HR", result.getResultList().get(0).get("department").toString());
}
@Test
public void testCacheQuery() throws Exception {
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"));
QueryStructReq queryStructReq2 = buildQueryStructReq(Arrays.asList("department"));
SemanticQueryResp result1 = queryService.queryByReq(queryStructReq1, User.getFakeUser());
SemanticQueryResp result2 = queryService.queryByReq(queryStructReq2, User.getFakeUser());
assertEquals(result1, result2);
}
@Test
public void testAuthorization() {
User alice = new User(2L, "alice", "alice", "alice@email", 0);
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"));
assertThrows(InvalidPermissionException.class,
() -> queryService.queryByReq(queryStructReq1, alice));
}
}

22
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryDimensionTest.java Normal file
View File

@@ -0,0 +1,22 @@
package com.tencent.supersonic.headless;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
public class QueryDimensionTest extends BaseTest {
@Test
public void testQueryDimValue() {
QueryDimValueReq queryDimValueReq = new QueryDimValueReq();
queryDimValueReq.setModelId(1L);
queryDimValueReq.setDimensionBizName("department");
SemanticQueryResp queryResp = queryService.queryDimValue(queryDimValueReq, User.getFakeUser());
Assert.assertNotNull(queryResp.getResultList());
Assert.assertEquals(4, queryResp.getResultList().size());
}
}

41
launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/BaseTest.java
View File

@@ -1,41 +0,0 @@
package com.tencent.supersonic.headless.integration;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.QueryService;
import java.util.HashSet;
import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired;
public class BaseTest extends BaseApplication {
@Autowired
private QueryService queryService;
protected SemanticQueryResp queryBySql(String sql) throws Exception {
return queryBySql(sql, User.getFakeUser());
}
protected SemanticQueryResp queryBySql(String sql, User user) throws Exception {
return queryService.queryByReq(buildQuerySqlReq(sql), user);
}
protected SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception {
return queryService.queryByReq(queryReq, user);
}
protected SemanticQueryReq buildQuerySqlReq(String sql) {
QuerySqlReq querySqlCmd = new QuerySqlReq();
querySqlCmd.setSql(sql);
Set<Long> modelIds = new HashSet<>();
modelIds.add(1L);
modelIds.add(2L);
modelIds.add(3L);
querySqlCmd.setModelIds(modelIds);
return querySqlCmd;
}
}

11
launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/ExplainTest.java
View File

@@ -1,11 +0,0 @@
package com.tencent.supersonic.headless.integration;
import org.junit.jupiter.api.Test;
public class ExplainTest extends BaseTest {
@Test
public void testSumExplain() {
}
}

86
launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/QueryByStructTest.java
View File

@@ -1,86 +0,0 @@
package com.tencent.supersonic.headless.integration;
import static java.time.LocalDate.now;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.junit.jupiter.api.Test;
public class QueryByStructTest extends BaseTest {
@Test
public void testSumQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(null);
SemanticQueryResp semanticQueryResp = queryByReq(queryStructReq, User.getFakeUser());
assertEquals(1, semanticQueryResp.getColumns().size());
QueryColumn queryColumn = semanticQueryResp.getColumns().get(0);
assertEquals("访问次数", queryColumn.getName());
assertEquals(1, semanticQueryResp.getResultList().size());
}
@Test
public void testGroupByQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"));
SemanticQueryResp result = queryByReq(queryStructReq, User.getFakeUser());
assertEquals(2, result.getColumns().size());
QueryColumn firstColumn = result.getColumns().get(0);
QueryColumn secondColumn = result.getColumns().get(1);
assertEquals("部门", firstColumn.getName());
assertEquals("访问次数", secondColumn.getName());
assertNotNull(result.getResultList().size());
}
@Test
public void testCacheQuery() throws Exception {
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"));
QueryStructReq queryStructReq2 = buildQueryStructReq(Arrays.asList("department"));
SemanticQueryResp result1 = queryByReq(queryStructReq1, User.getFakeUser());
SemanticQueryResp result2 = queryByReq(queryStructReq2, User.getFakeUser());
assertEquals(result1, result2);
}
private QueryStructReq buildQueryStructReq(List<String> groups) {
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.addModelId(1L);
queryStructReq.addModelId(2L);
queryStructReq.addModelId(3L);
queryStructReq.setQueryType(QueryType.METRIC);
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("pv");
queryStructReq.setAggregators(Arrays.asList(aggregator));
if (CollectionUtils.isNotEmpty(groups)) {
queryStructReq.setGroups(groups);
queryStructReq.setGroups(Arrays.asList("department"));
}
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateMode.BETWEEN);
dateConf.setStartDate(now().plusDays(-1).toString());
dateConf.setEndDate(now().plusDays(-10).toString());
queryStructReq.setDateInfo(dateConf);
List<Order> orders = new ArrayList<>();
Order order = new Order();
order.setColumn("pv");
orders.add(order);
queryStructReq.setOrders(orders);
return queryStructReq;
}
}

11
launchers/standalone/src/test/java/com/tencent/supersonic/chat/integration/util/DataUtils.java → launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.integration.util;
package com.tencent.supersonic.util;
import static java.time.LocalDate.now;
@@ -15,6 +15,8 @@ import com.tencent.supersonic.chat.core.agent.PluginTool;
import com.tencent.supersonic.chat.core.agent.RuleParserTool;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import java.util.HashSet;
import java.util.Set;
public class DataUtils {
@@ -128,4 +130,11 @@ public class DataUtils {
return pluginTool;
}
public static Set<Long> getMetricAgentIModelIds() {
Set<Long> result = new HashSet<>();
result.add(1L);
result.add(2L);
result.add(3L);
return result;
}
}