topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-18 16:27:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)Refactor headless to abstract SemanticTranslator explicitly.

Browse Source
This commit is contained in:
jerryjzhang
2024-06-21 20:20:22 +08:00
parent 5af76fa7dc
commit e293be3ebf
104 changed files with 469 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -25,21 +25,21 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\
### headless-core SPIs
com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter=\
com.tencent.supersonic.headless.core.parser.converter.DefaultDimValueConverter,\
com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter,\
com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter,\
com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter
com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\
com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlVariableParseConverter,\
com.tencent.supersonic.headless.core.translator.converter.CalculateAggConverter,\
com.tencent.supersonic.headless.core.translator.converter.ParserDefaultConverter
com.tencent.supersonic.headless.core.planner.QueryOptimizer=\
com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer
com.tencent.supersonic.headless.core.translator.QueryOptimizer=\
com.tencent.supersonic.headless.core.translator.DetailQueryOptimizer
com.tencent.supersonic.headless.core.translator.QueryParser=\
com.tencent.supersonic.headless.core.translator.calcite.CalciteQueryParser
com.tencent.supersonic.headless.core.executor.QueryExecutor=\
com.tencent.supersonic.headless.core.executor.JdbcExecutor
com.tencent.supersonic.headless.core.parser.SqlParser=\
com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser
com.tencent.supersonic.headless.core.cache.QueryCache=\
com.tencent.supersonic.headless.core.cache.DefaultQueryCache

3
launchers/standalone/src/test/java/com/tencent/supersonic/evaluation/Text2SQLEval.java
View File

@@ -84,7 +84,7 @@ public class Text2SQLEval extends BaseTest {
AgentConfig agentConfig = new AgentConfig();
agentConfig.getTools().add(getLLMQueryTool());
agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
agent.setLlmConfig(getLLMConfig(LLMType.GPT));
agent.setLlmConfig(getLLMConfig(LLMType.DEEPSEEK));
MultiTurnConfig multiTurnConfig = new MultiTurnConfig();
multiTurnConfig.setEnableMultiTurn(enableMultiturn);
agent.setMultiTurnConfig(multiTurnConfig);
@@ -146,4 +146,5 @@ public class Text2SQLEval extends BaseTest {
return new LLMConfig("open_ai",
baseUrl, apiKey, modelName, temperature);
}
}

6
launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java
View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SemanticLayerService;
import com.tencent.supersonic.util.DataUtils;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -26,14 +26,14 @@ import static java.time.LocalDate.now;
public class BaseTest extends BaseApplication {
@Autowired
protected QueryService queryService;
protected SemanticLayerService semanticLayerService;
protected SemanticQueryResp queryBySql(String sql) throws Exception {
return queryBySql(sql, User.getFakeUser());
}
protected SemanticQueryResp queryBySql(String sql, User user) throws Exception {
return queryService.queryByReq(buildQuerySqlReq(sql), user);
return semanticLayerService.queryByReq(buildQuerySqlReq(sql), user);
}
protected SemanticQueryReq buildQuerySqlReq(String sql) {

4
launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java
View File

@@ -25,7 +25,7 @@ public class ExplainTest extends BaseTest {
.queryTypeEnum(QueryMethod.SQL)
.queryReq(QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentView()))
.build();
ExplainResp explain = queryService.explain(explainSqlReq, User.getFakeUser());
ExplainResp explain = semanticLayerService.explain(explainSqlReq, User.getFakeUser());
assertNotNull(explain);
assertNotNull(explain.getSql());
assertTrue(explain.getSql().contains("department"));
@@ -39,7 +39,7 @@ public class ExplainTest extends BaseTest {
.queryTypeEnum(QueryMethod.STRUCT)
.queryReq(queryStructReq)
.build();
ExplainResp explain = queryService.explain(explainSqlReq, User.getFakeUser());
ExplainResp explain = semanticLayerService.explain(explainSqlReq, User.getFakeUser());
assertNotNull(explain);
assertNotNull(explain.getSql());
assertTrue(explain.getSql().contains("department"));

2
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java
View File

@@ -68,6 +68,6 @@ public class QueryByMetricTest extends BaseTest {
private SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception {
QueryStructReq convert = metricService.convert(queryMetricReq);
return queryService.queryByReq(convert.convert(), user);
return semanticLayerService.queryByReq(convert.convert(), user);
}
}

14
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java
View File

@@ -46,7 +46,7 @@ public class QueryByStructTest extends BaseTest {
public void testDetailQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("user_name", "department"),
QueryType.DETAIL);
SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryStructReq, User.getFakeUser());
SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq, User.getFakeUser());
assertEquals(3, semanticQueryResp.getColumns().size());
QueryColumn firstColumn = semanticQueryResp.getColumns().get(0);
assertEquals("用户", firstColumn.getName());
@@ -60,7 +60,7 @@ public class QueryByStructTest extends BaseTest {
@Test
public void testSumQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(null);
SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryStructReq, User.getFakeUser());
SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq, User.getFakeUser());
assertEquals(1, semanticQueryResp.getColumns().size());
QueryColumn queryColumn = semanticQueryResp.getColumns().get(0);
assertEquals("访问次数", queryColumn.getName());
@@ -70,7 +70,7 @@ public class QueryByStructTest extends BaseTest {
@Test
public void testGroupByQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"));
SemanticQueryResp result = queryService.queryByReq(queryStructReq, User.getFakeUser());
SemanticQueryResp result = semanticLayerService.queryByReq(queryStructReq, User.getFakeUser());
assertEquals(2, result.getColumns().size());
QueryColumn firstColumn = result.getColumns().get(0);
QueryColumn secondColumn = result.getColumns().get(1);
@@ -91,7 +91,7 @@ public class QueryByStructTest extends BaseTest {
dimensionFilters.add(filter);
queryStructReq.setDimensionFilters(dimensionFilters);
SemanticQueryResp result = queryService.queryByReq(queryStructReq, User.getFakeUser());
SemanticQueryResp result = semanticLayerService.queryByReq(queryStructReq, User.getFakeUser());
assertEquals(2, result.getColumns().size());
QueryColumn firstColumn = result.getColumns().get(0);
QueryColumn secondColumn = result.getColumns().get(1);
@@ -106,7 +106,7 @@ public class QueryByStructTest extends BaseTest {
User alice = new User(2L, "alice", "alice", "alice@email", 0);
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"));
assertThrows(InvalidPermissionException.class,
() -> queryService.queryByReq(queryStructReq1, alice));
() -> semanticLayerService.queryByReq(queryStructReq1, alice));
}
@Test
@@ -116,7 +116,7 @@ public class QueryByStructTest extends BaseTest {
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("stay_hours");
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator);
SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryStructReq1, tom);
SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom);
Assertions.assertEquals(false, semanticQueryResp.getColumns().get(1).getAuthorized());
Assertions.assertEquals("******", semanticQueryResp.getResultList().get(0).get("stay_hours"));
}
@@ -128,7 +128,7 @@ public class QueryByStructTest extends BaseTest {
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("stay_hours");
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator);
SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryStructReq1, tom);
SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom);
Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage());
Assertions.assertTrue(semanticQueryResp.getSql().contains("`user_name` = 'tom'"));
}

2
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryDimensionTest.java
View File

@@ -14,7 +14,7 @@ public class QueryDimensionTest extends BaseTest {
queryDimValueReq.setModelId(1L);
queryDimValueReq.setDimensionBizName("department");
SemanticQueryResp queryResp = queryService.queryDimValue(queryDimValueReq, User.getFakeUser());
SemanticQueryResp queryResp = semanticLayerService.queryDimValue(queryDimValueReq, User.getFakeUser());
Assert.assertNotNull(queryResp.getResultList());
Assert.assertEquals(4, queryResp.getResultList().size());
}

90
launchers/standalone/src/test/resources/META-INF/spring.factories
View File

@@ -1,90 +0,0 @@
### headless-chat SPIs
com.tencent.supersonic.headless.chat.mapper.SchemaMapper=\
com.tencent.supersonic.headless.chat.mapper.EmbeddingMapper, \
com.tencent.supersonic.headless.chat.mapper.KeywordMapper, \
com.tencent.supersonic.headless.chat.mapper.QueryFilterMapper, \
com.tencent.supersonic.headless.chat.mapper.EntityMapper
com.tencent.supersonic.headless.chat.parser.SemanticParser=\
com.tencent.supersonic.headless.chat.parser.rule.RuleSqlParser, \
com.tencent.supersonic.headless.chat.parser.llm.LLMSqlParser, \
com.tencent.supersonic.headless.chat.parser.QueryTypeParser
com.tencent.supersonic.headless.chat.corrector.SemanticCorrector=\
com.tencent.supersonic.headless.chat.corrector.SchemaCorrector, \
com.tencent.supersonic.headless.chat.corrector.TimeCorrector, \
com.tencent.supersonic.headless.chat.corrector.GrammarCorrector
com.tencent.supersonic.headless.chat.knowledge.file.FileHandler=\
com.tencent.supersonic.headless.chat.knowledge.file.FileHandlerImpl
com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\
com.tencent.supersonic.headless.chat.parser.llm.HeuristicDataSetResolver
### headless-core SPIs
com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter=\
com.tencent.supersonic.headless.core.parser.converter.DefaultDimValueConverter,\
com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter,\
com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter,\
com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter
com.tencent.supersonic.headless.core.planner.QueryOptimizer=\
com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer
com.tencent.supersonic.headless.core.executor.QueryExecutor=\
com.tencent.supersonic.headless.core.executor.JdbcExecutor
com.tencent.supersonic.headless.core.parser.SqlParser=\
com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser
com.tencent.supersonic.headless.core.cache.QueryCache=\
com.tencent.supersonic.headless.core.cache.DefaultQueryCache
### headless-server SPIs
com.tencent.supersonic.headless.server.processor.ResultProcessor=\
com.tencent.supersonic.headless.server.processor.ParseInfoProcessor, \
com.tencent.supersonic.headless.server.processor.SqlInfoProcessor
### chat-server SPIs
com.tencent.supersonic.chat.server.parser.ChatParser=\
com.tencent.supersonic.chat.server.parser.NL2PluginParser, \
com.tencent.supersonic.chat.server.parser.MultiTurnParser,\
com.tencent.supersonic.chat.server.parser.NL2SQLParser
com.tencent.supersonic.chat.server.executor.ChatExecutor=\
com.tencent.supersonic.chat.server.executor.PluginExecutor, \
com.tencent.supersonic.chat.server.executor.SqlExecutor
com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer=\
com.tencent.supersonic.chat.server.plugin.recognize.embedding.EmbeddingRecallRecognizer
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.parse.EntityInfoProcessor,\
com.tencent.supersonic.chat.server.processor.parse.TimeCostProcessor
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\
com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor
### auth-authentication SPIs
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor
com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor
### common SPIs
dev.langchain4j.store.embedding.S2EmbeddingStore=\
dev.langchain4j.store.embedding.InMemoryS2EmbeddingStore