From bc69d2221a2e4b74bc7219df99c3f0baf6d0b7f9 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Thu, 14 Sep 2023 14:01:51 +0800 Subject: [PATCH] (improvement)(semantic) support dimension value query in SemanticLayer (#90) --- assembly/bin/supersonic-daemon.bat | 2 +- .../chat/api/component/SemanticLayer.java | 2 + .../semantic/DefaultSemanticConfig.java | 3 ++ .../semantic/LocalSemanticLayer.java | 15 ++++++-- .../semantic/RemoteSemanticLayer.java | 9 +++++ .../api/query/request/QueryDimValueReq.java | 14 +++++++ .../semantic/query/rest/QueryController.java | 10 +++++ .../semantic/query/service/QueryService.java | 3 ++ .../query/service/QueryServiceImpl.java | 38 +++++++++++++++++++ 9 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryDimValueReq.java diff --git a/assembly/bin/supersonic-daemon.bat b/assembly/bin/supersonic-daemon.bat index 8639d0f4b..3d806ee95 100644 --- a/assembly/bin/supersonic-daemon.bat +++ b/assembly/bin/supersonic-daemon.bat @@ -16,7 +16,7 @@ set "python_path=python" set "pip_path=pip3" set "llm_host=127.0.0.1" set "llm_port=9092" -set "start_name=supersonic-llmparser" +set "start_name=api_service" if "%module%"=="" ( diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java index 54a9a06ab..dbd0226ee 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java @@ -11,6 +11,7 @@ import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; @@ -33,6 +34,7 @@ public interface SemanticLayer { QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user); QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user); + QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); List getModelSchema(); List getModelSchema(List ids); ModelSchema getModelSchema(Long model, Boolean cacheEnable); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java index b8df82afa..a28b71682 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java @@ -20,6 +20,9 @@ public class DefaultSemanticConfig { @Value("${searchByStruct.path:/api/semantic/query/sql}") private String searchBySqlPath; + @Value("${searchByStruct.path:/api/semantic/query/queryDimValue}") + private String queryDimValuePath; + @Value("${fetchModelSchemaPath.path:/api/semantic/schema}") private String fetchModelSchemaPath; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java index d11d05017..d0dceaaa5 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java @@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; @@ -33,18 +34,19 @@ public class LocalSemanticLayer extends BaseSemanticLayer { private ModelService modelService; private DimensionService dimensionService; private MetricService metricService; + private QueryService queryService; @SneakyThrows @Override public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) { - QueryService queryService = ContextUtils.getBean(QueryService.class); + queryService = ContextUtils.getBean(QueryService.class); return queryService.queryByStructWithAuth(queryStructReq, user); } @Override public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { try { - QueryService queryService = ContextUtils.getBean(QueryService.class); + queryService = ContextUtils.getBean(QueryService.class); return queryService.queryByMultiStruct(queryMultiStructReq, user); } catch (Exception e) { log.info("queryByMultiStruct has an exception:{}", e); @@ -55,7 +57,7 @@ public class LocalSemanticLayer extends BaseSemanticLayer { @Override public QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user) { try { - QueryService queryService = ContextUtils.getBean(QueryService.class); + queryService = ContextUtils.getBean(QueryService.class); Object object = queryService.queryBySql(queryDslReq, user); QueryResultWithSchemaResp queryResultWithSchemaResp = JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp.class); @@ -66,6 +68,13 @@ public class LocalSemanticLayer extends BaseSemanticLayer { return null; } + @Override + @SneakyThrows + public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { + queryService = ContextUtils.getBean(QueryService.class); + return queryService.queryDimValue(queryDimValueReq, user); + } + @Override public List doFetchModelSchema(List ids) { ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java index 1319d0e81..05630263d 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java @@ -20,6 +20,7 @@ import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; @@ -113,6 +114,14 @@ public class RemoteSemanticLayer extends BaseSemanticLayer { throw new CommonException(responseBody.getCode(), responseBody.getMsg()); } + @Override + public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { + DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); + return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + + defaultSemanticConfig.getQueryDimValuePath(), + new Gson().toJson(queryDimValueReq)); + } + @Override public List doFetchModelSchema(List ids) { HttpHeaders headers = new HttpHeaders(); diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryDimValueReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryDimValueReq.java new file mode 100644 index 000000000..b4e4d74bc --- /dev/null +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryDimValueReq.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.semantic.api.query.request; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class QueryDimValueReq { + + private Long modelId; + private String dimensionBizName; + private String value; + +} diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java index 2a124c05f..ef77abbe5 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java @@ -2,7 +2,9 @@ package com.tencent.supersonic.semantic.query.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.SqlParserResp; +import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; @@ -85,4 +87,12 @@ public class QueryController { return queryService.getStatInfo(itemUseReq); } + @PostMapping("/queryDimValue") + public QueryResultWithSchemaResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return queryService.queryDimValue(queryDimValueReq, user); + } + } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java index 97da0f761..28b04ed95 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.query.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; +import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; @@ -20,6 +21,8 @@ public interface QueryService { QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception; + QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); + List getStatInfo(ItemUseReq itemUseCommend); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index 62e45b554..1b94f9651 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -1,14 +1,19 @@ package com.tencent.supersonic.semantic.query.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Cache; +import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; +import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; @@ -19,6 +24,7 @@ import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.utils.QueryUtils; import com.tencent.supersonic.semantic.query.utils.StatUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import lombok.SneakyThrows; @@ -155,6 +161,13 @@ public class QueryServiceImpl implements QueryService { } } + @Override + @SneakyThrows + public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { + QueryStructReq queryStructReq = generateDimValueQueryStruct(queryDimValueReq); + return queryByStruct(queryStructReq, user); + } + private void handleGlobalCacheDisable(QueryStructReq queryStructCmd) { if (!cacheEnable) { @@ -202,5 +215,30 @@ public class QueryServiceImpl implements QueryService { return null; } + private QueryStructReq generateDimValueQueryStruct(QueryDimValueReq queryDimValueReq) { + QueryStructReq queryStructReq = new QueryStructReq(); + + queryStructReq.setModelId(queryDimValueReq.getModelId()); + queryStructReq.setGroups(Collections.singletonList(queryDimValueReq.getDimensionBizName())); + + if (!Objects.isNull(queryDimValueReq.getValue())) { + List dimensionFilters = new ArrayList<>(); + Filter dimensionFilter = new Filter(); + dimensionFilter.setOperator(FilterOperatorEnum.LIKE); + dimensionFilter.setRelation(Filter.Relation.FILTER); + dimensionFilter.setBizName(queryDimValueReq.getDimensionBizName()); + dimensionFilter.setValue(queryDimValueReq.getValue()); + dimensionFilters.add(dimensionFilter); + queryStructReq.setDimensionFilters(dimensionFilters); + } + List aggregators = new ArrayList<>(); + queryStructReq.setAggregators(aggregators); + + DateConf dateInfo = new DateConf(); + dateInfo.setDateMode(DateConf.DateMode.RECENT); + dateInfo.setUnit(1); + queryStructReq.setDateInfo(dateInfo); + return queryStructReq; + } }