[improvement](Headless) Optimize the issue of integrating the Headless knowledge dictionary. (#731)

2025-12-11 03:58:14 +00:00 · 2024-02-20 20:36:57 +08:00
parent 33240cc382
commit d10801ef38
12 changed files with 136 additions and 59 deletions
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java
@@ -0,0 +1,21 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import java.util.List;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class QueryMetricReq {
+
+    private Long domainId;
+
+    private List<Long> metricIds;
+
+    private List<String> metricNames;
+
+    private List<Long> dimensionIds;
+
+    private List<String> dimensionNames;
+
+}
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetaFilter.java
@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.server.pojo;

+import com.google.common.base.Objects;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import java.util.List;
@@ -36,4 +37,29 @@ public class MetaFilter {
    public MetaFilter(List<Long> modelIds) {
        this.modelIds = modelIds;
    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        MetaFilter that = (MetaFilter) o;
+        return Objects.equal(id, that.id) && Objects.equal(name, that.name)
+                && Objects.equal(bizName, that.bizName) && Objects.equal(
+                createdBy, that.createdBy) && Objects.equal(modelIds, that.modelIds)
+                && Objects.equal(domainId, that.domainId) && Objects.equal(
+                viewId, that.viewId) && Objects.equal(sensitiveLevel, that.sensitiveLevel)
+                && Objects.equal(status, that.status) && Objects.equal(key,
+                that.key) && Objects.equal(ids, that.ids) && Objects.equal(
+                fieldsDepend, that.fieldsDepend);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(id, name, bizName, createdBy, modelIds, domainId, viewId, sensitiveLevel, status, key,
+                ids, fieldsDepend);
+    }
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricApiController.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/MetricApiController.java
@@ -0,0 +1,32 @@
+package com.tencent.supersonic.headless.server.rest;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
+import com.tencent.supersonic.headless.server.service.QueryService;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/api/semantic/query")
+@Slf4j
+public class MetricApiController {
+
+    @Autowired
+    private QueryService queryService;
+
+    @PostMapping("/metric")
+    public Object queryBySql(@RequestBody QueryMetricReq queryMetricReq,
+            HttpServletRequest request,
+            HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        return queryService.queryByMetric(queryMetricReq, user);
+    }
+
+}
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
@@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
 import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
 import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
@@ -27,4 +28,6 @@ public interface QueryService {
    @ApiHeaderCheck
    ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception;

+    SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception;
+
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ViewService.java
@@ -6,6 +6,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;

 import java.util.List;
+import java.util.Map;

 public interface ViewService {

@@ -19,7 +20,7 @@ public interface ViewService {

    void delete(Long id, User user);

-    List<ViewResp> getViewListByCache(MetaFilter metaFilter);
+    Map<Long, List<Long>> getModelIdToViewIds(List<Long> viewIds);

    List<ViewResp> getViews(User user);

--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java
@@ -2,23 +2,19 @@ package com.tencent.supersonic.headless.server.service.impl;

 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.headless.api.pojo.response.S2Term;
-import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
 import com.tencent.supersonic.headless.core.knowledge.DictWord;
 import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
 import com.tencent.supersonic.headless.core.knowledge.SearchService;
 import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
-import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.service.KnowledgeService;
 import com.tencent.supersonic.headless.server.service.ViewService;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
-
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;

@Service
@Slf4j
@@ -77,7 +73,8 @@ public class KnowledgeServiceImpl implements KnowledgeService {

    @Override
    public List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> viewIds) {
-        return prefixSearch(key, limit, viewIds2ModelIdList(viewIds));
+        Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds));
+        return prefixSearchByModel(key, limit, modelIdToViewIds.keySet());
    }

    public List<HanlpMapResult> prefixSearchByModel(String key, int limit, Set<Long> models) {
@@ -86,22 +83,12 @@ public class KnowledgeServiceImpl implements KnowledgeService {

    @Override
    public List<HanlpMapResult> suffixSearch(String key, int limit, Set<Long> viewIds) {
-        return suffixSearch(key, limit, viewIds2ModelIdList(viewIds));
+        Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds));
+        return suffixSearchByModel(key, limit, modelIdToViewIds.keySet());
    }

    public List<HanlpMapResult> suffixSearchByModel(String key, int limit, Set<Long> models) {
        return SearchService.suffixSearch(key, limit, models);
    }

-    private Set<Long> viewIds2ModelIdList(Set<Long> viewIds) {
-        Set<Long> modelIds = new HashSet<>();
-        MetaFilter filter = new MetaFilter();
-        filter.setIds(new ArrayList<>(viewIds));
-        List<ViewResp> viewList = viewService.getViewList(filter);
-        if (CollectionUtils.isEmpty(viewList)) {
-            return modelIds;
-        }
-        viewList.stream().forEach(view -> modelIds.addAll(view.getAllModels()));
-        return modelIds;
-    }
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaEmbeddingServiceImpl.java
@@ -2,14 +2,11 @@ package com.tencent.supersonic.headless.server.service.impl;

 import com.tencent.supersonic.common.config.EmbeddingConfig;
 import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.util.ComponentFactory;
 import com.tencent.supersonic.common.util.embedding.Retrieval;
 import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
 import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
 import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
-import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
-import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.service.MetaEmbeddingService;
 import com.tencent.supersonic.headless.server.service.ViewService;
 import java.util.ArrayList;
@@ -22,7 +19,6 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -41,16 +37,8 @@ public class MetaEmbeddingServiceImpl implements MetaEmbeddingService {
    @Override
    public List<RetrieveQueryResult> retrieveQuery(List<Long> viewIds, RetrieveQuery retrieveQuery, int num) {
        // viewIds->modelIds
-        MetaFilter metaFilter = new MetaFilter();
-        metaFilter.setStatus(StatusEnum.ONLINE.getCode());
-        metaFilter.setIds(viewIds);
-        List<ViewResp> viewListByCache = viewService.getViewListByCache(metaFilter);
-        Set<Long> allModels = getModels(viewListByCache);
-
-        Map<Long, List<Long>> modelIdToViewIds = viewListByCache.stream()
-                .flatMap(viewResp -> viewResp.getAllModels().stream()
-                        .map(modelId -> Pair.of(modelId, viewResp.getId())))
-                .collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList())));
+        Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(viewIds);
+        Set<Long> allModels = modelIdToViewIds.keySet();

        if (CollectionUtils.isNotEmpty(allModels) && allModels.size() == 1) {
            Map<String, String> filterCondition = new HashMap<>();
@@ -105,10 +93,4 @@ public class MetaEmbeddingServiceImpl implements MetaEmbeddingService {
                .filter(retrieveQueryResult -> CollectionUtils.isNotEmpty(retrieveQueryResult.getRetrieval()))
                .collect(Collectors.toList());
    }
-
-    private Set<Long> getModels(List<ViewResp> viewListByCache) {
-        return viewListByCache.stream()
-                .flatMap(viewResp -> viewResp.getAllModels().stream())
-                .collect(Collectors.toSet());
-    }
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
@@ -18,6 +18,7 @@ import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
@@ -65,6 +66,7 @@ import org.springframework.stereotype.Service;
@Service
@Slf4j
 public class QueryServiceImpl implements QueryService {
+
    private StatUtils statUtils;
    private final QueryUtils queryUtils;
    private final QueryReqConverter queryReqConverter;
@@ -167,7 +169,6 @@ public class QueryServiceImpl implements QueryService {
        QueryStatement queryStatement = new QueryStatement();
        QueryParam queryParam = new QueryParam();
        queryReqConverter.convert(queryStructReq, queryParam);
-        //queryStatement.setQueryStructReq(queryStructReq);
        queryStatement.setQueryParam(queryParam);
        queryStatement.setIsS2SQL(false);
        queryStatement.setEnableOptimize(queryUtils.enableOptimize());
@@ -239,6 +240,11 @@ public class QueryServiceImpl implements QueryService {
        return ItemQueryResultResp.builder().results(results).build();
    }

+    @Override
+    public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception {
+        return null;
+    }
+
    private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception {
        MetricResp metricResp = catalog.getMetric(item.getId());
        item.setCreatedBy(metricResp.getCreatedBy());
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java
@@ -26,10 +26,12 @@ import java.util.Comparator;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
@@ -159,13 +161,18 @@ public class ViewServiceImpl
    }

    @Override
-    public List<ViewResp> getViewListByCache(MetaFilter metaFilter) {
+    public Map<Long, List<Long>> getModelIdToViewIds(List<Long> viewIds) {
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setStatus(StatusEnum.ONLINE.getCode());
+        metaFilter.setIds(viewIds);
        List<ViewResp> viewList = viewSchemaCache.getIfPresent(metaFilter);
        if (CollectionUtils.isEmpty(viewList)) {
            viewList = getViewList(metaFilter);
            viewSchemaCache.put(metaFilter, viewList);
        }
-        return viewList;
+        return viewList.stream()
+                .flatMap(
+                        viewResp -> viewResp.getAllModels().stream().map(modelId -> Pair.of(modelId, viewResp.getId())))
+                .collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList())));
    }
-
 }