(improvement)(headless) transfer term nature modelId to viewId before providing it to chat and put the modelId of metadata into the dict word instead of viewId (#739)

Co-authored-by: jolunoluo
2025-12-15 06:27:21 +00:00 · 2024-02-23 10:29:21 +08:00
parent 16643e8d75
commit e95a528219
52 changed files with 456 additions and 245 deletions
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/MetricDrillDownChecker.java
@@ -11,11 +11,13 @@ import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.server.service.MetricService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.aspectj.lang.ProceedingJoinPoint;
 import org.aspectj.lang.annotation.Around;
 import org.aspectj.lang.annotation.Aspect;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 import java.util.Collection;
@@ -28,6 +30,9 @@ import java.util.stream.Collectors;
@Slf4j
 public class MetricDrillDownChecker {

+    @Autowired
+    private MetricService metricService;
+
    @Around("execution(* com.tencent.supersonic.headless.core.parser.QueryParser.parse(..))")
    public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
        Object[] objects = joinPoint.getArgs();
@@ -52,7 +57,7 @@ public class MetricDrillDownChecker {
            List<DimensionResp> necessaryDimensions = getNecessaryDimensions(metric, semanticSchemaResp);
            List<DimensionResp> dimensionsMissing = getNecessaryDimensionMissing(necessaryDimensions, dimensionFields);
            if (!CollectionUtils.isEmpty(dimensionsMissing)) {
-                String errMsg = String.format("指标:%s 缺失必要维度:%s", metric.getName(),
+                String errMsg = String.format("指标:%s 缺失必要下钻维度:%s", metric.getName(),
                        dimensionsMissing.stream().map(DimensionResp::getName).collect(Collectors.toList()));
                throw new InvalidArgumentException(errMsg);
            }
@@ -92,8 +97,9 @@ public class MetricDrillDownChecker {
            return true;
        }
        List<String> relateDimensions = metricResps.stream()
-                .filter(metric -> !CollectionUtils.isEmpty(metric.getDrillDownDimensions()))
-                .map(metric -> metric.getDrillDownDimensions().stream()
+                .map(this::getDrillDownDimensions)
+                .filter(drillDownDimensions -> !CollectionUtils.isEmpty(drillDownDimensions))
+                .map(drillDownDimensions -> drillDownDimensions.stream()
                        .map(DrillDownDimension::getDimensionId).collect(Collectors.toList()))
                .flatMap(Collection::stream)
                .map(id -> convertDimensionIdToBizName(id, semanticSchemaResp))
@@ -111,7 +117,7 @@ public class MetricDrillDownChecker {
        if (metric == null) {
            return Lists.newArrayList();
        }
-        List<DrillDownDimension> drillDownDimensions = metric.getDrillDownDimensions();
+        List<DrillDownDimension> drillDownDimensions = getDrillDownDimensions(metric);
        if (CollectionUtils.isEmpty(drillDownDimensions)) {
            return Lists.newArrayList();
        }
@@ -147,4 +153,8 @@ public class MetricDrillDownChecker {
        return dimension.getBizName();
    }

+    private List<DrillDownDimension> getDrillDownDimensions(MetricResp metricResp) {
+        return metricService.getDrillDownDimension(metricResp.getId());
+    }
+
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
@@ -30,7 +30,7 @@ import java.util.stream.Collectors;
@Slf4j
 public class ModelYamlManager {

-    public static DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) {
+    public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) {
        ModelDetail modelDetail = modelResp.getModelDetail();
        DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
        SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
@@ -109,8 +109,9 @@ public class DownloadServiceImpl implements DownloadService {
        metaFilter.setIds(metricIds);
        List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
        Map<String, List<MetricResp>> metricMap = getMetricMap(metricResps);
-        List<Long> dimensionIds = metricResps.stream().map(MetricResp::getRelateDimension)
-                .map(RelateDimension::getDrillDownDimensions).flatMap(Collection::stream)
+        List<Long> dimensionIds = metricResps.stream()
+                .map(metricResp -> metricService.getDrillDownDimension(metricResp.getId()))
+                .flatMap(Collection::stream)
                .map(DrillDownDimension::getDimensionId).collect(Collectors.toList());
        metaFilter.setIds(dimensionIds);
        Map<Long, DimensionResp> dimensionRespMap = dimensionService.getDimensions(metaFilter)
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/KnowledgeServiceImpl.java
@@ -8,13 +8,14 @@ import com.tencent.supersonic.headless.core.knowledge.SearchService;
 import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
 import com.tencent.supersonic.headless.server.service.KnowledgeService;
 import com.tencent.supersonic.headless.server.service.ViewService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;

@Service
@Slf4j
@@ -68,17 +69,19 @@ public class KnowledgeServiceImpl implements KnowledgeService {

    @Override
    public List<S2Term> getTerms(String text) {
-        return HanlpHelper.getTerms(text);
+        Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>());
+        return HanlpHelper.getTerms(text, modelIdToViewIds);
    }

    @Override
    public List<HanlpMapResult> prefixSearch(String key, int limit, Set<Long> viewIds) {
        Map<Long, List<Long>> modelIdToViewIds = viewService.getModelIdToViewIds(new ArrayList<>(viewIds));
-        return prefixSearchByModel(key, limit, modelIdToViewIds.keySet());
+        return prefixSearchByModel(key, limit, modelIdToViewIds);
    }

-    public List<HanlpMapResult> prefixSearchByModel(String key, int limit, Set<Long> models) {
-        return SearchService.prefixSearch(key, limit, models);
+    public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
+                                                    Map<Long, List<Long>> modelIdToViewIds) {
+        return SearchService.prefixSearch(key, limit, modelIdToViewIds);
    }

    @Override
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
@@ -5,6 +5,7 @@ import com.alibaba.fastjson.TypeReference;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.DataEvent;
@@ -16,6 +17,7 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.common.util.ChatGptHelper;
 import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
+import com.tencent.supersonic.headless.api.pojo.MeasureParam;
 import com.tencent.supersonic.headless.api.pojo.MetricParam;
 import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
 import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
@@ -40,6 +42,12 @@ import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.ViewService;
 import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
 import com.tencent.supersonic.headless.server.utils.MetricConverter;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
+import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
@@ -48,11 +56,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.BeanUtils;
-import org.springframework.context.ApplicationEventPublisher;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;

@Service
@Slf4j
@@ -228,28 +231,44 @@ public class MetricServiceImpl implements MetricService {
    }

    private List<MetricResp> filterByField(List<MetricResp> metricResps, List<String> fields) {
-        List<MetricResp> metricRespFiltered = Lists.newArrayList();
+        Set<MetricResp> metricRespFiltered = Sets.newHashSet();
        for (MetricResp metricResp : metricResps) {
-            for (String field : fields) {
-                if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
-                    List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
-                            .stream().map(MetricParam::getId).collect(Collectors.toList());
-                    List<MetricResp> metricById = metricResps.stream()
-                            .filter(metric -> ids.contains(metric.getId()))
-                            .collect(Collectors.toList());
-                    for (MetricResp metric : metricById) {
-                        if (metric.getExpr().contains(field)) {
-                            metricRespFiltered.add(metricResp);
-                        }
-                    }
-                } else {
-                    if (metricResp.getExpr().contains(field)) {
-                        metricRespFiltered.add(metricResp);
-                    }
+            filterByField(metricResps, metricResp, fields, metricRespFiltered);
+        }
+        return new ArrayList<>(metricRespFiltered);
+    }
+
+    private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
+                                  List<String> fields, Set<MetricResp> metricRespFiltered) {
+        if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
+            List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
+                    .stream().map(MetricParam::getId).collect(Collectors.toList());
+            List<MetricResp> metricById = metricResps.stream()
+                    .filter(metric -> ids.contains(metric.getId()))
+                    .collect(Collectors.toList());
+            for (MetricResp metric : metricById) {
+                if (filterByField(metricResps, metric, fields, metricRespFiltered)) {
+                    metricRespFiltered.add(metricResp);
+                    return true;
                }
            }
+        } else if (MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())) {
+            if (fields.stream().anyMatch(field -> metricResp.getExpr().contains(field))) {
+                metricRespFiltered.add(metricResp);
+                return true;
+            }
+        } else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
+            List<MeasureParam> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
+            List<String> fieldNameDepended = measures.stream().map(MeasureParam::getBizName)
+                    //measure bizName = model bizName_fieldName
+                    .map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
+                    .collect(Collectors.toList());
+            if (fields.stream().anyMatch(fieldNameDepended::contains)) {
+                metricRespFiltered.add(metricResp);
+                return true;
+            }
        }
-        return metricRespFiltered;
+        return false;
    }

    @Override
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java
@@ -64,6 +64,13 @@ public class SchemaServiceImpl implements SchemaService {

    protected final Cache<String, List<ItemUseResp>> itemUseCache =
            CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
+
+    protected final Cache<ViewFilterReq, List<ViewSchemaResp>> viewSchemaCache =
+            CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build();
+
+    protected final Cache<SchemaFilterReq, SemanticSchemaResp> semanticSchemaCache =
+            CacheBuilder.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).build();
+
    private final StatUtils statUtils;
    private final ModelService modelService;
    private final DimensionService dimensionService;
@@ -91,6 +98,22 @@ public class SchemaServiceImpl implements SchemaService {
    @SneakyThrows
    @Override
    public List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter) {
+        List<ViewSchemaResp> viewList = viewSchemaCache.getIfPresent(filter);
+        if (CollectionUtils.isEmpty(viewList)) {
+            viewList = buildViewSchema(filter);
+            viewSchemaCache.put(filter, viewList);
+        }
+        return viewList;
+    }
+
+    public ViewSchemaResp fetchViewSchema(Long viewId) {
+        if (viewId == null) {
+            return null;
+        }
+        return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null);
+    }
+
+    public List<ViewSchemaResp> buildViewSchema(ViewFilterReq filter) {
        List<ViewSchemaResp> viewSchemaResps = new ArrayList<>();
        List<Long> viewIds = filter.getViewIds();
        MetaFilter metaFilter = new MetaFilter();
@@ -127,13 +150,6 @@ public class SchemaServiceImpl implements SchemaService {
        return viewSchemaResps;
    }

-    public ViewSchemaResp fetchViewSchema(Long viewId) {
-        if (viewId == null) {
-            return null;
-        }
-        return fetchViewSchema(new ViewFilterReq(viewId)).stream().findFirst().orElse(null);
-    }
-
    public List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds) {
        List<ModelSchemaResp> modelSchemaResps = Lists.newArrayList();
        if (CollectionUtils.isEmpty(modelIds)) {
@@ -258,8 +274,7 @@ public class SchemaServiceImpl implements SchemaService {
        return viewService.getViewList(metaFilter);
    }

-    @Override
-    public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) {
+    public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) {
        SemanticSchemaResp semanticSchemaResp = new SemanticSchemaResp();
        semanticSchemaResp.setViewId(schemaFilterReq.getViewId());
        semanticSchemaResp.setModelIds(schemaFilterReq.getModelIds());
@@ -294,6 +309,16 @@ public class SchemaServiceImpl implements SchemaService {
        return semanticSchemaResp;
    }

+    @Override
+    public SemanticSchemaResp fetchSemanticSchema(SchemaFilterReq schemaFilterReq) {
+        SemanticSchemaResp semanticSchemaResp = semanticSchemaCache.getIfPresent(schemaFilterReq);
+        if (semanticSchemaResp == null) {
+            semanticSchemaResp = buildSemanticSchema(schemaFilterReq);
+            semanticSchemaCache.put(schemaFilterReq, semanticSchemaResp);
+        }
+        return semanticSchemaResp;
+    }
+
    @SneakyThrows
    @Override
    public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ViewServiceImpl.java
@@ -10,17 +10,29 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
+import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.common.util.BeanMapper;
 import com.tencent.supersonic.headless.api.pojo.QueryConfig;
 import com.tencent.supersonic.headless.api.pojo.ViewDetail;
 import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
+import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
 import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
+import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
 import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
 import com.tencent.supersonic.headless.server.persistence.dataobject.ViewDO;
 import com.tencent.supersonic.headless.server.persistence.mapper.ViewDOMapper;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
+import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.service.DomainService;
+import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ViewService;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Date;
@@ -29,12 +41,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.stream.Collectors;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;

@Service
 public class ViewServiceImpl
@@ -46,21 +54,33 @@ public class ViewServiceImpl
    @Autowired
    private DomainService domainService;

+    @Lazy
+    @Autowired
+    private DimensionService dimensionService;
+
+    @Lazy
+    @Autowired
+    private MetricService metricService;
+
    @Override
    public ViewResp save(ViewReq viewReq, User user) {
        viewReq.createdBy(user.getName());
        ViewDO viewDO = convert(viewReq);
        viewDO.setStatus(StatusEnum.ONLINE.getCode());
+        ViewResp viewResp = convert(viewDO);
+        conflictCheck(viewResp);
        save(viewDO);
-        return convert(viewDO);
+        return viewResp;
    }

    @Override
    public ViewResp update(ViewReq viewReq, User user) {
        viewReq.updatedBy(user.getName());
        ViewDO viewDO = convert(viewReq);
+        ViewResp viewResp = convert(viewDO);
+        conflictCheck(viewResp);
        updateById(viewDO);
-        return convert(viewDO);
+        return viewResp;
    }

    @Override
@@ -78,6 +98,9 @@ public class ViewServiceImpl
        if (!CollectionUtils.isEmpty(metaFilter.getIds())) {
            wrapper.lambda().in(ViewDO::getId, metaFilter.getIds());
        }
+        if (metaFilter.getStatus() != null) {
+            wrapper.lambda().eq(ViewDO::getStatus, metaFilter.getStatus());
+        }
        wrapper.lambda().ne(ViewDO::getStatus, StatusEnum.DELETED.getCode());
        return list(wrapper).stream().map(this::convert).collect(Collectors.toList());
    }
@@ -175,4 +198,43 @@ public class ViewServiceImpl
                        viewResp -> viewResp.getAllModels().stream().map(modelId -> Pair.of(modelId, viewResp.getId())))
                .collect(Collectors.groupingBy(Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toList())));
    }
+
+    private void conflictCheck(ViewResp viewResp) {
+        List<Long> allDimensionIds = viewResp.getAllDimensions();
+        List<Long> allMetricIds = viewResp.getAllMetrics();
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setIds(allDimensionIds);
+        List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
+        metaFilter.setIds(allMetricIds);
+        List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
+
+        List<String> duplicateDimensionNames = findDuplicates(dimensionResps, DimensionResp::getName);
+        List<String> duplicateDimensionBizNames = findDuplicates(dimensionResps, DimensionResp::getBizName);
+
+        List<String> duplicateMetricNames = findDuplicates(metricResps, MetricResp::getName);
+        List<String> duplicateMetricBizNames = findDuplicates(metricResps, MetricResp::getBizName);
+        if (!duplicateDimensionNames.isEmpty()) {
+            throw new InvalidArgumentException("存在相同的维度名: " + duplicateDimensionNames);
+        }
+        if (!duplicateDimensionBizNames.isEmpty()) {
+            throw new InvalidArgumentException("存在相同的维度英文名: " + duplicateDimensionBizNames);
+        }
+        if (!duplicateMetricNames.isEmpty()) {
+            throw new InvalidArgumentException("存在相同的指标名: " + duplicateMetricNames);
+        }
+        if (!duplicateMetricBizNames.isEmpty()) {
+            throw new InvalidArgumentException("存在相同的指标英文名: " + duplicateMetricBizNames);
+        }
+    }
+
+    private <T, R> List<String> findDuplicates(List<T> list, Function<T, R> keyExtractor) {
+        return list.stream()
+                .collect(Collectors.groupingBy(keyExtractor, Collectors.counting()))
+                .entrySet().stream()
+                .filter(entry -> entry.getValue() > 1)
+                .map(Map.Entry::getKey)
+                .map(Object::toString)
+                .collect(Collectors.toList());
+    }
+
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricConverter.java
@@ -72,6 +72,7 @@ public class MetricConverter {
        ModelResp modelResp = modelMap.get(metricDO.getModelId());
        if (modelResp != null) {
            metricResp.setModelName(modelResp.getName());
+            metricResp.setModelBizName(modelResp.getBizName());
            metricResp.setDomainId(modelResp.getDomainId());
        }
        metricResp.setIsCollect(collect != null && collect.contains(metricDO.getId()));