[improvement](supersonic) based on version 0.7.2 (#34)

Co-authored-by: zuopengge <hwzuopengge@tencent.com>
2025-12-11 12:07:42 +00:00 · 2023-08-20 17:30:35 +08:00
parent c93e60ced7
commit cf1b5336c3
122 changed files with 4045 additions and 1075 deletions
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java
@@ -1,26 +1,13 @@
 package com.tencent.supersonic.semantic.model.application;

 import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
-import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
-import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
-import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
-import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
-import com.tencent.supersonic.semantic.api.model.response.MetricResp;
-import com.tencent.supersonic.semantic.api.model.response.ModelResp;
+import com.tencent.supersonic.semantic.api.model.response.*;
 import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
 import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
 import com.tencent.supersonic.semantic.api.model.yaml.MetricYamlTpl;
-import com.tencent.supersonic.semantic.model.domain.Catalog;
-import com.tencent.supersonic.semantic.model.domain.DatasourceService;
-import com.tencent.supersonic.semantic.model.domain.DimensionService;
-import com.tencent.supersonic.semantic.model.domain.MetricService;
-import com.tencent.supersonic.semantic.model.domain.ModelService;
-import com.tencent.supersonic.semantic.model.domain.dataobject.DatabaseDO;
-import com.tencent.supersonic.semantic.model.domain.repository.DatabaseRepository;
-import com.tencent.supersonic.semantic.model.domain.utils.DatabaseConverter;
+import com.tencent.supersonic.semantic.model.domain.*;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
@@ -29,17 +16,17 @@ import org.springframework.stereotype.Component;
@Component
 public class CatalogImpl implements Catalog {

-    private final DatabaseRepository databaseRepository;
+    private final DatabaseService databaseService;
    private final ModelService modelService;
    private final DimensionService dimensionService;
    private final DatasourceService datasourceService;
    private final MetricService metricService;

-    public CatalogImpl(DatabaseRepository databaseRepository,
+    public CatalogImpl(DatabaseService databaseService,
            ModelService modelService, DimensionService dimensionService,
            DatasourceService datasourceService,
            MetricService metricService) {
-        this.databaseRepository = databaseRepository;
+        this.databaseService = databaseService;
        this.modelService = modelService;
        this.dimensionService = dimensionService;
        this.datasourceService = datasourceService;
@@ -47,14 +34,11 @@ public class CatalogImpl implements Catalog {
    }

    public DatabaseResp getDatabase(Long id) {
-        DatabaseDO databaseDO = databaseRepository.getDatabase(id);
-        return DatabaseConverter.convert(databaseDO);
+        return databaseService.getDatabase(id);
    }

    public DatabaseResp getDatabaseByModelId(Long modelId) {
-        List<DatabaseDO> databaseDOS = databaseRepository.getDatabaseByDomainId(modelId);
-        Optional<DatabaseDO> databaseDO = databaseDOS.stream().findFirst();
-        return databaseDO.map(DatabaseConverter::convert).orElse(null);
+        return databaseService.getDatabaseByModelId(modelId);
    }

    @Override
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java
@@ -1,27 +1,37 @@
 package com.tencent.supersonic.semantic.model.application;

+import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
+import com.tencent.supersonic.common.pojo.QueryColumn;
+import com.tencent.supersonic.common.util.ChatGptHelper;
+import com.tencent.supersonic.semantic.api.model.pojo.DatasourceDetail;
+import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
 import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
 import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
+import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
 import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
+import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
+import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
+import com.tencent.supersonic.semantic.model.domain.DatabaseService;
+import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO;
+import com.tencent.supersonic.semantic.model.domain.repository.DimensionRepository;
+import com.tencent.supersonic.semantic.model.domain.utils.DimensionConverter;
 import com.tencent.supersonic.semantic.model.domain.DatasourceService;
 import com.tencent.supersonic.semantic.model.domain.DimensionService;
 import com.tencent.supersonic.semantic.model.domain.DomainService;
-import com.tencent.supersonic.semantic.model.domain.dataobject.DimensionDO;
 import com.tencent.supersonic.semantic.model.domain.pojo.Dimension;
 import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter;
-import com.tencent.supersonic.semantic.model.domain.repository.DimensionRepository;
-import com.tencent.supersonic.semantic.model.domain.utils.DimensionConverter;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+
+import java.util.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
@@ -39,13 +49,22 @@ public class DimensionServiceImpl implements DimensionService {

    private DomainService domainService;

+    private ChatGptHelper chatGptHelper;
+
+    private DatabaseService databaseService;
+

    public DimensionServiceImpl(DimensionRepository dimensionRepository,
            DomainService domainService,
-            DatasourceService datasourceService) {
+            DatasourceService datasourceService,
+                                ChatGptHelper chatGptHelper,
+                                DatabaseService databaseService) {
        this.domainService = domainService;
        this.dimensionRepository = dimensionRepository;
        this.datasourceService = datasourceService;
+        this.chatGptHelper  = chatGptHelper;
+        this.databaseService = databaseService;
+
    }

    @Override
@@ -238,6 +257,53 @@ public class DimensionServiceImpl implements DimensionService {
        dimensionRepository.deleteDimension(id);
    }

+    @Override
+    public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
+        String mockAlias = chatGptHelper.mockAlias(mockType,dimensionReq.getName(), dimensionReq.getBizName(), "", dimensionReq.getDescription() ,false);
+        return JSONObject.parseObject(mockAlias, new TypeReference<List<String>>() {});
+    }
+
+    @Override
+    public List<DimValueMap> mockDimensionValueAlias(DimensionReq dimensionReq, User user) {
+
+        List<DatasourceResp> datasourceList = datasourceService.getDatasourceList();
+        List<DatasourceResp> collect = datasourceList.stream().filter(datasourceResp -> datasourceResp.getId().equals(dimensionReq.getDatasourceId())).collect(Collectors.toList());
+
+        if (collect.isEmpty()){
+            return null;
+        }
+        DatasourceResp datasourceResp = collect.get(0);
+        DatasourceDetail datasourceDetail = datasourceResp.getDatasourceDetail();
+        String sqlQuery = datasourceDetail.getSqlQuery();
+
+        DatabaseResp database = databaseService.getDatabase(datasourceResp.getDatabaseId());
+
+        String sql = "select ai_talk."+dimensionReq.getBizName()+" from ("+sqlQuery +") as ai_talk group by ai_talk."+dimensionReq.getBizName();
+        QueryResultWithSchemaResp queryResultWithSchemaResp = databaseService.executeSql(sql, database);
+        List<Map<String, Object>> resultList = queryResultWithSchemaResp.getResultList();
+        List<String> valueList = new ArrayList<>();
+        for (Map<String, Object> stringObjectMap : resultList) {
+            String value = (String) stringObjectMap.get(dimensionReq.getBizName());
+            valueList.add(value);
+        }
+        String json = chatGptHelper.mockDimensionValueAlias(JSON.toJSONString(valueList));
+        log.info("return llm res is :{}",json);
+
+        JSONObject jsonObject = JSON.parseObject(json);
+
+        List<DimValueMap> dimValueMapsResp = new ArrayList<>();
+        int i = 0;
+        for (Map<String, Object> stringObjectMap : resultList) {
+            DimValueMap dimValueMap = new DimValueMap();
+            dimValueMap.setTechName((String) stringObjectMap.get(dimensionReq.getBizName()));
+            dimValueMap.setBizName(jsonObject.getJSONArray("tran").getString(i));
+            dimValueMap. setAlias(jsonObject.getJSONObject("alias").getJSONArray((String) stringObjectMap.get(dimensionReq.getBizName())).toJavaList(String.class));
+            dimValueMapsResp.add(dimValueMap);
+            i ++ ;
+        }
+        return dimValueMapsResp;
+    }
+

    private void checkExist(List<DimensionReq> dimensionReqs) {
        Long modelId = dimensionReqs.get(0).getModelId();
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java
@@ -16,16 +16,8 @@ import com.tencent.supersonic.semantic.model.domain.dataobject.DomainDO;
 import com.tencent.supersonic.semantic.model.domain.pojo.Domain;
 import com.tencent.supersonic.semantic.model.domain.repository.DomainRepository;
 import com.tencent.supersonic.semantic.model.domain.utils.DomainConvert;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.Set;
+
+import java.util.*;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.assertj.core.util.Sets;
@@ -101,7 +93,8 @@ public class DomainServiceImpl implements DomainService {
            List<Long> domainIds = modelResps.stream().map(ModelResp::getDomainId).collect(Collectors.toList());
            domainWithAuthAll.addAll(getParentDomain(domainIds));
        }
-        return new ArrayList<>(domainWithAuthAll);
+        return new ArrayList<>(domainWithAuthAll).stream()
+                .sorted(Comparator.comparingLong(DomainResp::getId)).collect(Collectors.toList());
    }

    @Override
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java
@@ -1,26 +1,29 @@
 package com.tencent.supersonic.semantic.model.application;

 import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
 import com.github.pagehelper.PageHelper;
 import com.github.pagehelper.PageInfo;
 import com.google.common.collect.Lists;
+import com.plexpt.chatgpt.ChatGPT;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
+import com.tencent.supersonic.common.util.ChatGptHelper;
 import com.tencent.supersonic.semantic.api.model.pojo.Measure;
 import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
 import com.tencent.supersonic.semantic.api.model.request.MetricReq;
 import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
 import com.tencent.supersonic.semantic.api.model.response.DomainResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
+import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
 import com.tencent.supersonic.semantic.api.model.response.ModelResp;
 import com.tencent.supersonic.semantic.model.domain.DomainService;
-import com.tencent.supersonic.semantic.model.domain.MetricService;
 import com.tencent.supersonic.semantic.model.domain.ModelService;
 import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO;
-import com.tencent.supersonic.semantic.model.domain.pojo.Metric;
 import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter;
 import com.tencent.supersonic.semantic.model.domain.repository.MetricRepository;
 import com.tencent.supersonic.semantic.model.domain.utils.MetricConverter;
+import com.tencent.supersonic.semantic.model.domain.MetricService;
+import com.tencent.supersonic.semantic.model.domain.pojo.Metric;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -42,12 +45,16 @@ public class MetricServiceImpl implements MetricService {

    private DomainService domainService;

+    private ChatGptHelper chatGptHelper;
+
    public MetricServiceImpl(MetricRepository metricRepository,
-            ModelService modelService,
-            DomainService domainService) {
+                             ModelService modelService,
+                             DomainService domainService,
+                             ChatGptHelper chatGptHelper) {
        this.domainService = domainService;
        this.metricRepository = metricRepository;
        this.modelService = modelService;
+        this.chatGptHelper = chatGptHelper;
    }

    @Override
@@ -74,7 +81,7 @@ public class MetricServiceImpl implements MetricService {
        log.info("[insert metric] object:{}", JSONObject.toJSONString(metricToInsert));
        saveMetricBatch(metricToInsert, user);
    }
-
+    
    @Override
    public List<MetricResp> getMetrics(Long modelId) {
        return convertList(metricRepository.getMetricList(modelId));
@@ -201,6 +208,14 @@ public class MetricServiceImpl implements MetricService {
        metricRepository.deleteMetric(id);
    }

+    @Override
+    public List<String> mockAlias(MetricReq metricReq,String mockType,User user) {
+
+        String mockAlias = chatGptHelper.mockAlias(mockType,metricReq.getName(), metricReq.getBizName(), "", metricReq.getDescription() ,!"".equals(metricReq.getDataFormatType()));
+        return JSONObject.parseObject(mockAlias, new TypeReference<List<String>>() {});
+    }
+
+

    private void saveMetricBatch(List<Metric> metrics, User user) {
        if (CollectionUtils.isEmpty(metrics)) {
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java
@@ -2,7 +2,9 @@ package com.tencent.supersonic.semantic.model.domain;

 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
 import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
+import com.tencent.supersonic.semantic.api.model.request.MetricReq;
 import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import java.util.List;
@@ -32,4 +34,8 @@ public interface DimensionService {
    List<DimensionResp> getAllHighSensitiveDimension();

    void deleteDimension(Long id) throws Exception;
+
+    List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user);
+
+    List<DimValueMap> mockDimensionValueAlias(DimensionReq dimensionReq, User user);
 }
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java
@@ -32,4 +32,6 @@ public interface MetricService {
    List<MetricResp> getAllHighSensitiveMetric();

    void deleteMetric(Long id) throws Exception;
+
+    List<String> mockAlias(MetricReq metricReq,String mockType,User user);
 }
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DimensionConverter.java
@@ -30,6 +30,8 @@ public class DimensionConverter {
        dimensionDO.setDefaultValues(JSONObject.toJSONString(dimension.getDefaultValues()));
        if (!CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
            dimensionDO.setDimValueMaps(JSONObject.toJSONString(dimension.getDimValueMaps()));
+        } else {
+            dimensionDO.setDimValueMaps(JSONObject.toJSONString(new ArrayList<>()));
        }
        return dimensionDO;
    }
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DimensionController.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/DimensionController.java
@@ -3,13 +3,17 @@ package com.tencent.supersonic.semantic.model.rest;
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
 import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
+import com.tencent.supersonic.semantic.api.model.request.MetricReq;
 import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.model.domain.DimensionService;
 import java.util.List;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+
+import com.tencent.supersonic.semantic.model.domain.MetricService;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -27,7 +31,10 @@ public class DimensionController {
    private DimensionService dimensionService;


-    public DimensionController(DimensionService dimensionService) {
+    private MetricService metricService;
+
+    public DimensionController(DimensionService dimensionService,MetricService metricService) {
+        this.metricService = metricService;
        this.dimensionService = dimensionService;
    }

@@ -56,6 +63,22 @@ public class DimensionController {
        return true;
    }

+    @PostMapping("/mockDimensionAlias")
+    public List<String> mockMetricAlias(@RequestBody DimensionReq dimensionReq,
+                                        HttpServletRequest request,
+                                        HttpServletResponse response){
+        User user = UserHolder.findUser(request, response);
+        return  dimensionService.mockAlias(dimensionReq,"dimension",user);
+    }
+
+
+    @PostMapping("/mockDimensionValuesAlias")
+    public List<DimValueMap> mockDimensionValuesAlias(@RequestBody DimensionReq dimensionReq,
+                                                      HttpServletRequest request,
+                                                      HttpServletResponse response){
+        User user = UserHolder.findUser(request, response);
+        return  dimensionService.mockDimensionValueAlias(dimensionReq,user);
+    }

    @GetMapping("/getDimensionList/{modelId}")
    public List<DimensionResp> getDimension(@PathVariable("modelId") Long modelId) {
--- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java
+++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/rest/MetricController.java
@@ -11,13 +11,9 @@ import com.tencent.supersonic.semantic.model.domain.MetricService;
 import java.util.List;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import org.springframework.web.bind.annotation.DeleteMapping;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.web.bind.annotation.*;


@RestController
@@ -52,6 +48,14 @@ public class MetricController {
    }


+    @PostMapping("/mockMetricAlias")
+    public List<String> mockMetricAlias(@RequestBody MetricReq metricReq,
+                                        HttpServletRequest request,
+                                        HttpServletResponse response){
+        User user = UserHolder.findUser(request, response);
+        return  metricService.mockAlias(metricReq,"indicator",user);
+    }
+
    @GetMapping("/getMetricList/{modelId}")
    public List<MetricResp> getMetricList(@PathVariable("modelId") Long modelId) {
        return metricService.getMetrics(modelId);