(improvement)(semantic) Metric homepage download supports choosing whether to transform (#480)

Co-authored-by: jolunoluo

auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.auth.authorization.application;
 
 import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
 import com.google.gson.Gson;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.service.UserService;
@@ -108,8 +109,11 @@ public class AuthServiceImpl implements AuthService {
             }
         }
 
-        if (req.getModelId() != null) {
+        if (!CollectionUtils.isEmpty(req.getModelIds())) {
-            List<AuthGroup> authGroups = authGroupsByModelId.get(req.getModelId());
+            List<AuthGroup> authGroups = Lists.newArrayList();
+            for (Long modelId : authGroupsByModelId.keySet()) {
+                authGroups.addAll(authGroupsByModelId.getOrDefault(modelId, Lists.newArrayList()));
+            }
             if (!CollectionUtils.isEmpty(authGroups)) {
                 for (AuthGroup group : authGroups) {
                     if (group.getDimensionFilters() != null

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.api.pojo.response;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import lombok.Data;
+import org.apache.commons.collections.CollectionUtils;
 import java.util.List;
 
 @Data
@@ -21,4 +22,14 @@ public class ParseResp {
         FAILED
     }
 
+    //Compatible with front-end
+    public List<SemanticParseInfo> getSelectedParses() {
+        selectedParses = Lists.newArrayList();
+        if (CollectionUtils.isNotEmpty(candidateParses)) {
+            selectedParses.addAll(candidateParses);
+            candidateParses.clear();
+        }
+        return selectedParses;
+    }
+
 }

chat/core/src/main/java/com/tencent/supersonic/chat/agent/DataAnalyticsTool.java

@@ -1,16 +1,11 @@
 package com.tencent.supersonic.chat.agent;
 
-import com.tencent.supersonic.chat.parser.analytics.MetricOption;
 import lombok.Data;
 
-import java.util.List;
-
 
 @Data
 public class DataAnalyticsTool extends AgentTool {
 
     private Long modelId;
 
-    private List<MetricOption> metricOptions;
-
 }

chat/core/src/main/java/com/tencent/supersonic/chat/parser/analytics/MetricAnalyzeParser.java

@@ -1,151 +0,0 @@
-package com.tencent.supersonic.chat.parser.analytics;
-
-import com.alibaba.fastjson.JSONObject;
-import com.google.common.collect.Sets;
-import com.tencent.supersonic.chat.agent.Agent;
-import com.tencent.supersonic.chat.agent.AgentToolType;
-import com.tencent.supersonic.chat.agent.DataAnalyticsTool;
-import com.tencent.supersonic.chat.api.component.SemanticParser;
-import com.tencent.supersonic.chat.api.pojo.ChatContext;
-import com.tencent.supersonic.chat.api.pojo.QueryContext;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
-import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
-import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
-import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
-import com.tencent.supersonic.chat.parser.SatisfactionChecker;
-import com.tencent.supersonic.chat.query.QueryManager;
-import com.tencent.supersonic.chat.query.llm.LLMSemanticQuery;
-import com.tencent.supersonic.chat.query.llm.analytics.MetricAnalyzeQuery;
-import com.tencent.supersonic.chat.service.AgentService;
-import com.tencent.supersonic.chat.service.SemanticService;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.ModelCluster;
-import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
-import com.tencent.supersonic.common.util.ContextUtils;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.util.CollectionUtils;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-@Slf4j
-public class MetricAnalyzeParser implements SemanticParser {
-
-    @Override
-    public void parse(QueryContext queryContext, ChatContext chatContext) {
-        if (SatisfactionChecker.isSkip(queryContext)) {
-            log.info("skip MetricAnalyzeParser");
-            return;
-        }
-        Map<Long, DataAnalyticsTool> metricInterpretToolMap =
-                getMetricInterpretTools(queryContext.getRequest().getAgentId());
-        log.info("metric interpret tool : {}", metricInterpretToolMap);
-        if (CollectionUtils.isEmpty(metricInterpretToolMap)) {
-            return;
-        }
-        Map<Long, List<SchemaElementMatch>> elementMatches = queryContext.getMapInfo().getModelElementMatches();
-        for (Long modelId : elementMatches.keySet()) {
-            DataAnalyticsTool metricInterpretTool = metricInterpretToolMap.get(modelId);
-            if (metricInterpretTool == null) {
-                continue;
-            }
-            if (CollectionUtils.isEmpty(elementMatches.get(modelId))) {
-                continue;
-            }
-            List<MetricOption> metricOptions = metricInterpretTool.getMetricOptions();
-            if (!CollectionUtils.isEmpty(metricOptions)) {
-                List<Long> metricIds = metricOptions.stream()
-                        .map(MetricOption::getMetricId).collect(Collectors.toList());
-                String name = metricInterpretTool.getName();
-                buildQuery(modelId, queryContext, metricIds, elementMatches.get(modelId), name);
-            }
-        }
-    }
-
-    private void buildQuery(Long modelId, QueryContext queryContext,
-                            List<Long> metricIds, List<SchemaElementMatch> schemaElementMatches, String toolName) {
-        LLMSemanticQuery metricInterpretQuery = QueryManager.createLLMQuery(MetricAnalyzeQuery.QUERY_MODE);
-        Set<SchemaElement> metrics = getMetrics(metricIds, modelId);
-        SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(modelId, queryContext.getRequest(),
-                metrics, schemaElementMatches, toolName);
-        semanticParseInfo.setQueryMode(metricInterpretQuery.getQueryMode());
-        semanticParseInfo.getProperties().put("queryText", queryContext.getRequest().getQueryText());
-        metricInterpretQuery.setParseInfo(semanticParseInfo);
-        queryContext.getCandidateQueries().add(metricInterpretQuery);
-    }
-
-    public Set<SchemaElement> getMetrics(List<Long> metricIds, Long modelId) {
-        SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
-        List<SchemaElement> metrics = semanticService.getSemanticSchema().getMetrics();
-        return metrics.stream().filter(schemaElement -> metricIds.contains(schemaElement.getId()))
-                .collect(Collectors.toSet());
-    }
-
-    private Map<Long, DataAnalyticsTool> getMetricInterpretTools(Integer agentId) {
-        AgentService agentService = ContextUtils.getBean(AgentService.class);
-        Agent agent = agentService.getAgent(agentId);
-        if (agent == null) {
-            return new HashMap<>();
-        }
-        List<String> tools = agent.getTools(AgentToolType.ANALYTICS);
-        if (CollectionUtils.isEmpty(tools)) {
-            return new HashMap<>();
-        }
-        List<DataAnalyticsTool> metricInterpretTools = tools.stream().map(tool ->
-                        JSONObject.parseObject(tool, DataAnalyticsTool.class))
-                .filter(tool -> !CollectionUtils.isEmpty(tool.getMetricOptions()))
-                .collect(Collectors.toList());
-        Map<Long, DataAnalyticsTool> metricInterpretToolMap = new HashMap<>();
-        for (DataAnalyticsTool metricInterpretTool : metricInterpretTools) {
-            metricInterpretToolMap.putIfAbsent(metricInterpretTool.getModelId(),
-                    metricInterpretTool);
-        }
-        return metricInterpretToolMap;
-    }
-
-    private SemanticParseInfo buildSemanticParseInfo(Long modelId, QueryReq queryReq, Set<SchemaElement> metrics,
-                                                     List<SchemaElementMatch> schemaElementMatches, String toolName) {
-        SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
-        semanticParseInfo.setMetrics(metrics);
-        SchemaElement dimension = new SchemaElement();
-        dimension.setBizName(TimeDimensionEnum.DAY.getName());
-        semanticParseInfo.setDimensions(Sets.newHashSet(dimension));
-        semanticParseInfo.setElementMatches(schemaElementMatches);
-        semanticParseInfo.setModel(ModelCluster.build(Sets.newHashSet(modelId)));
-        semanticParseInfo.setScore(queryReq.getQueryText().length());
-        DateConf dateConf = new DateConf();
-        dateConf.setDateMode(DateConf.DateMode.RECENT);
-        dateConf.setUnit(15);
-        semanticParseInfo.setDateInfo(dateConf);
-        Map<String, Object> properties = new HashMap<>();
-        properties.put("type", "internal");
-        properties.put("name", toolName);
-        semanticParseInfo.setProperties(properties);
-        fillSemanticParseInfo(semanticParseInfo);
-        return semanticParseInfo;
-    }
-
-    private void fillSemanticParseInfo(SemanticParseInfo semanticParseInfo) {
-        List<SchemaElementMatch> schemaElementMatches = semanticParseInfo.getElementMatches();
-        if (!CollectionUtils.isEmpty(schemaElementMatches)) {
-            schemaElementMatches.stream().filter(schemaElementMatch ->
-                            SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
-                                    || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
-                    .forEach(schemaElementMatch -> {
-                        QueryFilter queryFilter = new QueryFilter();
-                        queryFilter.setValue(schemaElementMatch.getWord());
-                        queryFilter.setElementID(schemaElementMatch.getElement().getId());
-                        queryFilter.setName(schemaElementMatch.getElement().getName());
-                        queryFilter.setOperator(FilterOperatorEnum.EQUALS);
-                        queryFilter.setBizName(schemaElementMatch.getElement().getBizName());
-                        semanticParseInfo.getDimensionFilters().add(queryFilter);
-                    });
-        }
-    }
-
-}

chat/core/src/main/java/com/tencent/supersonic/chat/parser/analytics/MetricOption.java

@@ -1,14 +0,0 @@
-package com.tencent.supersonic.chat.parser.analytics;
-
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class MetricOption {
-
-    private Long metricId;
-}

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java

@@ -70,8 +70,8 @@ public class DownloadServiceImpl implements DownloadService {
         String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
         File file = FileUtils.createTmpFile(fileName);
         try {
-            QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(downloadStructReq, user);
+            QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
-            DataDownload dataDownload = buildDataDownload(queryResultWithSchemaResp, downloadStructReq);
+            DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
             EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
         } catch (RuntimeException e) {
             EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())