(improvement)(project) dsl support get topN dimension/metric by useCount and fix semanticService get dimension/metric usrCount error (#105)

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java

@@ -67,8 +67,11 @@ public class QueryServiceImpl implements QueryService {
 
     @Override
     public Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception {
+        statUtils.initStatInfo(querySqlCmd, user);
         QueryStatement queryStatement = convertToQueryStatement(querySqlCmd, user);
-        return semanticQueryEngine.execute(queryStatement);
+        QueryResultWithSchemaResp results = semanticQueryEngine.execute(queryStatement);
+        statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
+        return results;
     }
 
     private QueryStatement convertToQueryStatement(QueryDslReq querySqlCmd, User user) throws Exception {

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/SchemaServiceImpl.java

@@ -55,7 +55,10 @@ public class SchemaServiceImpl implements SchemaService {
     @Override
     public List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq filter, User user) {
         List<ModelSchemaResp> domainSchemaDescList = modelService.fetchModelSchema(filter);
-        List<ItemUseResp> statInfos = queryService.getStatInfo(new ItemUseReq());
+        ItemUseReq itemUseCommend = new ItemUseReq();
+        itemUseCommend.setModelIds(filter.getModelIds());
+
+        List<ItemUseResp> statInfos = queryService.getStatInfo(itemUseCommend);
         log.debug("statInfos:{}", statInfos);
         fillCnt(domainSchemaDescList, statInfos);
         return domainSchemaDescList;

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java

@@ -4,22 +4,30 @@ import com.alibaba.ttl.TransmittableThreadLocal;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.semantic.api.model.enums.QueryTypeBackEnum;
 import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
 import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
+import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
+import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
 import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
+import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.semantic.model.domain.ModelService;
 import com.tencent.supersonic.semantic.query.persistence.repository.StatRepository;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
 
 @Component
 @Slf4j
@@ -28,13 +36,17 @@ public class StatUtils {
     private static final TransmittableThreadLocal<QueryStat> STATS = new TransmittableThreadLocal<>();
     private final StatRepository statRepository;
     private final SqlFilterUtils sqlFilterUtils;
+
+    private final ModelService modelService;
     private final ObjectMapper objectMapper = new ObjectMapper();
 
     public StatUtils(StatRepository statRepository,
-            SqlFilterUtils sqlFilterUtils) {
+            SqlFilterUtils sqlFilterUtils,
+            ModelService modelService) {
 
         this.statRepository = statRepository;
         this.sqlFilterUtils = sqlFilterUtils;
+        this.modelService = modelService;
     }
 
     public static QueryStat get() {
@@ -69,6 +81,44 @@ public class StatUtils {
         return true;
     }
 
+
+    public void initStatInfo(QueryDslReq queryDslReq, User facadeUser) {
+        QueryStat queryStatInfo = new QueryStat();
+        List<String> allFields = SqlParserSelectHelper.getAllFields(queryDslReq.getSql());
+        queryStatInfo.setModelId(queryDslReq.getModelId());
+        ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryDslReq.getModelId());
+
+        List<String> dimensions = new ArrayList<>();
+        if (Objects.nonNull(modelSchemaResp)) {
+            dimensions = getFieldNames(allFields, modelSchemaResp.getDimensions());
+        }
+
+        List<String> metrics = new ArrayList<>();
+        if (Objects.nonNull(modelSchemaResp)) {
+            metrics = getFieldNames(allFields, modelSchemaResp.getMetrics());
+        }
+
+        String userName = getUserName(facadeUser);
+        try {
+            queryStatInfo.setTraceId("")
+                    .setModelId(queryDslReq.getModelId())
+                    .setUser(userName)
+                    .setQueryType(QueryTypeEnum.SQL.getValue())
+                    .setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())
+                    .setQuerySqlCmd(queryDslReq.toString())
+                    .setQuerySqlCmdMd5(DigestUtils.md5Hex(queryDslReq.toString()))
+                    .setStartTime(System.currentTimeMillis())
+                    .setUseResultCache(true)
+                    .setUseSqlCache(true)
+                    .setMetrics(objectMapper.writeValueAsString(metrics))
+                    .setDimensions(objectMapper.writeValueAsString(dimensions));
+        } catch (JsonProcessingException e) {
+            log.error("initStatInfo:{}", e);
+        }
+        StatUtils.set(queryStatInfo);
+
+    }
+
     public void initStatInfo(QueryStructReq queryStructCmd, User facadeUser) {
         QueryStat queryStatInfo = new QueryStat();
         String traceId = "";
@@ -76,12 +126,11 @@ public class StatUtils {
 
         List<String> metrics = new ArrayList<>();
         queryStructCmd.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn()));
-        String user = (Objects.nonNull(facadeUser) && Strings.isNotEmpty(facadeUser.getName())) ? facadeUser.getName()
-                : "Admin";
+        String user = getUserName(facadeUser);
 
         try {
             queryStatInfo.setTraceId(traceId)
-                    .setClassId(queryStructCmd.getModelId())
+                    .setModelId(queryStructCmd.getModelId())
                     .setUser(user)
                     .setQueryType(QueryTypeEnum.STRUCT.getValue())
                     .setQueryTypeBack(QueryTypeBackEnum.NORMAL.getState())
@@ -105,6 +154,25 @@ public class StatUtils {
 
     }
 
+    private List<String> getFieldNames(List<String> allFields, List<? extends SchemaItem> schemaItems) {
+        Set<String> fieldNames = schemaItems
+                .stream()
+                .map(dimSchemaResp -> dimSchemaResp.getBizName())
+                .collect(Collectors.toSet());
+        if (!CollectionUtils.isEmpty(fieldNames)) {
+            return allFields.stream().filter(fieldName -> fieldNames.contains(fieldName))
+                    .collect(Collectors.toList());
+        }
+        return new ArrayList<>();
+    }
+
+    private String getUserName(User facadeUser) {
+        return (Objects.nonNull(facadeUser) && Strings.isNotEmpty(facadeUser.getName())) ? facadeUser.getName()
+                : "Admin";
+    }
+
+
+
     public List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend) {
         return statRepository.getStatInfo(itemUseCommend);
     }

semantic/query/src/main/resources/mapper/StatMapper.xml

@@ -64,6 +64,12 @@
             <if test="modelId != null">
                 and model_id = #{modelId}
             </if>
+            <if test="modelIds != null and modelIds.size() > 0">
+                and model_id in
+                <foreach item="id" collection="modelIds" open="(" separator="," close=")">
+                    #{id}
+                </foreach>
+            </if>
             <if test="metric != null">
                 and metrics like concat('%',#{metric},'%')
             </if>