diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java index 92434bf47..0d4678286 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java @@ -2,16 +2,14 @@ package com.tencent.supersonic.chat.api.pojo; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import org.springframework.util.CollectionUtils; - import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; +import org.springframework.util.CollectionUtils; public class SemanticSchema implements Serializable { @@ -54,35 +52,6 @@ public class SemanticSchema implements Serializable { } } - public SchemaElement getElementByName(SchemaElementType elementType, String name) { - Optional element = Optional.empty(); - - switch (elementType) { - case ENTITY: - element = getElementsByNameOrAlias(name, getEntities()); - break; - case VIEW: - element = getElementsByNameOrAlias(name, getViews()); - break; - case METRIC: - element = getElementsByNameOrAlias(name, getMetrics()); - break; - case DIMENSION: - element = getElementsByNameOrAlias(name, getDimensions()); - break; - case VALUE: - element = getElementsByNameOrAlias(name, getDimensionValues()); - break; - default: - } - - if (element.isPresent()) { - return element.get(); - } else { - return null; - } - } - public Map getViewIdToName() { return viewSchemaList.stream() .collect(Collectors.toMap(a -> a.getView().getId(), a -> a.getView().getName(), (k1, k2) -> k1)); @@ -159,14 +128,6 @@ public class SemanticSchema implements Serializable { .findFirst(); } - private Optional getElementsByNameOrAlias(String name, List elements) { - return elements.stream() - .filter(schemaElement -> - name.equals(schemaElement.getName()) || (Objects.nonNull(schemaElement.getAlias()) - && schemaElement.getAlias().contains(name)) - ).findFirst(); - } - public SchemaElement getView(Long viewId) { List views = getViews(); return getElementsById(viewId, views).orElse(null); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java index 03fc03164..3a38c49b0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java @@ -121,7 +121,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } QueryStructReq queryStructReq = convertQueryStruct(); convertBizNameToName(semanticSchema, queryStructReq); - QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); + QuerySqlReq querySQLReq = queryStructReq.convert(); parseInfo.getSqlInfo().setS2SQL(querySQLReq.getSql()); parseInfo.getSqlInfo().setCorrectS2SQL(querySQLReq.getSql()); } diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java index 6a6e222f6..59e3dc51d 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java +++ b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java @@ -52,14 +52,14 @@ class QueryReqBuilderTest { orders.add(order); queryStructReq.setOrders(orders); - QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); + QuerySqlReq querySQLReq = queryStructReq.convert(); Assert.assertEquals( "SELECT department, SUM(pv) AS pv FROM 内容库 " + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP " + "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql()); queryStructReq.setQueryType(QueryType.TAG); - querySQLReq = queryStructReq.convert(queryStructReq); + querySQLReq = queryStructReq.convert(); Assert.assertEquals( "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " + "ORDER BY uv LIMIT 2000", diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java index 4ba3b6dbc..7e9c11459 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.request; +import com.tencent.supersonic.common.pojo.DateConf; import java.util.List; import lombok.Data; import lombok.ToString; @@ -18,4 +19,8 @@ public class QueryMetricReq { private List dimensionNames; + private DateConf dateInfo = new DateConf(); + + private Long limit = 2000L; + } \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 679563c9d..470a2afdf 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -44,6 +44,7 @@ import java.util.stream.Collectors; @Data @Slf4j public class QueryStructReq extends SemanticQueryReq { + private List groups = new ArrayList<>(); private List aggregators = new ArrayList<>(); private List orders = new ArrayList<>(); @@ -151,28 +152,27 @@ public class QueryStructReq extends SemanticQueryReq { return sb.toString(); } - public QuerySqlReq convert(QueryStructReq queryStructReq) { - return convert(queryStructReq, false); + public QuerySqlReq convert() { + return convert(false); } /** * convert queryStructReq to QueryS2SQLReq * - * @param queryStructReq * @return */ - public QuerySqlReq convert(QueryStructReq queryStructReq, boolean isBizName) { + public QuerySqlReq convert(boolean isBizName) { String sql = null; try { - sql = buildSql(queryStructReq, isBizName); + sql = buildSql(this, isBizName); } catch (Exception e) { log.error("buildSql error", e); } QuerySqlReq result = new QuerySqlReq(); result.setSql(sql); - result.setViewId(queryStructReq.getViewId()); - result.setModelIds(queryStructReq.getModelIdSet()); + result.setViewId(this.getViewId()); + result.setModelIds(this.getModelIdSet()); result.setParams(new ArrayList<>()); return result; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java index f96617aeb..ac515e6e9 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelSchemaResp.java @@ -1,12 +1,14 @@ package com.tencent.supersonic.headless.api.pojo.response; +import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.ModelRela; -import com.tencent.supersonic.headless.api.pojo.Identify; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; - -import java.util.List; +import org.apache.commons.collections4.CollectionUtils; @Data @AllArgsConstructor @@ -17,18 +19,17 @@ public class ModelSchemaResp extends ModelResp { private List dimensions; private List modelRelas; - public DimSchemaResp getPrimaryKey() { - Identify identify = getPrimaryIdentify(); - if (identify == null) { - return null; + public Set getModelClusterSet() { + if (CollectionUtils.isEmpty(this.modelRelas)) { + return Sets.newHashSet(); + } else { + Set modelClusterSet = new HashSet(); + this.modelRelas.forEach((modelRela) -> { + modelClusterSet.add(modelRela.getToModelId()); + modelClusterSet.add(modelRela.getFromModelId()); + }); + return modelClusterSet; } - for (DimSchemaResp dimension : dimensions) { - if (identify.getBizName().equals(dimension.getBizName())) { - dimension.setEntityAlias(identify.getEntityNames()); - return dimension; - } - } - return null; } } \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java index a3ed936a1..03504b989 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; -import org.apache.ibatis.annotations.Mapper; - +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import java.util.List; +import org.apache.ibatis.annotations.Mapper; @Mapper public interface DimensionDOCustomMapper { @@ -16,4 +16,7 @@ public interface DimensionDOCustomMapper { void batchUpdateStatus(List dimensionDOS); List query(DimensionFilter dimensionFilter); + + List queryDimensions(DimensionsFilter dimensionsFilter); + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java index 0c785847e..1c93f17ba 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.pojo.MetricFilter; -import org.apache.ibatis.annotations.Mapper; - +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; +import org.apache.ibatis.annotations.Mapper; @Mapper public interface MetricDOCustomMapper { @@ -15,4 +15,6 @@ public interface MetricDOCustomMapper { List query(MetricFilter metricFilter); + List queryMetrics(MetricsFilter metricsFilter); + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java index df1cf295b..84ee389e5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java @@ -4,6 +4,7 @@ package com.tencent.supersonic.headless.server.persistence.repository; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import java.util.List; public interface DimensionRepository { @@ -19,4 +20,6 @@ public interface DimensionRepository { DimensionDO getDimensionById(Long id); List getDimension(DimensionFilter dimensionFilter); + + List getDimensions(DimensionsFilter dimensionsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java index 36fc060f3..4ba881c94 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO; import com.tencent.supersonic.headless.server.pojo.MetricFilter; +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; public interface MetricRepository { @@ -21,6 +22,8 @@ public interface MetricRepository { List getMetric(MetricFilter metricFilter); + List getMetrics(MetricsFilter metricsFilter); + void saveDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO); void updateDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java index f609abfeb..392c5547f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java @@ -5,9 +5,9 @@ import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOCust import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; -import org.springframework.stereotype.Service; - +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import java.util.List; +import org.springframework.stereotype.Service; @Service public class DimensionRepositoryImpl implements DimensionRepository { @@ -52,4 +52,9 @@ public class DimensionRepositoryImpl implements DimensionRepository { return dimensionDOCustomMapper.query(dimensionFilter); } + @Override + public List getDimensions(DimensionsFilter dimensionsFilter) { + return dimensionDOCustomMapper.queryDimensions(dimensionsFilter); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java index cb8e5ae4d..3eabcf2b7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java @@ -8,9 +8,9 @@ import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper; import com.tencent.supersonic.headless.server.persistence.mapper.MetricQueryDefaultConfigDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.pojo.MetricFilter; -import org.springframework.stereotype.Component; - +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; +import org.springframework.stereotype.Component; @Component @@ -62,6 +62,11 @@ public class MetricRepositoryImpl implements MetricRepository { return metricDOCustomMapper.query(metricFilter); } + @Override + public List getMetrics(MetricsFilter metricsFilter) { + return metricDOCustomMapper.queryMetrics(metricsFilter); + } + @Override public void saveDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO) { metricQueryDefaultConfigDOMapper.insert(defaultConfigDO); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java new file mode 100644 index 000000000..e1e382967 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.headless.server.pojo; + +import java.util.List; +import lombok.Data; + +@Data +public class DimensionsFilter { + + private List modelIds; + + private List dimensionIds; + + private List dimensionNames; + +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java new file mode 100644 index 000000000..256555a83 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.headless.server.pojo; + +import java.util.List; +import lombok.Data; + +@Data +public class MetricsFilter { + + private List modelIds; + + private List metricIds; + + private List metricNames; + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java new file mode 100644 index 000000000..b329d6054 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java @@ -0,0 +1,60 @@ +package com.tencent.supersonic.headless.server.pojo; + +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Data +public class ModelCluster { + + private static final String split = "_"; + + private Set modelIds = new LinkedHashSet<>(); + + private Set modelNames = new LinkedHashSet<>(); + + private String key; + + private String name; + + public static ModelCluster build(Set modelIds) { + ModelCluster modelCluster = new ModelCluster(); + modelCluster.setModelIds(modelIds); + modelCluster.setKey(StringUtils.join(modelIds, split)); + return modelCluster; + } + + public static ModelCluster build(String key) { + ModelCluster modelCluster = new ModelCluster(); + modelCluster.setModelIds(getModelIdFromKey(key)); + modelCluster.setKey(key); + return modelCluster; + } + + public void buildName(Map modelNameMap) { + modelNames = modelNameMap.entrySet().stream().filter(entry -> + modelIds.contains(entry.getKey())).map(Map.Entry::getValue) + .collect(Collectors.toSet()); + name = String.join(split, modelNames); + } + + public static Set getModelIdFromKey(String key) { + return Arrays.stream(key.split(split)) + .map(Long::parseLong).collect(Collectors.toSet()); + } + + public Long getFirstModel() { + if (CollectionUtils.isEmpty(modelIds)) { + return -1L; + } + return new ArrayList<>(modelIds).get(0); + } + +} \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index c08001777..7c222d4f1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -54,7 +54,7 @@ public class QueryController { HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq, true); + QuerySqlReq querySqlReq = queryStructReq.convert(true); return queryService.queryByReq(querySqlReq, user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java index 2015def62..d5f88639a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DimensionService.java @@ -8,8 +8,8 @@ import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter; - import java.util.List; public interface DimensionService { @@ -30,6 +30,8 @@ public interface DimensionService { PageInfo queryDimension(PageDimensionReq pageDimensionReq); + List queryDimensions(DimensionsFilter dimensionsFilter); + void deleteDimension(Long id, User user); List getDimensionInModelCluster(Long modelId); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java index 4d710bd4c..d974851da 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/MetricService.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; - +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import java.util.List; import java.util.Set; @@ -50,4 +50,6 @@ public interface MetricService { MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user); void sendMetricEventBatch(List modelIds, EventType eventType); + + List queryMetrics(MetricsFilter metricsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java index fe03efb2c..fe7d9d5d0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/ModelService.java @@ -37,6 +37,8 @@ public interface ModelService { List getModelByDomainIds(List domainIds); + List getAllModelByDomainIds(List domainIds); + ModelResp getModel(Long id); List getModelAdmin(Long id); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java index 86a51237f..2cb90fdac 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SchemaService.java @@ -15,6 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemResp; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; @@ -26,6 +27,8 @@ public interface SchemaService { List fetchViewSchema(ViewFilterReq filter); + List fetchModelSchemaResps(List modelIds); + PageInfo queryDimension(PageDimensionReq pageDimensionReq, User user); PageInfo queryMetric(PageMetricReq pageMetricReq, User user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 990757a1c..e3182e93f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -29,6 +29,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.DimensionService; @@ -37,19 +38,18 @@ import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.NameCheckUtils; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Service @Slf4j @@ -217,6 +217,12 @@ public class DimensionServiceImpl implements DimensionService { return dimensionRepository.getDimension(dimensionFilter); } + @Override + public List queryDimensions(DimensionsFilter dimensionsFilter) { + List dimensions = dimensionRepository.getDimensions(dimensionsFilter); + return convertList(dimensions, modelService.getModelMap()); + } + @Override public List getDimensions(MetaFilter metaFilter) { DimensionFilter dimensionFilter = new DimensionFilter(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java index d0a62ed61..974670442 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java @@ -66,7 +66,7 @@ public class DownloadServiceImpl implements DownloadService { private QueryService queryService; public DownloadServiceImpl(MetricService metricService, - DimensionService dimensionService, QueryService queryService) { + DimensionService dimensionService, QueryService queryService) { this.metricService = metricService; this.dimensionService = dimensionService; this.queryService = queryService; @@ -74,11 +74,11 @@ public class DownloadServiceImpl implements DownloadService { @Override public void downloadByStruct(DownloadStructReq downloadStructReq, - User user, HttpServletResponse response) throws Exception { + User user, HttpServletResponse response) throws Exception { String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); File file = FileUtils.createTmpFile(fileName); try { - QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq, true); + QuerySqlReq querySqlReq = downloadStructReq.convert(true); SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user); DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq); EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData()); @@ -92,7 +92,7 @@ public class DownloadServiceImpl implements DownloadService { @Override public void batchDownload(BatchDownloadReq batchDownloadReq, User user, - HttpServletResponse response) throws Exception { + HttpServletResponse response) throws Exception { String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); File file = FileUtils.createTmpFile(fileName); List metricIds = batchDownloadReq.getMetricIds(); @@ -126,7 +126,7 @@ public class DownloadServiceImpl implements DownloadService { for (MetricResp metric : metrics) { try { DownloadStructReq downloadStructReq = buildDownloadReq(dimensions, metric, batchDownloadReq); - QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq); + QuerySqlReq querySqlReq = downloadStructReq.convert(); querySqlReq.setNeedAuth(true); SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user); DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq); @@ -192,7 +192,7 @@ public class DownloadServiceImpl implements DownloadService { } private List> buildData(List> headers, Map nameMap, - List> dataTransformed, String metricName) { + List> dataTransformed, String metricName) { List> data = Lists.newArrayList(); for (Map map : dataTransformed) { List row = Lists.newArrayList(); @@ -234,7 +234,7 @@ public class DownloadServiceImpl implements DownloadService { } private DownloadStructReq buildDownloadReq(List dimensionResps, MetricResp metricResp, - BatchDownloadReq batchDownloadReq) { + BatchDownloadReq batchDownloadReq) { DateConf dateConf = batchDownloadReq.getDateInfo(); Set modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); modelIds.add(metricResp.getModelId()); @@ -277,7 +277,7 @@ public class DownloadServiceImpl implements DownloadService { } private List getMetricRelaDimensions(MetricResp metricResp, - Map dimensionRespMap) { + Map dimensionRespMap) { if (metricResp.getRelateDimension() == null || CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) { return Lists.newArrayList(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index e026a6966..ac77d5c90 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -15,15 +15,14 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.ChatGptHelper; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq; import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq; -import com.tencent.supersonic.headless.api.pojo.response.DomainResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp; @@ -33,6 +32,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQuery import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.MetricFilter; +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import com.tencent.supersonic.headless.server.service.CollectService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; @@ -40,12 +40,7 @@ import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.MetricCheckUtils; import com.tencent.supersonic.headless.server.utils.MetricConverter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -53,6 +48,11 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Service @Slf4j @@ -184,9 +184,7 @@ public class MetricServiceImpl implements MetricService { public PageInfo queryMetric(PageMetricReq pageMetricReq, User user) { MetricFilter metricFilter = new MetricFilter(); BeanUtils.copyProperties(pageMetricReq, metricFilter); - Set domainResps = domainService.getDomainChildren(pageMetricReq.getDomainIds()); - List domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList()); - List modelResps = modelService.getModelByDomainIds(domainIds); + List modelResps = modelService.getAllModelByDomainIds(pageMetricReq.getDomainIds()); List modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()); pageMetricReq.getModelIds().addAll(modelIds); metricFilter.setModelIds(pageMetricReq.getModelIds()); @@ -435,6 +433,12 @@ public class MetricServiceImpl implements MetricService { sendEventBatch(metricDOS, eventType); } + @Override + public List queryMetrics(MetricsFilter metricsFilter) { + List metricDOS = metricRepository.getMetrics(metricsFilter); + return convertList(metricDOS, new ArrayList<>()); + } + private void sendEventBatch(List metricDOS, EventType eventType) { List dataItems = metricDOS.stream().map(this::getDataItem) .collect(Collectors.toList()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java index e9be8e689..f674656d7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java @@ -81,13 +81,13 @@ public class ModelServiceImpl implements ModelService { private DateInfoRepository dateInfoRepository; public ModelServiceImpl(ModelRepository modelRepository, - DatabaseService databaseService, - @Lazy DimensionService dimensionService, - @Lazy MetricService metricService, - DomainService domainService, - UserService userService, - ViewService viewService, - DateInfoRepository dateInfoRepository) { + DatabaseService databaseService, + @Lazy DimensionService dimensionService, + @Lazy MetricService metricService, + DomainService domainService, + UserService userService, + ViewService viewService, + DateInfoRepository dateInfoRepository) { this.modelRepository = modelRepository; this.databaseService = databaseService; this.dimensionService = dimensionService; @@ -350,6 +350,13 @@ public class ModelServiceImpl implements ModelService { domainIds.contains(modelResp.getDomainId())).collect(Collectors.toList()); } + @Override + public List getAllModelByDomainIds(List domainIds) { + Set domainResps = domainService.getDomainChildren(domainIds); + List allDomainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList()); + return getModelByDomainIds(allDomainIds); + } + @Override public ModelResp getModel(Long id) { ModelDO modelDO = getModelDO(id); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index cd0385b62..ba8a3d3fd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Item; import com.tencent.supersonic.headless.api.pojo.QueryParam; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; @@ -44,13 +45,24 @@ import com.tencent.supersonic.headless.server.annotation.S2DataPermission; import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; +import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; +import com.tencent.supersonic.headless.server.pojo.MetricsFilter; +import com.tencent.supersonic.headless.server.pojo.ModelCluster; import com.tencent.supersonic.headless.server.service.AppService; import com.tencent.supersonic.headless.server.service.Catalog; +import com.tencent.supersonic.headless.server.service.DimensionService; +import com.tencent.supersonic.headless.server.service.MetricService; +import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.QueryService; +import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder; import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -60,6 +72,7 @@ import javax.servlet.http.HttpServletRequest; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -79,6 +92,12 @@ public class QueryServiceImpl implements QueryService { private final QueryPlanner queryPlanner; + private final MetricService metricService; + + private final ModelService modelService; + + private final DimensionService dimensionService; + public QueryServiceImpl( StatUtils statUtils, QueryUtils queryUtils, @@ -88,7 +107,10 @@ public class QueryServiceImpl implements QueryService { QueryCache queryCache, SemanticSchemaManager semanticSchemaManager, DefaultQueryParser queryParser, - QueryPlanner queryPlanner) { + QueryPlanner queryPlanner, + MetricService metricService, + ModelService modelService, + DimensionService dimensionService) { this.statUtils = statUtils; this.queryUtils = queryUtils; this.queryReqConverter = queryReqConverter; @@ -98,6 +120,9 @@ public class QueryServiceImpl implements QueryService { this.semanticSchemaManager = semanticSchemaManager; this.queryParser = queryParser; this.queryPlanner = queryPlanner; + this.metricService = metricService; + this.modelService = modelService; + this.dimensionService = dimensionService; } @Override @@ -241,8 +266,130 @@ public class QueryServiceImpl implements QueryService { } @Override - public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception { - return null; + public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) { + QueryStructReq queryStructReq = buildQueryStructReq(queryMetricReq); + return queryByReq(queryStructReq.convert(), user); + } + + private QueryStructReq buildQueryStructReq(QueryMetricReq queryMetricReq) { + //1. If a domainId exists, the modelIds obtained from the domainId. + Set modelIdsByDomainId = getModelIdsByDomainId(queryMetricReq); + + //2. get metrics and dimensions + List metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId); + + List dimensionResps = getDimensionResps(queryMetricReq, modelIdsByDomainId); + + //3. choose ModelCluster + Set modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps); + ModelCluster modelCluster = getModelCluster(metricResps, modelIds); + + //4. set groups + List dimensionBizNames = dimensionResps.stream() + .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId())) + .map(entry -> entry.getBizName()).collect(Collectors.toList()); + + QueryStructReq queryStructReq = new QueryStructReq(); + if (CollectionUtils.isNotEmpty(dimensionBizNames)) { + queryStructReq.setGroups(dimensionBizNames); + } + //5. set aggregators + List metricBizNames = metricResps.stream() + .filter(entry -> modelCluster.getModelIds().contains(entry.getModelId())) + .map(entry -> entry.getBizName()).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(metricBizNames)) { + throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics"); + } + List aggregators = new ArrayList<>(); + for (String metricBizName : metricBizNames) { + Aggregator aggregator = new Aggregator(); + aggregator.setColumn(metricBizName); + aggregators.add(aggregator); + } + queryStructReq.setAggregators(aggregators); + queryStructReq.setLimit(queryMetricReq.getLimit()); + //6. set modelIds + for (Long modelId : modelCluster.getModelIds()) { + queryStructReq.addModelId(modelId); + } + //7. set dateInfo + queryStructReq.setDateInfo(queryMetricReq.getDateInfo()); + return queryStructReq; + } + + private QueryStructReq buildQueryStructReq(List dimensionResps, + MetricResp metricResp, DateConf dateConf, Long limit) { + Set modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); + modelIds.add(metricResp.getModelId()); + QueryStructReq queryStructReq = new QueryStructReq(); + queryStructReq.setGroups(dimensionResps.stream() + .map(DimensionResp::getBizName).collect(Collectors.toList())); + queryStructReq.getGroups().add(0, getTimeDimension(dateConf)); + Aggregator aggregator = new Aggregator(); + aggregator.setColumn(metricResp.getBizName()); + queryStructReq.setAggregators(Lists.newArrayList(aggregator)); + queryStructReq.setDateInfo(dateConf); + queryStructReq.setModelIds(modelIds); + queryStructReq.setLimit(limit); + return queryStructReq; + } + + private ModelCluster getModelCluster(List metricResps, Set modelIds) { + Map modelClusterMap = ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds)); + + Map> modelClusterToMatchCount = new HashMap<>(); + for (ModelCluster modelCluster : modelClusterMap.values()) { + for (MetricResp metricResp : metricResps) { + if (modelCluster.getModelIds().contains(metricResp.getModelId())) { + modelClusterToMatchCount.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>()) + .add(metricResp); + } + } + } + String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream() + .max(Comparator.comparingInt(entry -> entry.getValue().size())) + .map(Map.Entry::getKey) + .orElse(null); + + return modelClusterMap.get(keyWithMaxSize); + } + + private Set getModelIds(Set modelIdsByDomainId, List metricResps, + List dimensionResps) { + Set result = new HashSet<>(); + if (CollectionUtils.isNotEmpty(modelIdsByDomainId)) { + result.addAll(modelIdsByDomainId); + return result; + } + Set metricModelIds = metricResps.stream().map(entry -> entry.getModelId()) + .collect(Collectors.toSet()); + result.addAll(metricModelIds); + + Set dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId()) + .collect(Collectors.toSet()); + result.addAll(dimensionModelIds); + return result; + } + + private List getDimensionResps(QueryMetricReq queryMetricReq, Set modelIds) { + DimensionsFilter dimensionsFilter = new DimensionsFilter(); + BeanUtils.copyProperties(queryMetricReq, dimensionsFilter); + dimensionsFilter.setModelIds(new ArrayList<>(modelIds)); + List dimensionResps = dimensionService.queryDimensions(dimensionsFilter); + return dimensionResps; + } + + private List getMetricResps(QueryMetricReq queryMetricReq, Set modelIds) { + MetricsFilter metricsFilter = new MetricsFilter(); + BeanUtils.copyProperties(queryMetricReq, metricsFilter); + metricsFilter.setModelIds(new ArrayList<>(modelIds)); + return metricService.queryMetrics(metricsFilter); + } + + private Set getModelIdsByDomainId(QueryMetricReq queryMetricReq) { + List modelResps = modelService.getAllModelByDomainIds( + Collections.singletonList(queryMetricReq.getDomainId())); + return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet()); } private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception { @@ -271,23 +418,6 @@ public class QueryServiceImpl implements QueryService { return appService.getApp(appId); } - private QueryStructReq buildQueryStructReq(List dimensionResps, - MetricResp metricResp, DateConf dateConf, Long limit) { - Set modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet()); - modelIds.add(metricResp.getModelId()); - QueryStructReq queryStructReq = new QueryStructReq(); - queryStructReq.setGroups(dimensionResps.stream() - .map(DimensionResp::getBizName).collect(Collectors.toList())); - queryStructReq.getGroups().add(0, getTimeDimension(dateConf)); - Aggregator aggregator = new Aggregator(); - aggregator.setColumn(metricResp.getBizName()); - queryStructReq.setAggregators(Lists.newArrayList(aggregator)); - queryStructReq.setDateInfo(dateConf); - queryStructReq.setModelIds(modelIds); - queryStructReq.setLimit(limit); - return queryStructReq; - } - private String getTimeDimension(DateConf dateConf) { if (Constants.MONTH.equals(dateConf.getPeriod())) { return TimeDimensionEnum.MONTH.getName(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 4de9272a4..8c69a0cf4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -1,5 +1,10 @@ package com.tencent.supersonic.headless.server.utils; +import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; +import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; +import static com.tencent.supersonic.common.pojo.Constants.COMMA; +import static com.tencent.supersonic.common.pojo.Constants.SPACE; + import com.google.common.base.Strings; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; @@ -31,11 +36,6 @@ import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.QueryService; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -48,11 +48,10 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.StringJoiner; - -import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; -import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; -import static com.tencent.supersonic.common.pojo.Constants.COMMA; -import static com.tencent.supersonic.common.pojo.Constants.SPACE; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; @Component public class DictUtils { @@ -79,9 +78,9 @@ public class DictUtils { private final ModelService modelService; public DictUtils(DimensionService dimensionService, - MetricService metricService, - QueryService queryService, - ModelService modelService) { + MetricService metricService, + QueryService queryService, + ModelService modelService) { this.dimensionService = dimensionService; this.metricService = metricService; this.queryService = queryService; @@ -222,7 +221,7 @@ public class DictUtils { && Objects.nonNull(dictItemResp.getConfig().getMetricId())) { // 查询默认指标 QueryStructReq queryStructReq = generateQueryStruct(dictItemResp); - return queryStructReq.convert(queryStructReq, true); + return queryStructReq.convert(true); } // count(1) 作为指标 return constructQuerySqlReq(dictItemResp); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java new file mode 100644 index 000000000..161d34889 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelClusterBuilder.java @@ -0,0 +1,47 @@ +package com.tencent.supersonic.headless.server.utils; + + +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; +import com.tencent.supersonic.headless.server.pojo.ModelCluster; +import com.tencent.supersonic.headless.server.service.SchemaService; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class ModelClusterBuilder { + + public static Map buildModelClusters(List modelIds) { + SchemaService schemaService = ContextUtils.getBean(SchemaService.class); + List modelSchemaResps = schemaService.fetchModelSchemaResps(modelIds); + Map modelIdToModelSchema = modelSchemaResps.stream() + .collect(Collectors.toMap(ModelSchemaResp::getId, value -> value, (k1, k2) -> k1)); + + Set visited = new HashSet<>(); + List> modelClusters = new ArrayList<>(); + for (ModelSchemaResp model : modelSchemaResps) { + if (!visited.contains(model.getId())) { + Set modelCluster = new HashSet<>(); + dfs(model, modelIdToModelSchema, visited, modelCluster); + modelClusters.add(modelCluster); + } + } + return modelClusters.stream().map(ModelCluster::build) + .collect(Collectors.toMap(ModelCluster::getKey, value -> value, (k1, k2) -> k1)); + } + + private static void dfs(ModelSchemaResp model, Map modelMap, + Set visited, Set modelCluster) { + visited.add(model.getId()); + modelCluster.add(model.getId()); + for (Long neighborId : model.getModelClusterSet()) { + if (!visited.contains(neighborId)) { + dfs(modelMap.get(neighborId), modelMap, visited, modelCluster); + } + } + } + +} \ No newline at end of file diff --git a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index d9fda8391..4f82ebd56 100644 --- a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -167,4 +167,40 @@ + + + diff --git a/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml index 3a8fca6b6..ba3eede7d 100644 --- a/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml @@ -2,27 +2,29 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - + + @@ -56,14 +58,16 @@ - id, model_id, name, biz_name, description, status, sensitive_level, type, created_at, + id + , model_id, name, biz_name, description, status, sensitive_level, type, created_at, created_by, updated_at, updated_by, data_format_type, data_format, alias, tags, define_type type_params - + insert into s2_metric (model_id, name, biz_name, description, type,status,sensitive_level, created_at, created_by, updated_at, @@ -94,9 +98,9 @@ + + + diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java index 240dc7e28..d4861b53d 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java @@ -23,7 +23,6 @@ import com.tencent.supersonic.common.pojo.SysParameter; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.service.SysParameterService; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.server.service.KnowledgeService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -53,8 +52,6 @@ public class ChatDemoLoader implements CommandLineRunner { private AgentService agentService; @Autowired private SysParameterService sysParameterService; - @Autowired - private KnowledgeService knowledgeService; @Value("${demo.enabled:false}") private boolean demoEnabled; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java new file mode 100644 index 000000000..f8aac7726 --- /dev/null +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java @@ -0,0 +1,62 @@ +package com.tencent.supersonic.headless; + +import static org.junit.Assert.assertThrows; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq; +import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import java.util.Arrays; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +public class QueryByMetricTest extends BaseTest { + + @Test + public void testWithMetricAndDimensionBizNames() throws Exception { + QueryMetricReq queryMetricReq = new QueryMetricReq(); + queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv")); + queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department")); + SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser()); + Assert.assertNotNull(queryResp.getResultList()); + Assert.assertEquals(6, queryResp.getResultList().size()); + } + + @Test + public void testWithMetricAndDimensionNames() throws Exception { + QueryMetricReq queryMetricReq = new QueryMetricReq(); + queryMetricReq.setMetricNames(Arrays.asList("停留时长", "访问次数")); + queryMetricReq.setDimensionNames(Arrays.asList("用户", "部门")); + SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser()); + Assert.assertNotNull(queryResp.getResultList()); + Assert.assertEquals(6, queryResp.getResultList().size()); + } + + @Test + public void testWithDomainId() throws Exception { + QueryMetricReq queryMetricReq = new QueryMetricReq(); + queryMetricReq.setDomainId(1L); + queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv")); + queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department")); + SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser()); + Assert.assertNotNull(queryResp.getResultList()); + Assert.assertEquals(6, queryResp.getResultList().size()); + + queryMetricReq.setDomainId(2L); + queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv")); + queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department")); + assertThrows(IllegalArgumentException.class, + () -> queryService.queryByMetric(queryMetricReq, User.getFakeUser())); + } + + @Test + public void testWithMetricAndDimensionIds() throws Exception { + QueryMetricReq queryMetricReq = new QueryMetricReq(); + queryMetricReq.setDomainId(1L); + queryMetricReq.setMetricIds(Arrays.asList(1L, 4L)); + queryMetricReq.setDimensionIds(Arrays.asList(1L, 2L)); + SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser()); + Assert.assertNotNull(queryResp.getResultList()); + Assert.assertEquals(6, queryResp.getResultList().size()); + } + +}