mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-11 03:58:14 +00:00
(improvement)(chat) Adding the Metric API to Headless. (#738)
This commit is contained in:
@@ -2,16 +2,14 @@ package com.tencent.supersonic.chat.api.pojo;
|
||||
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
public class SemanticSchema implements Serializable {
|
||||
|
||||
@@ -54,35 +52,6 @@ public class SemanticSchema implements Serializable {
|
||||
}
|
||||
}
|
||||
|
||||
public SchemaElement getElementByName(SchemaElementType elementType, String name) {
|
||||
Optional<SchemaElement> element = Optional.empty();
|
||||
|
||||
switch (elementType) {
|
||||
case ENTITY:
|
||||
element = getElementsByNameOrAlias(name, getEntities());
|
||||
break;
|
||||
case VIEW:
|
||||
element = getElementsByNameOrAlias(name, getViews());
|
||||
break;
|
||||
case METRIC:
|
||||
element = getElementsByNameOrAlias(name, getMetrics());
|
||||
break;
|
||||
case DIMENSION:
|
||||
element = getElementsByNameOrAlias(name, getDimensions());
|
||||
break;
|
||||
case VALUE:
|
||||
element = getElementsByNameOrAlias(name, getDimensionValues());
|
||||
break;
|
||||
default:
|
||||
}
|
||||
|
||||
if (element.isPresent()) {
|
||||
return element.get();
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public Map<Long, String> getViewIdToName() {
|
||||
return viewSchemaList.stream()
|
||||
.collect(Collectors.toMap(a -> a.getView().getId(), a -> a.getView().getName(), (k1, k2) -> k1));
|
||||
@@ -159,14 +128,6 @@ public class SemanticSchema implements Serializable {
|
||||
.findFirst();
|
||||
}
|
||||
|
||||
private Optional<SchemaElement> getElementsByNameOrAlias(String name, List<SchemaElement> elements) {
|
||||
return elements.stream()
|
||||
.filter(schemaElement ->
|
||||
name.equals(schemaElement.getName()) || (Objects.nonNull(schemaElement.getAlias())
|
||||
&& schemaElement.getAlias().contains(name))
|
||||
).findFirst();
|
||||
}
|
||||
|
||||
public SchemaElement getView(Long viewId) {
|
||||
List<SchemaElement> views = getViews();
|
||||
return getElementsById(viewId, views).orElse(null);
|
||||
|
||||
@@ -121,7 +121,7 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
|
||||
}
|
||||
QueryStructReq queryStructReq = convertQueryStruct();
|
||||
convertBizNameToName(semanticSchema, queryStructReq);
|
||||
QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq);
|
||||
QuerySqlReq querySQLReq = queryStructReq.convert();
|
||||
parseInfo.getSqlInfo().setS2SQL(querySQLReq.getSql());
|
||||
parseInfo.getSqlInfo().setCorrectS2SQL(querySQLReq.getSql());
|
||||
}
|
||||
|
||||
@@ -52,14 +52,14 @@ class QueryReqBuilderTest {
|
||||
orders.add(order);
|
||||
queryStructReq.setOrders(orders);
|
||||
|
||||
QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq);
|
||||
QuerySqlReq querySQLReq = queryStructReq.convert();
|
||||
Assert.assertEquals(
|
||||
"SELECT department, SUM(pv) AS pv FROM 内容库 "
|
||||
+ "WHERE (sys_imp_date IN ('2023-08-01')) GROUP "
|
||||
+ "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql());
|
||||
|
||||
queryStructReq.setQueryType(QueryType.TAG);
|
||||
querySQLReq = queryStructReq.convert(queryStructReq);
|
||||
querySQLReq = queryStructReq.convert();
|
||||
Assert.assertEquals(
|
||||
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
|
||||
+ "ORDER BY uv LIMIT 2000",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.tencent.supersonic.headless.api.pojo.request;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
@@ -18,4 +19,8 @@ public class QueryMetricReq {
|
||||
|
||||
private List<String> dimensionNames;
|
||||
|
||||
private DateConf dateInfo = new DateConf();
|
||||
|
||||
private Long limit = 2000L;
|
||||
|
||||
}
|
||||
@@ -44,6 +44,7 @@ import java.util.stream.Collectors;
|
||||
@Data
|
||||
@Slf4j
|
||||
public class QueryStructReq extends SemanticQueryReq {
|
||||
|
||||
private List<String> groups = new ArrayList<>();
|
||||
private List<Aggregator> aggregators = new ArrayList<>();
|
||||
private List<Order> orders = new ArrayList<>();
|
||||
@@ -151,28 +152,27 @@ public class QueryStructReq extends SemanticQueryReq {
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public QuerySqlReq convert(QueryStructReq queryStructReq) {
|
||||
return convert(queryStructReq, false);
|
||||
public QuerySqlReq convert() {
|
||||
return convert(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* convert queryStructReq to QueryS2SQLReq
|
||||
*
|
||||
* @param queryStructReq
|
||||
* @return
|
||||
*/
|
||||
public QuerySqlReq convert(QueryStructReq queryStructReq, boolean isBizName) {
|
||||
public QuerySqlReq convert(boolean isBizName) {
|
||||
String sql = null;
|
||||
try {
|
||||
sql = buildSql(queryStructReq, isBizName);
|
||||
sql = buildSql(this, isBizName);
|
||||
} catch (Exception e) {
|
||||
log.error("buildSql error", e);
|
||||
}
|
||||
|
||||
QuerySqlReq result = new QuerySqlReq();
|
||||
result.setSql(sql);
|
||||
result.setViewId(queryStructReq.getViewId());
|
||||
result.setModelIds(queryStructReq.getModelIdSet());
|
||||
result.setViewId(this.getViewId());
|
||||
result.setModelIds(this.getModelIdSet());
|
||||
result.setParams(new ArrayList<>());
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
package com.tencent.supersonic.headless.api.pojo.response;
|
||||
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.common.pojo.ModelRela;
|
||||
import com.tencent.supersonic.headless.api.pojo.Identify;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@@ -17,18 +19,17 @@ public class ModelSchemaResp extends ModelResp {
|
||||
private List<DimSchemaResp> dimensions;
|
||||
private List<ModelRela> modelRelas;
|
||||
|
||||
public DimSchemaResp getPrimaryKey() {
|
||||
Identify identify = getPrimaryIdentify();
|
||||
if (identify == null) {
|
||||
return null;
|
||||
public Set<Long> getModelClusterSet() {
|
||||
if (CollectionUtils.isEmpty(this.modelRelas)) {
|
||||
return Sets.newHashSet();
|
||||
} else {
|
||||
Set<Long> modelClusterSet = new HashSet();
|
||||
this.modelRelas.forEach((modelRela) -> {
|
||||
modelClusterSet.add(modelRela.getToModelId());
|
||||
modelClusterSet.add(modelRela.getFromModelId());
|
||||
});
|
||||
return modelClusterSet;
|
||||
}
|
||||
for (DimSchemaResp dimension : dimensions) {
|
||||
if (identify.getBizName().equals(dimension.getBizName())) {
|
||||
dimension.setEntityAlias(identify.getEntityNames());
|
||||
return dimension;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,9 +2,9 @@ package com.tencent.supersonic.headless.server.persistence.mapper;
|
||||
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import java.util.List;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface DimensionDOCustomMapper {
|
||||
@@ -16,4 +16,7 @@ public interface DimensionDOCustomMapper {
|
||||
void batchUpdateStatus(List<DimensionDO> dimensionDOS);
|
||||
|
||||
List<DimensionDO> query(DimensionFilter dimensionFilter);
|
||||
|
||||
List<DimensionDO> queryDimensions(DimensionsFilter dimensionsFilter);
|
||||
|
||||
}
|
||||
|
||||
@@ -2,9 +2,9 @@ package com.tencent.supersonic.headless.server.persistence.mapper;
|
||||
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import java.util.List;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface MetricDOCustomMapper {
|
||||
@@ -15,4 +15,6 @@ public interface MetricDOCustomMapper {
|
||||
|
||||
List<MetricDO> query(MetricFilter metricFilter);
|
||||
|
||||
List<MetricDO> queryMetrics(MetricsFilter metricsFilter);
|
||||
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ package com.tencent.supersonic.headless.server.persistence.repository;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import java.util.List;
|
||||
|
||||
public interface DimensionRepository {
|
||||
@@ -19,4 +20,6 @@ public interface DimensionRepository {
|
||||
DimensionDO getDimensionById(Long id);
|
||||
|
||||
List<DimensionDO> getDimension(DimensionFilter dimensionFilter);
|
||||
|
||||
List<DimensionDO> getDimensions(DimensionsFilter dimensionsFilter);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
|
||||
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import java.util.List;
|
||||
|
||||
public interface MetricRepository {
|
||||
@@ -21,6 +22,8 @@ public interface MetricRepository {
|
||||
|
||||
List<MetricDO> getMetric(MetricFilter metricFilter);
|
||||
|
||||
List<MetricDO> getMetrics(MetricsFilter metricsFilter);
|
||||
|
||||
void saveDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO);
|
||||
|
||||
void updateDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO);
|
||||
|
||||
@@ -5,9 +5,9 @@ import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOCust
|
||||
import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import java.util.List;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class DimensionRepositoryImpl implements DimensionRepository {
|
||||
@@ -52,4 +52,9 @@ public class DimensionRepositoryImpl implements DimensionRepository {
|
||||
return dimensionDOCustomMapper.query(dimensionFilter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DimensionDO> getDimensions(DimensionsFilter dimensionsFilter) {
|
||||
return dimensionDOCustomMapper.queryDimensions(dimensionsFilter);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -8,9 +8,9 @@ import com.tencent.supersonic.headless.server.persistence.mapper.MetricDOMapper;
|
||||
import com.tencent.supersonic.headless.server.persistence.mapper.MetricQueryDefaultConfigDOMapper;
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import java.util.List;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
|
||||
@Component
|
||||
@@ -62,6 +62,11 @@ public class MetricRepositoryImpl implements MetricRepository {
|
||||
return metricDOCustomMapper.query(metricFilter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<MetricDO> getMetrics(MetricsFilter metricsFilter) {
|
||||
return metricDOCustomMapper.queryMetrics(metricsFilter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveDefaultQueryConfig(MetricQueryDefaultConfigDO defaultConfigDO) {
|
||||
metricQueryDefaultConfigDOMapper.insert(defaultConfigDO);
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.tencent.supersonic.headless.server.pojo;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class DimensionsFilter {
|
||||
|
||||
private List<Long> modelIds;
|
||||
|
||||
private List<Long> dimensionIds;
|
||||
|
||||
private List<String> dimensionNames;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.tencent.supersonic.headless.server.pojo;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class MetricsFilter {
|
||||
|
||||
private List<Long> modelIds;
|
||||
|
||||
private List<Long> metricIds;
|
||||
|
||||
private List<String> metricNames;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package com.tencent.supersonic.headless.server.pojo;
|
||||
|
||||
import lombok.Data;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Data
|
||||
public class ModelCluster {
|
||||
|
||||
private static final String split = "_";
|
||||
|
||||
private Set<Long> modelIds = new LinkedHashSet<>();
|
||||
|
||||
private Set<String> modelNames = new LinkedHashSet<>();
|
||||
|
||||
private String key;
|
||||
|
||||
private String name;
|
||||
|
||||
public static ModelCluster build(Set<Long> modelIds) {
|
||||
ModelCluster modelCluster = new ModelCluster();
|
||||
modelCluster.setModelIds(modelIds);
|
||||
modelCluster.setKey(StringUtils.join(modelIds, split));
|
||||
return modelCluster;
|
||||
}
|
||||
|
||||
public static ModelCluster build(String key) {
|
||||
ModelCluster modelCluster = new ModelCluster();
|
||||
modelCluster.setModelIds(getModelIdFromKey(key));
|
||||
modelCluster.setKey(key);
|
||||
return modelCluster;
|
||||
}
|
||||
|
||||
public void buildName(Map<Long, String> modelNameMap) {
|
||||
modelNames = modelNameMap.entrySet().stream().filter(entry ->
|
||||
modelIds.contains(entry.getKey())).map(Map.Entry::getValue)
|
||||
.collect(Collectors.toSet());
|
||||
name = String.join(split, modelNames);
|
||||
}
|
||||
|
||||
public static Set<Long> getModelIdFromKey(String key) {
|
||||
return Arrays.stream(key.split(split))
|
||||
.map(Long::parseLong).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public Long getFirstModel() {
|
||||
if (CollectionUtils.isEmpty(modelIds)) {
|
||||
return -1L;
|
||||
}
|
||||
return new ArrayList<>(modelIds).get(0);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -54,7 +54,7 @@ public class QueryController {
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq, true);
|
||||
QuerySqlReq querySqlReq = queryStructReq.convert(true);
|
||||
return queryService.queryByReq(querySqlReq, user);
|
||||
}
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@ import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface DimensionService {
|
||||
@@ -30,6 +30,8 @@ public interface DimensionService {
|
||||
|
||||
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq);
|
||||
|
||||
List<DimensionResp> queryDimensions(DimensionsFilter dimensionsFilter);
|
||||
|
||||
void deleteDimension(Long id, User user);
|
||||
|
||||
List<DimensionResp> getDimensionInModelCluster(Long modelId);
|
||||
|
||||
@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@@ -50,4 +50,6 @@ public interface MetricService {
|
||||
MetricQueryDefaultConfig getMetricQueryDefaultConfig(Long metricId, User user);
|
||||
|
||||
void sendMetricEventBatch(List<Long> modelIds, EventType eventType);
|
||||
|
||||
List<MetricResp> queryMetrics(MetricsFilter metricsFilter);
|
||||
}
|
||||
|
||||
@@ -37,6 +37,8 @@ public interface ModelService {
|
||||
|
||||
List<ModelResp> getModelByDomainIds(List<Long> domainIds);
|
||||
|
||||
List<ModelResp> getAllModelByDomainIds(List<Long> domainIds);
|
||||
|
||||
ModelResp getModel(Long id);
|
||||
|
||||
List<String> getModelAdmin(Long id);
|
||||
|
||||
@@ -15,6 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp;
|
||||
@@ -26,6 +27,8 @@ public interface SchemaService {
|
||||
|
||||
List<ViewSchemaResp> fetchViewSchema(ViewFilterReq filter);
|
||||
|
||||
List<ModelSchemaResp> fetchModelSchemaResps(List<Long> modelIds);
|
||||
|
||||
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq, User user);
|
||||
|
||||
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
|
||||
|
||||
@@ -29,6 +29,7 @@ import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
|
||||
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DatabaseService;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
@@ -37,19 +38,18 @@ import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.service.ViewService;
|
||||
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@@ -217,6 +217,12 @@ public class DimensionServiceImpl implements DimensionService {
|
||||
return dimensionRepository.getDimension(dimensionFilter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DimensionResp> queryDimensions(DimensionsFilter dimensionsFilter) {
|
||||
List<DimensionDO> dimensions = dimensionRepository.getDimensions(dimensionsFilter);
|
||||
return convertList(dimensions, modelService.getModelMap());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DimensionResp> getDimensions(MetaFilter metaFilter) {
|
||||
DimensionFilter dimensionFilter = new DimensionFilter();
|
||||
|
||||
@@ -66,7 +66,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
private QueryService queryService;
|
||||
|
||||
public DownloadServiceImpl(MetricService metricService,
|
||||
DimensionService dimensionService, QueryService queryService) {
|
||||
DimensionService dimensionService, QueryService queryService) {
|
||||
this.metricService = metricService;
|
||||
this.dimensionService = dimensionService;
|
||||
this.queryService = queryService;
|
||||
@@ -74,11 +74,11 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
|
||||
@Override
|
||||
public void downloadByStruct(DownloadStructReq downloadStructReq,
|
||||
User user, HttpServletResponse response) throws Exception {
|
||||
User user, HttpServletResponse response) throws Exception {
|
||||
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
|
||||
File file = FileUtils.createTmpFile(fileName);
|
||||
try {
|
||||
QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq, true);
|
||||
QuerySqlReq querySqlReq = downloadStructReq.convert(true);
|
||||
SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user);
|
||||
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
|
||||
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
|
||||
@@ -92,7 +92,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
|
||||
@Override
|
||||
public void batchDownload(BatchDownloadReq batchDownloadReq, User user,
|
||||
HttpServletResponse response) throws Exception {
|
||||
HttpServletResponse response) throws Exception {
|
||||
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
|
||||
File file = FileUtils.createTmpFile(fileName);
|
||||
List<Long> metricIds = batchDownloadReq.getMetricIds();
|
||||
@@ -126,7 +126,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
for (MetricResp metric : metrics) {
|
||||
try {
|
||||
DownloadStructReq downloadStructReq = buildDownloadReq(dimensions, metric, batchDownloadReq);
|
||||
QuerySqlReq querySqlReq = downloadStructReq.convert(downloadStructReq);
|
||||
QuerySqlReq querySqlReq = downloadStructReq.convert();
|
||||
querySqlReq.setNeedAuth(true);
|
||||
SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user);
|
||||
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
|
||||
@@ -192,7 +192,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
}
|
||||
|
||||
private List<List<String>> buildData(List<List<String>> headers, Map<String, String> nameMap,
|
||||
List<Map<String, Object>> dataTransformed, String metricName) {
|
||||
List<Map<String, Object>> dataTransformed, String metricName) {
|
||||
List<List<String>> data = Lists.newArrayList();
|
||||
for (Map<String, Object> map : dataTransformed) {
|
||||
List<String> row = Lists.newArrayList();
|
||||
@@ -234,7 +234,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
}
|
||||
|
||||
private DownloadStructReq buildDownloadReq(List<DimensionResp> dimensionResps, MetricResp metricResp,
|
||||
BatchDownloadReq batchDownloadReq) {
|
||||
BatchDownloadReq batchDownloadReq) {
|
||||
DateConf dateConf = batchDownloadReq.getDateInfo();
|
||||
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
|
||||
modelIds.add(metricResp.getModelId());
|
||||
@@ -277,7 +277,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
}
|
||||
|
||||
private List<DimensionResp> getMetricRelaDimensions(MetricResp metricResp,
|
||||
Map<Long, DimensionResp> dimensionRespMap) {
|
||||
Map<Long, DimensionResp> dimensionRespMap) {
|
||||
if (metricResp.getRelateDimension() == null
|
||||
|| CollectionUtils.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) {
|
||||
return Lists.newArrayList();
|
||||
|
||||
@@ -15,15 +15,14 @@ import com.tencent.supersonic.common.pojo.enums.StatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.common.util.BeanMapper;
|
||||
import com.tencent.supersonic.common.util.ChatGptHelper;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
|
||||
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
|
||||
@@ -33,6 +32,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQuery
|
||||
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import com.tencent.supersonic.headless.server.service.CollectService;
|
||||
import com.tencent.supersonic.headless.server.service.DomainService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
@@ -40,12 +40,7 @@ import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.service.ViewService;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.MetricConverter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@@ -53,6 +48,11 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@@ -184,9 +184,7 @@ public class MetricServiceImpl implements MetricService {
|
||||
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user) {
|
||||
MetricFilter metricFilter = new MetricFilter();
|
||||
BeanUtils.copyProperties(pageMetricReq, metricFilter);
|
||||
Set<DomainResp> domainResps = domainService.getDomainChildren(pageMetricReq.getDomainIds());
|
||||
List<Long> domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList());
|
||||
List<ModelResp> modelResps = modelService.getModelByDomainIds(domainIds);
|
||||
List<ModelResp> modelResps = modelService.getAllModelByDomainIds(pageMetricReq.getDomainIds());
|
||||
List<Long> modelIds = modelResps.stream().map(ModelResp::getId).collect(Collectors.toList());
|
||||
pageMetricReq.getModelIds().addAll(modelIds);
|
||||
metricFilter.setModelIds(pageMetricReq.getModelIds());
|
||||
@@ -435,6 +433,12 @@ public class MetricServiceImpl implements MetricService {
|
||||
sendEventBatch(metricDOS, eventType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<MetricResp> queryMetrics(MetricsFilter metricsFilter) {
|
||||
List<MetricDO> metricDOS = metricRepository.getMetrics(metricsFilter);
|
||||
return convertList(metricDOS, new ArrayList<>());
|
||||
}
|
||||
|
||||
private void sendEventBatch(List<MetricDO> metricDOS, EventType eventType) {
|
||||
List<DataItem> dataItems = metricDOS.stream().map(this::getDataItem)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
@@ -81,13 +81,13 @@ public class ModelServiceImpl implements ModelService {
|
||||
private DateInfoRepository dateInfoRepository;
|
||||
|
||||
public ModelServiceImpl(ModelRepository modelRepository,
|
||||
DatabaseService databaseService,
|
||||
@Lazy DimensionService dimensionService,
|
||||
@Lazy MetricService metricService,
|
||||
DomainService domainService,
|
||||
UserService userService,
|
||||
ViewService viewService,
|
||||
DateInfoRepository dateInfoRepository) {
|
||||
DatabaseService databaseService,
|
||||
@Lazy DimensionService dimensionService,
|
||||
@Lazy MetricService metricService,
|
||||
DomainService domainService,
|
||||
UserService userService,
|
||||
ViewService viewService,
|
||||
DateInfoRepository dateInfoRepository) {
|
||||
this.modelRepository = modelRepository;
|
||||
this.databaseService = databaseService;
|
||||
this.dimensionService = dimensionService;
|
||||
@@ -350,6 +350,13 @@ public class ModelServiceImpl implements ModelService {
|
||||
domainIds.contains(modelResp.getDomainId())).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ModelResp> getAllModelByDomainIds(List<Long> domainIds) {
|
||||
Set<DomainResp> domainResps = domainService.getDomainChildren(domainIds);
|
||||
List<Long> allDomainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList());
|
||||
return getModelByDomainIds(allDomainIds);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ModelResp getModel(Long id) {
|
||||
ModelDO modelDO = getModelDO(id);
|
||||
|
||||
@@ -13,6 +13,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.headless.api.pojo.Dim;
|
||||
import com.tencent.supersonic.headless.api.pojo.Item;
|
||||
import com.tencent.supersonic.headless.api.pojo.QueryParam;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
|
||||
@@ -44,13 +45,24 @@ import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
|
||||
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
|
||||
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
|
||||
import com.tencent.supersonic.headless.server.service.AppService;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.service.QueryService;
|
||||
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.StatUtils;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
@@ -60,6 +72,7 @@ import javax.servlet.http.HttpServletRequest;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
|
||||
@@ -79,6 +92,12 @@ public class QueryServiceImpl implements QueryService {
|
||||
|
||||
private final QueryPlanner queryPlanner;
|
||||
|
||||
private final MetricService metricService;
|
||||
|
||||
private final ModelService modelService;
|
||||
|
||||
private final DimensionService dimensionService;
|
||||
|
||||
public QueryServiceImpl(
|
||||
StatUtils statUtils,
|
||||
QueryUtils queryUtils,
|
||||
@@ -88,7 +107,10 @@ public class QueryServiceImpl implements QueryService {
|
||||
QueryCache queryCache,
|
||||
SemanticSchemaManager semanticSchemaManager,
|
||||
DefaultQueryParser queryParser,
|
||||
QueryPlanner queryPlanner) {
|
||||
QueryPlanner queryPlanner,
|
||||
MetricService metricService,
|
||||
ModelService modelService,
|
||||
DimensionService dimensionService) {
|
||||
this.statUtils = statUtils;
|
||||
this.queryUtils = queryUtils;
|
||||
this.queryReqConverter = queryReqConverter;
|
||||
@@ -98,6 +120,9 @@ public class QueryServiceImpl implements QueryService {
|
||||
this.semanticSchemaManager = semanticSchemaManager;
|
||||
this.queryParser = queryParser;
|
||||
this.queryPlanner = queryPlanner;
|
||||
this.metricService = metricService;
|
||||
this.modelService = modelService;
|
||||
this.dimensionService = dimensionService;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -241,8 +266,130 @@ public class QueryServiceImpl implements QueryService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception {
|
||||
return null;
|
||||
public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) {
|
||||
QueryStructReq queryStructReq = buildQueryStructReq(queryMetricReq);
|
||||
return queryByReq(queryStructReq.convert(), user);
|
||||
}
|
||||
|
||||
private QueryStructReq buildQueryStructReq(QueryMetricReq queryMetricReq) {
|
||||
//1. If a domainId exists, the modelIds obtained from the domainId.
|
||||
Set<Long> modelIdsByDomainId = getModelIdsByDomainId(queryMetricReq);
|
||||
|
||||
//2. get metrics and dimensions
|
||||
List<MetricResp> metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId);
|
||||
|
||||
List<DimensionResp> dimensionResps = getDimensionResps(queryMetricReq, modelIdsByDomainId);
|
||||
|
||||
//3. choose ModelCluster
|
||||
Set<Long> modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps);
|
||||
ModelCluster modelCluster = getModelCluster(metricResps, modelIds);
|
||||
|
||||
//4. set groups
|
||||
List<String> dimensionBizNames = dimensionResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.map(entry -> entry.getBizName()).collect(Collectors.toList());
|
||||
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
if (CollectionUtils.isNotEmpty(dimensionBizNames)) {
|
||||
queryStructReq.setGroups(dimensionBizNames);
|
||||
}
|
||||
//5. set aggregators
|
||||
List<String> metricBizNames = metricResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.map(entry -> entry.getBizName()).collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(metricBizNames)) {
|
||||
throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
|
||||
}
|
||||
List<Aggregator> aggregators = new ArrayList<>();
|
||||
for (String metricBizName : metricBizNames) {
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricBizName);
|
||||
aggregators.add(aggregator);
|
||||
}
|
||||
queryStructReq.setAggregators(aggregators);
|
||||
queryStructReq.setLimit(queryMetricReq.getLimit());
|
||||
//6. set modelIds
|
||||
for (Long modelId : modelCluster.getModelIds()) {
|
||||
queryStructReq.addModelId(modelId);
|
||||
}
|
||||
//7. set dateInfo
|
||||
queryStructReq.setDateInfo(queryMetricReq.getDateInfo());
|
||||
return queryStructReq;
|
||||
}
|
||||
|
||||
private QueryStructReq buildQueryStructReq(List<DimensionResp> dimensionResps,
|
||||
MetricResp metricResp, DateConf dateConf, Long limit) {
|
||||
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
|
||||
modelIds.add(metricResp.getModelId());
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setGroups(dimensionResps.stream()
|
||||
.map(DimensionResp::getBizName).collect(Collectors.toList()));
|
||||
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricResp.getBizName());
|
||||
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
|
||||
queryStructReq.setDateInfo(dateConf);
|
||||
queryStructReq.setModelIds(modelIds);
|
||||
queryStructReq.setLimit(limit);
|
||||
return queryStructReq;
|
||||
}
|
||||
|
||||
private ModelCluster getModelCluster(List<MetricResp> metricResps, Set<Long> modelIds) {
|
||||
Map<String, ModelCluster> modelClusterMap = ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds));
|
||||
|
||||
Map<String, List<SchemaItem>> modelClusterToMatchCount = new HashMap<>();
|
||||
for (ModelCluster modelCluster : modelClusterMap.values()) {
|
||||
for (MetricResp metricResp : metricResps) {
|
||||
if (modelCluster.getModelIds().contains(metricResp.getModelId())) {
|
||||
modelClusterToMatchCount.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>())
|
||||
.add(metricResp);
|
||||
}
|
||||
}
|
||||
}
|
||||
String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream()
|
||||
.max(Comparator.comparingInt(entry -> entry.getValue().size()))
|
||||
.map(Map.Entry::getKey)
|
||||
.orElse(null);
|
||||
|
||||
return modelClusterMap.get(keyWithMaxSize);
|
||||
}
|
||||
|
||||
private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
|
||||
List<DimensionResp> dimensionResps) {
|
||||
Set<Long> result = new HashSet<>();
|
||||
if (CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
|
||||
result.addAll(modelIdsByDomainId);
|
||||
return result;
|
||||
}
|
||||
Set<Long> metricModelIds = metricResps.stream().map(entry -> entry.getModelId())
|
||||
.collect(Collectors.toSet());
|
||||
result.addAll(metricModelIds);
|
||||
|
||||
Set<Long> dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId())
|
||||
.collect(Collectors.toSet());
|
||||
result.addAll(dimensionModelIds);
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<DimensionResp> getDimensionResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
|
||||
DimensionsFilter dimensionsFilter = new DimensionsFilter();
|
||||
BeanUtils.copyProperties(queryMetricReq, dimensionsFilter);
|
||||
dimensionsFilter.setModelIds(new ArrayList<>(modelIds));
|
||||
List<DimensionResp> dimensionResps = dimensionService.queryDimensions(dimensionsFilter);
|
||||
return dimensionResps;
|
||||
}
|
||||
|
||||
private List<MetricResp> getMetricResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
|
||||
MetricsFilter metricsFilter = new MetricsFilter();
|
||||
BeanUtils.copyProperties(queryMetricReq, metricsFilter);
|
||||
metricsFilter.setModelIds(new ArrayList<>(modelIds));
|
||||
return metricService.queryMetrics(metricsFilter);
|
||||
}
|
||||
|
||||
private Set<Long> getModelIdsByDomainId(QueryMetricReq queryMetricReq) {
|
||||
List<ModelResp> modelResps = modelService.getAllModelByDomainIds(
|
||||
Collections.singletonList(queryMetricReq.getDomainId()));
|
||||
return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception {
|
||||
@@ -271,23 +418,6 @@ public class QueryServiceImpl implements QueryService {
|
||||
return appService.getApp(appId);
|
||||
}
|
||||
|
||||
private QueryStructReq buildQueryStructReq(List<DimensionResp> dimensionResps,
|
||||
MetricResp metricResp, DateConf dateConf, Long limit) {
|
||||
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
|
||||
modelIds.add(metricResp.getModelId());
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setGroups(dimensionResps.stream()
|
||||
.map(DimensionResp::getBizName).collect(Collectors.toList()));
|
||||
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricResp.getBizName());
|
||||
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
|
||||
queryStructReq.setDateInfo(dateConf);
|
||||
queryStructReq.setModelIds(modelIds);
|
||||
queryStructReq.setLimit(limit);
|
||||
return queryStructReq;
|
||||
}
|
||||
|
||||
private String getTimeDimension(DateConf dateConf) {
|
||||
if (Constants.MONTH.equals(dateConf.getPeriod())) {
|
||||
return TimeDimensionEnum.MONTH.getName();
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
package com.tencent.supersonic.headless.server.utils;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
@@ -31,11 +36,6 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import com.tencent.supersonic.headless.server.service.QueryService;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
@@ -48,11 +48,10 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Component
|
||||
public class DictUtils {
|
||||
@@ -79,9 +78,9 @@ public class DictUtils {
|
||||
private final ModelService modelService;
|
||||
|
||||
public DictUtils(DimensionService dimensionService,
|
||||
MetricService metricService,
|
||||
QueryService queryService,
|
||||
ModelService modelService) {
|
||||
MetricService metricService,
|
||||
QueryService queryService,
|
||||
ModelService modelService) {
|
||||
this.dimensionService = dimensionService;
|
||||
this.metricService = metricService;
|
||||
this.queryService = queryService;
|
||||
@@ -222,7 +221,7 @@ public class DictUtils {
|
||||
&& Objects.nonNull(dictItemResp.getConfig().getMetricId())) {
|
||||
// 查询默认指标
|
||||
QueryStructReq queryStructReq = generateQueryStruct(dictItemResp);
|
||||
return queryStructReq.convert(queryStructReq, true);
|
||||
return queryStructReq.convert(true);
|
||||
}
|
||||
// count(1) 作为指标
|
||||
return constructQuerySqlReq(dictItemResp);
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.tencent.supersonic.headless.server.utils;
|
||||
|
||||
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
|
||||
import com.tencent.supersonic.headless.server.service.SchemaService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ModelClusterBuilder {
|
||||
|
||||
public static Map<String, ModelCluster> buildModelClusters(List<Long> modelIds) {
|
||||
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
|
||||
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchemaResps(modelIds);
|
||||
Map<Long, ModelSchemaResp> modelIdToModelSchema = modelSchemaResps.stream()
|
||||
.collect(Collectors.toMap(ModelSchemaResp::getId, value -> value, (k1, k2) -> k1));
|
||||
|
||||
Set<Long> visited = new HashSet<>();
|
||||
List<Set<Long>> modelClusters = new ArrayList<>();
|
||||
for (ModelSchemaResp model : modelSchemaResps) {
|
||||
if (!visited.contains(model.getId())) {
|
||||
Set<Long> modelCluster = new HashSet<>();
|
||||
dfs(model, modelIdToModelSchema, visited, modelCluster);
|
||||
modelClusters.add(modelCluster);
|
||||
}
|
||||
}
|
||||
return modelClusters.stream().map(ModelCluster::build)
|
||||
.collect(Collectors.toMap(ModelCluster::getKey, value -> value, (k1, k2) -> k1));
|
||||
}
|
||||
|
||||
private static void dfs(ModelSchemaResp model, Map<Long, ModelSchemaResp> modelMap,
|
||||
Set<Long> visited, Set<Long> modelCluster) {
|
||||
visited.add(model.getId());
|
||||
modelCluster.add(model.getId());
|
||||
for (Long neighborId : model.getModelClusterSet()) {
|
||||
if (!visited.contains(neighborId)) {
|
||||
dfs(modelMap.get(neighborId), modelMap, visited, modelCluster);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -167,4 +167,40 @@
|
||||
</select>
|
||||
|
||||
|
||||
<select id="queryDimensions" resultMap="ResultMapWithBLOBs">
|
||||
select *
|
||||
from s2_dimension
|
||||
where status != 3
|
||||
<if test="modelIds != null and modelIds.size >0">
|
||||
and model_id in
|
||||
<foreach collection="modelIds" index="index" item="model" open="(" close=")"
|
||||
separator=",">
|
||||
#{model}
|
||||
</foreach>
|
||||
</if>
|
||||
<if test="dimensionIds != null and dimensionIds.size >0">
|
||||
and id in
|
||||
<foreach collection="dimensionIds" index="index" item="dimensionId" open="(" close=")"
|
||||
separator=",">
|
||||
#{dimensionId}
|
||||
</foreach>
|
||||
</if>
|
||||
<if test="dimensionNames != null and dimensionNames.size > 0">
|
||||
AND (
|
||||
(name IN
|
||||
<foreach collection="dimensionNames" index="index" item="dimensionName" open="(" close=")"
|
||||
separator=",">
|
||||
#{dimensionName}
|
||||
</foreach>)
|
||||
OR
|
||||
(biz_name IN
|
||||
<foreach collection="dimensionNames" index="index" item="dimensionName" open="(" close=")"
|
||||
separator=",">
|
||||
#{dimensionName}
|
||||
</foreach>)
|
||||
)
|
||||
</if>
|
||||
</select>
|
||||
|
||||
|
||||
</mapper>
|
||||
|
||||
@@ -2,27 +2,29 @@
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.tencent.supersonic.headless.server.persistence.mapper.MetricDOCustomMapper">
|
||||
<resultMap id="BaseResultMap" type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO">
|
||||
<id column="id" jdbcType="BIGINT" property="id" />
|
||||
<result column="model_id" jdbcType="BIGINT" property="modelId" />
|
||||
<result column="name" jdbcType="VARCHAR" property="name" />
|
||||
<result column="biz_name" jdbcType="VARCHAR" property="bizName" />
|
||||
<result column="description" jdbcType="VARCHAR" property="description" />
|
||||
<result column="status" jdbcType="INTEGER" property="status" />
|
||||
<result column="sensitive_level" jdbcType="INTEGER" property="sensitiveLevel" />
|
||||
<result column="type" jdbcType="VARCHAR" property="type" />
|
||||
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt" />
|
||||
<result column="created_by" jdbcType="VARCHAR" property="createdBy" />
|
||||
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt" />
|
||||
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy" />
|
||||
<result column="data_format_type" jdbcType="VARCHAR" property="dataFormatType" />
|
||||
<result column="data_format" jdbcType="VARCHAR" property="dataFormat" />
|
||||
<result column="alias" jdbcType="VARCHAR" property="alias" />
|
||||
<result column="tags" jdbcType="VARCHAR" property="tags" />
|
||||
<result column="define_type" jdbcType="VARCHAR" property="defineType" />
|
||||
<resultMap id="BaseResultMap"
|
||||
type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO">
|
||||
<id column="id" jdbcType="BIGINT" property="id"/>
|
||||
<result column="model_id" jdbcType="BIGINT" property="modelId"/>
|
||||
<result column="name" jdbcType="VARCHAR" property="name"/>
|
||||
<result column="biz_name" jdbcType="VARCHAR" property="bizName"/>
|
||||
<result column="description" jdbcType="VARCHAR" property="description"/>
|
||||
<result column="status" jdbcType="INTEGER" property="status"/>
|
||||
<result column="sensitive_level" jdbcType="INTEGER" property="sensitiveLevel"/>
|
||||
<result column="type" jdbcType="VARCHAR" property="type"/>
|
||||
<result column="created_at" jdbcType="TIMESTAMP" property="createdAt"/>
|
||||
<result column="created_by" jdbcType="VARCHAR" property="createdBy"/>
|
||||
<result column="updated_at" jdbcType="TIMESTAMP" property="updatedAt"/>
|
||||
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy"/>
|
||||
<result column="data_format_type" jdbcType="VARCHAR" property="dataFormatType"/>
|
||||
<result column="data_format" jdbcType="VARCHAR" property="dataFormat"/>
|
||||
<result column="alias" jdbcType="VARCHAR" property="alias"/>
|
||||
<result column="tags" jdbcType="VARCHAR" property="tags"/>
|
||||
<result column="define_type" jdbcType="VARCHAR" property="defineType"/>
|
||||
</resultMap>
|
||||
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO">
|
||||
<result column="type_params" jdbcType="LONGVARCHAR" property="typeParams" />
|
||||
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs"
|
||||
type="com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO">
|
||||
<result column="type_params" jdbcType="LONGVARCHAR" property="typeParams"/>
|
||||
</resultMap>
|
||||
<sql id="Example_Where_Clause">
|
||||
<where>
|
||||
@@ -56,14 +58,16 @@
|
||||
</where>
|
||||
</sql>
|
||||
<sql id="Base_Column_List">
|
||||
id, model_id, name, biz_name, description, status, sensitive_level, type, created_at,
|
||||
id
|
||||
, model_id, name, biz_name, description, status, sensitive_level, type, created_at,
|
||||
created_by, updated_at, updated_by, data_format_type, data_format, alias, tags, define_type
|
||||
</sql>
|
||||
<sql id="Blob_Column_List">
|
||||
type_params
|
||||
</sql>
|
||||
|
||||
<insert id="batchInsert" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
|
||||
<insert id="batchInsert" parameterType="java.util.List" useGeneratedKeys="true"
|
||||
keyProperty="id">
|
||||
insert into s2_metric (model_id, name,
|
||||
biz_name, description, type,status,sensitive_level,
|
||||
created_at, created_by, updated_at,
|
||||
@@ -94,9 +98,9 @@
|
||||
</update>
|
||||
|
||||
<select id="query" resultMap="ResultMapWithBLOBs">
|
||||
select *
|
||||
from s2_metric
|
||||
where status != 3
|
||||
select *
|
||||
from s2_metric
|
||||
where status != 3
|
||||
<if test="type != null and type != ''">
|
||||
and type = #{type}
|
||||
</if>
|
||||
@@ -126,14 +130,14 @@
|
||||
<if test="modelIds != null and modelIds.size >0">
|
||||
and model_id in
|
||||
<foreach collection="modelIds" index="index" item="model" open="(" close=")"
|
||||
separator=",">
|
||||
separator=",">
|
||||
#{model}
|
||||
</foreach>
|
||||
</if>
|
||||
<if test="ids != null and ids.size >0">
|
||||
and id in
|
||||
<foreach collection="ids" index="index" item="id" open="(" close=")"
|
||||
separator=",">
|
||||
separator=",">
|
||||
#{id}
|
||||
</foreach>
|
||||
</if>
|
||||
@@ -142,4 +146,40 @@
|
||||
</if>
|
||||
</select>
|
||||
|
||||
<select id="queryMetrics" resultMap="ResultMapWithBLOBs">
|
||||
select *
|
||||
from s2_metric
|
||||
where status != 3
|
||||
<if test="modelIds != null and modelIds.size >0">
|
||||
and model_id in
|
||||
<foreach collection="modelIds" index="index" item="model" open="(" close=")"
|
||||
separator=",">
|
||||
#{model}
|
||||
</foreach>
|
||||
</if>
|
||||
<if test="metricIds != null and metricIds.size >0">
|
||||
and id in
|
||||
<foreach collection="metricIds" index="index" item="metricId" open="(" close=")"
|
||||
separator=",">
|
||||
#{metricId}
|
||||
</foreach>
|
||||
</if>
|
||||
<if test="metricNames != null and metricNames.size > 0">
|
||||
AND (
|
||||
(name IN
|
||||
<foreach collection="metricNames" index="index" item="metricName" open="(" close=")"
|
||||
separator=",">
|
||||
#{metricName}
|
||||
</foreach>)
|
||||
OR
|
||||
(biz_name IN
|
||||
<foreach collection="metricNames" index="index" item="metricName" open="(" close=")"
|
||||
separator=",">
|
||||
#{metricName}
|
||||
</foreach>)
|
||||
)
|
||||
</if>
|
||||
</select>
|
||||
|
||||
|
||||
</mapper>
|
||||
|
||||
@@ -23,7 +23,6 @@ import com.tencent.supersonic.common.pojo.SysParameter;
|
||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||
import com.tencent.supersonic.common.service.SysParameterService;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.headless.server.service.KnowledgeService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
@@ -53,8 +52,6 @@ public class ChatDemoLoader implements CommandLineRunner {
|
||||
private AgentService agentService;
|
||||
@Autowired
|
||||
private SysParameterService sysParameterService;
|
||||
@Autowired
|
||||
private KnowledgeService knowledgeService;
|
||||
|
||||
@Value("${demo.enabled:false}")
|
||||
private boolean demoEnabled;
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.tencent.supersonic.headless;
|
||||
|
||||
import static org.junit.Assert.assertThrows;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||
import java.util.Arrays;
|
||||
import org.junit.Assert;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class QueryByMetricTest extends BaseTest {
|
||||
|
||||
@Test
|
||||
public void testWithMetricAndDimensionBizNames() throws Exception {
|
||||
QueryMetricReq queryMetricReq = new QueryMetricReq();
|
||||
queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
|
||||
queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
|
||||
SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
|
||||
Assert.assertNotNull(queryResp.getResultList());
|
||||
Assert.assertEquals(6, queryResp.getResultList().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithMetricAndDimensionNames() throws Exception {
|
||||
QueryMetricReq queryMetricReq = new QueryMetricReq();
|
||||
queryMetricReq.setMetricNames(Arrays.asList("停留时长", "访问次数"));
|
||||
queryMetricReq.setDimensionNames(Arrays.asList("用户", "部门"));
|
||||
SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
|
||||
Assert.assertNotNull(queryResp.getResultList());
|
||||
Assert.assertEquals(6, queryResp.getResultList().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithDomainId() throws Exception {
|
||||
QueryMetricReq queryMetricReq = new QueryMetricReq();
|
||||
queryMetricReq.setDomainId(1L);
|
||||
queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
|
||||
queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
|
||||
SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
|
||||
Assert.assertNotNull(queryResp.getResultList());
|
||||
Assert.assertEquals(6, queryResp.getResultList().size());
|
||||
|
||||
queryMetricReq.setDomainId(2L);
|
||||
queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
|
||||
queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> queryService.queryByMetric(queryMetricReq, User.getFakeUser()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithMetricAndDimensionIds() throws Exception {
|
||||
QueryMetricReq queryMetricReq = new QueryMetricReq();
|
||||
queryMetricReq.setDomainId(1L);
|
||||
queryMetricReq.setMetricIds(Arrays.asList(1L, 4L));
|
||||
queryMetricReq.setDimensionIds(Arrays.asList(1L, 2L));
|
||||
SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
|
||||
Assert.assertNotNull(queryResp.getResultList());
|
||||
Assert.assertEquals(6, queryResp.getResultList().size());
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user