(improvement)(Headless) Adding SQL, view, tag, and metric APIs, along with the addition of GrammarCorrector.

This commit is contained in:
lexluo
2024-02-27 14:06:10 +08:00
parent 3a38200448
commit c7c70208ff
19 changed files with 365 additions and 197 deletions

View File

@@ -0,0 +1,31 @@
package com.tencent.supersonic.chat.core.corrector;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
/**
* Correcting SQL syntax, primarily including fixes to select, where, groupBy, and Having clauses
*/
@Slf4j
public class GrammarCorrector extends BaseSemanticCorrector {
private List<BaseSemanticCorrector> correctors;
public GrammarCorrector() {
correctors = new ArrayList<>();
correctors.add(new SelectCorrector());
correctors.add(new WhereCorrector());
correctors.add(new GroupByCorrector());
correctors.add(new HavingCorrector());
}
@Override
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
for (BaseSemanticCorrector corrector : correctors) {
corrector.correct(queryContext, semanticParseInfo);
}
}
}

View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.headless.api.pojo.request;
import java.util.List;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class QueryTagReq {
private Long domainId;
private List<Long> tagIds;
private List<String> tagNames;
private Long limit = 2000L;
}

View File

@@ -0,0 +1,33 @@
package com.tencent.supersonic.headless.api.pojo.request;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Param;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class QueryViewReq {
private Long viewId;
private String viewName;
private String sql;
private boolean needAuth = true;
private List<Param> params = new ArrayList<>();
private Cache cacheInfo = new Cache();
private List<String> groups = new ArrayList<>();
private List<Aggregator> aggregators = new ArrayList<>();
private List<Order> orders = new ArrayList<>();
private List<Filter> dimensionFilters = new ArrayList<>();
private List<Filter> metricFilters = new ArrayList<>();
private DateConf dateInfo;
private Long limit = 2000L;
private QueryType queryType = QueryType.ID;
}

View File

@@ -41,14 +41,6 @@ public class QueryController {
@Autowired
private DownloadService downloadService;
@PostMapping("/sql")
public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return queryService.queryByReq(querySqlReq, user);
}
@PostMapping("/struct")
public Object queryByStruct(@RequestBody QueryStructReq queryStructReq,
HttpServletRequest request,

View File

@@ -0,0 +1,38 @@
package com.tencent.supersonic.headless.server.rest.api;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.QueryService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class MetricQueryApiController {
@Autowired
private QueryService queryService;
@Autowired
private MetricService metricService;
@PostMapping("/metric")
public Object queryByMetric(@RequestBody QueryMetricReq queryMetricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
QueryStructReq queryStructReq = metricService.convert(queryMetricReq);
return queryService.queryByReq(queryStructReq.convert(), user);
}
}

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.headless.server.rest;
package com.tencent.supersonic.headless.server.rest.api;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.server.service.QueryService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -16,17 +16,16 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class MetricApiController {
public class SqlQueryApiController {
@Autowired
private QueryService queryService;
@PostMapping("/metric")
public Object queryBySql(@RequestBody QueryMetricReq queryMetricReq,
@PostMapping("/sql")
public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return queryService.queryByMetric(queryMetricReq, user);
return queryService.queryByReq(querySqlReq, user);
}
}

View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.headless.server.rest.api;
import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class TagQueryApiController {
@PostMapping("/tag")
public Object queryByTag(@RequestBody QueryTagReq queryTagReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
//TODO
return null;
}
}

View File

@@ -0,0 +1,37 @@
package com.tencent.supersonic.headless.server.rest.api;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.ViewService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class ViewQueryApiController {
@Autowired
private ViewService viewService;
@Autowired
private QueryService queryService;
@PostMapping("/view")
public Object queryByView(@RequestBody QueryViewReq queryViewReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
SemanticQueryReq queryReq = viewService.convert(queryViewReq);
return queryService.queryByReq(queryReq, user);
}
}

View File

@@ -9,6 +9,8 @@ import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
@@ -52,4 +54,6 @@ public interface MetricService {
void sendMetricEventBatch(List<Long> modelIds, EventType eventType);
List<MetricResp> queryMetrics(MetricsFilter metricsFilter);
QueryStructReq convert(QueryMetricReq queryMetricReq);
}

View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.headless.api.pojo.response.ItemQueryResultResp;
@@ -28,6 +27,4 @@ public interface QueryService {
@ApiHeaderCheck
ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, HttpServletRequest request) throws Exception;
SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception;
}

View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
@@ -25,4 +27,6 @@ public interface ViewService {
List<ViewResp> getViews(User user);
List<ViewResp> getViewsInheritAuth(User user, Long domainId);
SemanticQueryReq convert(QueryViewReq queryViewReq);
}

View File

@@ -7,6 +7,7 @@ import com.github.pagehelper.PageInfo;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.DataItem;
@@ -20,11 +21,15 @@ import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.PageMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
@@ -32,23 +37,22 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -56,6 +60,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
@Slf4j
@@ -65,7 +74,7 @@ public class MetricServiceImpl implements MetricService {
private ModelService modelService;
private DomainService domainService;
private DimensionService dimensionService;
private ChatGptHelper chatGptHelper;
@@ -77,18 +86,18 @@ public class MetricServiceImpl implements MetricService {
public MetricServiceImpl(MetricRepository metricRepository,
ModelService modelService,
DomainService domainService,
ChatGptHelper chatGptHelper,
CollectService collectService,
ViewService viewService,
ApplicationEventPublisher eventPublisher) {
this.domainService = domainService;
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService) {
this.metricRepository = metricRepository;
this.modelService = modelService;
this.chatGptHelper = chatGptHelper;
this.eventPublisher = eventPublisher;
this.collectService = collectService;
this.viewService = viewService;
this.dimensionService = dimensionService;
}
@Override
@@ -239,7 +248,7 @@ public class MetricServiceImpl implements MetricService {
}
private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
List<String> fields, Set<MetricResp> metricRespFiltered) {
List<String> fields, Set<MetricResp> metricRespFiltered) {
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
.stream().map(MetricParam::getId).collect(Collectors.toList());
@@ -483,4 +492,107 @@ public class MetricServiceImpl implements MetricService {
.type(TypeEnums.METRIC).defaultAgg(metricResp.getDefaultAgg()).build();
}
public QueryStructReq convert(QueryMetricReq queryMetricReq) {
//1. If a domainId exists, the modelIds obtained from the domainId.
Set<Long> modelIdsByDomainId = getModelIdsByDomainId(queryMetricReq);
//2. get metrics and dimensions
List<MetricResp> metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId);
List<DimensionResp> dimensionResps = getDimensionResps(queryMetricReq, modelIdsByDomainId);
//3. choose ModelCluster
Set<Long> modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps);
ModelCluster modelCluster = getModelCluster(metricResps, modelIds);
//4. set groups
List<String> dimensionBizNames = dimensionResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.map(entry -> entry.getBizName()).collect(Collectors.toList());
QueryStructReq queryStructReq = new QueryStructReq();
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(dimensionBizNames)) {
queryStructReq.setGroups(dimensionBizNames);
}
//5. set aggregators
List<String> metricBizNames = metricResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.map(entry -> entry.getBizName()).collect(Collectors.toList());
if (org.apache.commons.collections.CollectionUtils.isEmpty(metricBizNames)) {
throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
}
List<Aggregator> aggregators = new ArrayList<>();
for (String metricBizName : metricBizNames) {
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricBizName);
aggregators.add(aggregator);
}
queryStructReq.setAggregators(aggregators);
queryStructReq.setLimit(queryMetricReq.getLimit());
//6. set modelIds
for (Long modelId : modelCluster.getModelIds()) {
queryStructReq.addModelId(modelId);
}
//7. set dateInfo
queryStructReq.setDateInfo(queryMetricReq.getDateInfo());
return queryStructReq;
}
private ModelCluster getModelCluster(List<MetricResp> metricResps, Set<Long> modelIds) {
Map<String, ModelCluster> modelClusterMap = ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds));
Map<String, List<SchemaItem>> modelClusterToMatchCount = new HashMap<>();
for (ModelCluster modelCluster : modelClusterMap.values()) {
for (MetricResp metricResp : metricResps) {
if (modelCluster.getModelIds().contains(metricResp.getModelId())) {
modelClusterToMatchCount.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>())
.add(metricResp);
}
}
}
String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream()
.max(Comparator.comparingInt(entry -> entry.getValue().size()))
.map(Map.Entry::getKey)
.orElse(null);
return modelClusterMap.get(keyWithMaxSize);
}
private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
List<DimensionResp> dimensionResps) {
Set<Long> result = new HashSet<>();
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
result.addAll(modelIdsByDomainId);
return result;
}
Set<Long> metricModelIds = metricResps.stream().map(entry -> entry.getModelId())
.collect(Collectors.toSet());
result.addAll(metricModelIds);
Set<Long> dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId())
.collect(Collectors.toSet());
result.addAll(dimensionModelIds);
return result;
}
private List<DimensionResp> getDimensionResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
DimensionsFilter dimensionsFilter = new DimensionsFilter();
BeanUtils.copyProperties(queryMetricReq, dimensionsFilter);
dimensionsFilter.setModelIds(new ArrayList<>(modelIds));
return dimensionService.queryDimensions(dimensionsFilter);
}
private List<MetricResp> getMetricResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
MetricsFilter metricsFilter = new MetricsFilter();
BeanUtils.copyProperties(queryMetricReq, metricsFilter);
metricsFilter.setModelIds(new ArrayList<>(modelIds));
return queryMetrics(metricsFilter);
}
private Set<Long> getModelIdsByDomainId(QueryMetricReq queryMetricReq) {
List<ModelResp> modelResps = modelService.getAllModelByDomainIds(
Collections.singletonList(queryMetricReq.getDomainId()));
return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet());
}
}

View File

@@ -13,13 +13,11 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Item;
import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
@@ -45,24 +43,13 @@ import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.service.AppService;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -72,7 +59,6 @@ import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@@ -87,17 +73,9 @@ public class QueryServiceImpl implements QueryService {
private final AppService appService;
private final QueryCache queryCache;
private final SemanticSchemaManager semanticSchemaManager;
private final QueryParser queryParser;
private final QueryPlanner queryPlanner;
private final MetricService metricService;
private final ModelService modelService;
private final DimensionService dimensionService;
public QueryServiceImpl(
StatUtils statUtils,
QueryUtils queryUtils,
@@ -107,10 +85,7 @@ public class QueryServiceImpl implements QueryService {
QueryCache queryCache,
SemanticSchemaManager semanticSchemaManager,
DefaultQueryParser queryParser,
QueryPlanner queryPlanner,
MetricService metricService,
ModelService modelService,
DimensionService dimensionService) {
QueryPlanner queryPlanner) {
this.statUtils = statUtils;
this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter;
@@ -120,9 +95,6 @@ public class QueryServiceImpl implements QueryService {
this.semanticSchemaManager = semanticSchemaManager;
this.queryParser = queryParser;
this.queryPlanner = queryPlanner;
this.metricService = metricService;
this.modelService = modelService;
this.dimensionService = dimensionService;
}
@Override
@@ -265,58 +237,6 @@ public class QueryServiceImpl implements QueryService {
return ItemQueryResultResp.builder().results(results).build();
}
@Override
public SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) {
QueryStructReq queryStructReq = buildQueryStructReq(queryMetricReq);
return queryByReq(queryStructReq.convert(), user);
}
private QueryStructReq buildQueryStructReq(QueryMetricReq queryMetricReq) {
//1. If a domainId exists, the modelIds obtained from the domainId.
Set<Long> modelIdsByDomainId = getModelIdsByDomainId(queryMetricReq);
//2. get metrics and dimensions
List<MetricResp> metricResps = getMetricResps(queryMetricReq, modelIdsByDomainId);
List<DimensionResp> dimensionResps = getDimensionResps(queryMetricReq, modelIdsByDomainId);
//3. choose ModelCluster
Set<Long> modelIds = getModelIds(modelIdsByDomainId, metricResps, dimensionResps);
ModelCluster modelCluster = getModelCluster(metricResps, modelIds);
//4. set groups
List<String> dimensionBizNames = dimensionResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.map(entry -> entry.getBizName()).collect(Collectors.toList());
QueryStructReq queryStructReq = new QueryStructReq();
if (CollectionUtils.isNotEmpty(dimensionBizNames)) {
queryStructReq.setGroups(dimensionBizNames);
}
//5. set aggregators
List<String> metricBizNames = metricResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.map(entry -> entry.getBizName()).collect(Collectors.toList());
if (CollectionUtils.isEmpty(metricBizNames)) {
throw new IllegalArgumentException("Invalid input parameters, unable to obtain valid metrics");
}
List<Aggregator> aggregators = new ArrayList<>();
for (String metricBizName : metricBizNames) {
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricBizName);
aggregators.add(aggregator);
}
queryStructReq.setAggregators(aggregators);
queryStructReq.setLimit(queryMetricReq.getLimit());
//6. set modelIds
for (Long modelId : modelCluster.getModelIds()) {
queryStructReq.addModelId(modelId);
}
//7. set dateInfo
queryStructReq.setDateInfo(queryMetricReq.getDateInfo());
return queryStructReq;
}
private QueryStructReq buildQueryStructReq(List<DimensionResp> dimensionResps,
MetricResp metricResp, DateConf dateConf, Long limit) {
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
@@ -334,64 +254,6 @@ public class QueryServiceImpl implements QueryService {
return queryStructReq;
}
private ModelCluster getModelCluster(List<MetricResp> metricResps, Set<Long> modelIds) {
Map<String, ModelCluster> modelClusterMap = ModelClusterBuilder.buildModelClusters(new ArrayList<>(modelIds));
Map<String, List<SchemaItem>> modelClusterToMatchCount = new HashMap<>();
for (ModelCluster modelCluster : modelClusterMap.values()) {
for (MetricResp metricResp : metricResps) {
if (modelCluster.getModelIds().contains(metricResp.getModelId())) {
modelClusterToMatchCount.computeIfAbsent(modelCluster.getKey(), k -> new ArrayList<>())
.add(metricResp);
}
}
}
String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream()
.max(Comparator.comparingInt(entry -> entry.getValue().size()))
.map(Map.Entry::getKey)
.orElse(null);
return modelClusterMap.get(keyWithMaxSize);
}
private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
List<DimensionResp> dimensionResps) {
Set<Long> result = new HashSet<>();
if (CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
result.addAll(modelIdsByDomainId);
return result;
}
Set<Long> metricModelIds = metricResps.stream().map(entry -> entry.getModelId())
.collect(Collectors.toSet());
result.addAll(metricModelIds);
Set<Long> dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId())
.collect(Collectors.toSet());
result.addAll(dimensionModelIds);
return result;
}
private List<DimensionResp> getDimensionResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
DimensionsFilter dimensionsFilter = new DimensionsFilter();
BeanUtils.copyProperties(queryMetricReq, dimensionsFilter);
dimensionsFilter.setModelIds(new ArrayList<>(modelIds));
List<DimensionResp> dimensionResps = dimensionService.queryDimensions(dimensionsFilter);
return dimensionResps;
}
private List<MetricResp> getMetricResps(QueryMetricReq queryMetricReq, Set<Long> modelIds) {
MetricsFilter metricsFilter = new MetricsFilter();
BeanUtils.copyProperties(queryMetricReq, metricsFilter);
metricsFilter.setModelIds(new ArrayList<>(modelIds));
return metricService.queryMetrics(metricsFilter);
}
private Set<Long> getModelIdsByDomainId(QueryMetricReq queryMetricReq) {
List<ModelResp> modelResps = modelService.getAllModelByDomainIds(
Collections.singletonList(queryMetricReq.getDomainId()));
return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet());
}
private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception {
MetricResp metricResp = catalog.getMetric(item.getId());
item.setCreatedBy(metricResp.getCreatedBy());

View File

@@ -14,6 +14,10 @@ import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.ViewDetail;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryViewReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.DomainResp;
@@ -28,6 +32,7 @@ import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ViewService;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@@ -174,6 +179,15 @@ public class ViewServiceImpl
return viewDO;
}
public SemanticQueryReq convert(QueryViewReq queryViewReq) {
SemanticQueryReq queryReq = new QueryStructReq();
if (StringUtils.isNotBlank(queryViewReq.getSql())) {
queryReq = new QuerySqlReq();
}
BeanUtils.copyProperties(queryViewReq, queryReq);
return queryReq;
}
public static boolean checkAdminPermission(User user, ViewResp viewResp) {
List<String> admins = viewResp.getAdmins();
if (user.isSuperAdmin()) {
@@ -239,5 +253,4 @@ public class ViewServiceImpl
.map(Object::toString)
.collect(Collectors.toList());
}
}

View File

@@ -1,5 +1,8 @@
package com.tencent.supersonic.headless.server.service;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DataFormat;
@@ -8,12 +11,12 @@ import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
@@ -22,13 +25,11 @@ import com.tencent.supersonic.headless.server.persistence.repository.MetricRepos
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.service.impl.ViewServiceImpl;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import java.util.HashMap;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.context.ApplicationEventPublisher;
import java.util.HashMap;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
public class MetricServiceImplTest {
@@ -61,14 +62,14 @@ public class MetricServiceImplTest {
}
private MetricService mockMetricService(MetricRepository metricRepository,
ModelService modelService) {
DomainService domainService = Mockito.mock(DomainService.class);
ModelService modelService) {
ChatGptHelper chatGptHelper = Mockito.mock(ChatGptHelper.class);
CollectService collectService = Mockito.mock(CollectService.class);
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
ViewService viewService = Mockito.mock(ViewServiceImpl.class);
return new MetricServiceImpl(metricRepository, modelService, domainService,
chatGptHelper, collectService, viewService, eventPublisher);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
return new MetricServiceImpl(metricRepository, modelService, chatGptHelper, collectService, viewService,
eventPublisher, dimensionService);
}
private MetricReq buildMetricReq() {
@@ -96,7 +97,7 @@ public class MetricServiceImplTest {
RelateDimension.builder().drillDownDimensions(Lists.newArrayList(
new DrillDownDimension(1L),
new DrillDownDimension(1L, false))
).build());
).build());
metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
metricReq.setExt(new HashMap<>());
return metricReq;

View File

@@ -14,10 +14,7 @@ com.tencent.supersonic.chat.core.parser.SemanticParser=\
com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \
com.tencent.supersonic.chat.core.corrector.TimeCorrector, \
com.tencent.supersonic.chat.core.corrector.SelectCorrector, \
com.tencent.supersonic.chat.core.corrector.WhereCorrector, \
com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.core.corrector.HavingCorrector
com.tencent.supersonic.chat.core.corrector.GrammarCorrector
com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.query.semantic.RemoteSemanticInterpreter

View File

@@ -14,10 +14,7 @@ com.tencent.supersonic.chat.core.parser.SemanticParser=\
com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \
com.tencent.supersonic.chat.core.corrector.TimeCorrector, \
com.tencent.supersonic.chat.core.corrector.SelectCorrector, \
com.tencent.supersonic.chat.core.corrector.WhereCorrector, \
com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.core.corrector.HavingCorrector
com.tencent.supersonic.chat.core.corrector.GrammarCorrector
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.ParseInfoProcessor, \

View File

@@ -4,19 +4,25 @@ import static org.junit.Assert.assertThrows;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.MetricService;
import java.util.Arrays;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
public class QueryByMetricTest extends BaseTest {
@Autowired
protected MetricService metricService;
@Test
public void testWithMetricAndDimensionBizNames() throws Exception {
QueryMetricReq queryMetricReq = new QueryMetricReq();
queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
SemanticQueryResp queryResp = queryByMetric(queryMetricReq, User.getFakeUser());
Assert.assertNotNull(queryResp.getResultList());
Assert.assertEquals(6, queryResp.getResultList().size());
}
@@ -26,7 +32,7 @@ public class QueryByMetricTest extends BaseTest {
QueryMetricReq queryMetricReq = new QueryMetricReq();
queryMetricReq.setMetricNames(Arrays.asList("停留时长", "访问次数"));
queryMetricReq.setDimensionNames(Arrays.asList("用户", "部门"));
SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
SemanticQueryResp queryResp = queryByMetric(queryMetricReq, User.getFakeUser());
Assert.assertNotNull(queryResp.getResultList());
Assert.assertEquals(6, queryResp.getResultList().size());
}
@@ -37,7 +43,7 @@ public class QueryByMetricTest extends BaseTest {
queryMetricReq.setDomainId(1L);
queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
SemanticQueryResp queryResp = queryByMetric(queryMetricReq, User.getFakeUser());
Assert.assertNotNull(queryResp.getResultList());
Assert.assertEquals(6, queryResp.getResultList().size());
@@ -45,7 +51,7 @@ public class QueryByMetricTest extends BaseTest {
queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv"));
queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department"));
assertThrows(IllegalArgumentException.class,
() -> queryService.queryByMetric(queryMetricReq, User.getFakeUser()));
() -> queryByMetric(queryMetricReq, User.getFakeUser()));
}
@Test
@@ -54,9 +60,13 @@ public class QueryByMetricTest extends BaseTest {
queryMetricReq.setDomainId(1L);
queryMetricReq.setMetricIds(Arrays.asList(1L, 4L));
queryMetricReq.setDimensionIds(Arrays.asList(1L, 2L));
SemanticQueryResp queryResp = queryService.queryByMetric(queryMetricReq, User.getFakeUser());
SemanticQueryResp queryResp = queryByMetric(queryMetricReq, User.getFakeUser());
Assert.assertNotNull(queryResp.getResultList());
Assert.assertEquals(6, queryResp.getResultList().size());
}
private SemanticQueryResp queryByMetric(QueryMetricReq queryMetricReq, User user) throws Exception {
QueryStructReq convert = metricService.convert(queryMetricReq);
return queryService.queryByReq(convert.convert(), user);
}
}

View File

@@ -13,10 +13,8 @@ com.tencent.supersonic.chat.core.parser.SemanticParser=\
com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
com.tencent.supersonic.chat.core.corrector.SchemaCorrector, \
com.tencent.supersonic.chat.core.corrector.SelectCorrector, \
com.tencent.supersonic.chat.core.corrector.WhereCorrector, \
com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.core.corrector.HavingCorrector
com.tencent.supersonic.chat.core.corrector.TimeCorrector, \
com.tencent.supersonic.chat.core.corrector.GrammarCorrector
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.ParseInfoProcessor, \