mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-14 13:47:09 +00:00
(improvement)(headless) Update server and core, server calls core one-way (#592)
* (improvement)(headless) Update server and core, server calls core one-way * (improvement)(Auth) When obtaining the user information, determine whether the user is a system admin. --------- Co-authored-by: jolunoluo
This commit is contained in:
LXW
@@ -0,0 +1,60 @@
|
||||
package com.tencent.supersonic.headless.core.adaptor.db;
|
||||
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class ClickHouseAdaptor extends DbAdaptor {
|
||||
|
||||
@Override
|
||||
public String getDateFormat(String dateType, String dateFormat, String column) {
|
||||
if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) {
|
||||
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
|
||||
return "formatDateTime(toDate(parseDateTimeBestEffort(toString(%s))),'%Y-%m')".replace("%s", column);
|
||||
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
|
||||
return "toMonday(toDate(parseDateTimeBestEffort(toString(%s))))".replace("%s", column);
|
||||
} else {
|
||||
return "toDate(parseDateTimeBestEffort(toString(%s)))".replace("%s", column);
|
||||
}
|
||||
} else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) {
|
||||
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
|
||||
return "formatDateTime(toDate(%s),'%Y-%m')".replace("%s", column);
|
||||
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
|
||||
return "toMonday(toDate(%s))".replace("%s", column);
|
||||
} else {
|
||||
return column;
|
||||
}
|
||||
}
|
||||
return column;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDbMetaQueryTpl() {
|
||||
return " "
|
||||
+ " select "
|
||||
+ " name from system.databases "
|
||||
+ " where name not in('_temporary_and_external_tables','benchmark','default','system');";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTableMetaQueryTpl() {
|
||||
return "select name from system.tables where database = '%s';";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String functionNameCorrector(String sql) {
|
||||
Map<String, String> functionMap = new HashMap<>();
|
||||
functionMap.put("MONTH".toLowerCase(), "toMonth");
|
||||
functionMap.put("DAY".toLowerCase(), "toDayOfMonth");
|
||||
functionMap.put("YEAR".toLowerCase(), "toYear");
|
||||
return SqlParserReplaceHelper.replaceFunction(sql, functionMap);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getColumnMetaQueryTpl() {
|
||||
return "select name,type as dataType, comment from system.columns where database = '%s' and table='%s'";
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.tencent.supersonic.headless.core.adaptor.db;
|
||||
|
||||
|
||||
public abstract class DbAdaptor {
|
||||
|
||||
public abstract String getDateFormat(String dateType, String dateFormat, String column);
|
||||
|
||||
public abstract String getColumnMetaQueryTpl();
|
||||
|
||||
public abstract String getDbMetaQueryTpl();
|
||||
|
||||
public abstract String getTableMetaQueryTpl();
|
||||
|
||||
public abstract String functionNameCorrector(String sql);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.tencent.supersonic.headless.core.adaptor.db;
|
||||
|
||||
import com.tencent.supersonic.headless.api.enums.EngineType;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
public class DbAdaptorFactory {
|
||||
|
||||
private static Map<String, DbAdaptor> dbAdaptorMap;
|
||||
|
||||
static {
|
||||
dbAdaptorMap = new HashMap<>();
|
||||
dbAdaptorMap.put(EngineType.CLICKHOUSE.getName(), new ClickHouseAdaptor());
|
||||
dbAdaptorMap.put(EngineType.MYSQL.getName(), new MysqlAdaptor());
|
||||
dbAdaptorMap.put(EngineType.H2.getName(), new H2Adaptor());
|
||||
}
|
||||
|
||||
public static DbAdaptor getEngineAdaptor(String engineType) {
|
||||
return dbAdaptorMap.get(engineType);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package com.tencent.supersonic.headless.core.adaptor.db;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
|
||||
public class H2Adaptor extends DbAdaptor {
|
||||
|
||||
@Override
|
||||
public String getDateFormat(String dateType, String dateFormat, String column) {
|
||||
if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) {
|
||||
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
|
||||
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM')".replace("%s", column);
|
||||
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
|
||||
return "DATE_TRUNC('week',%s)".replace("%s", column);
|
||||
} else {
|
||||
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyyMMdd'),'yyyy-MM-dd')".replace("%s", column);
|
||||
}
|
||||
} else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) {
|
||||
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
|
||||
return "FORMATDATETIME(PARSEDATETIME(%s, 'yyyy-MM-dd'),'yyyy-MM') ".replace("%s", column);
|
||||
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
|
||||
return "DATE_TRUNC('week',%s)".replace("%s", column);
|
||||
} else {
|
||||
return column;
|
||||
}
|
||||
}
|
||||
return column;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getColumnMetaQueryTpl() {
|
||||
return "SELECT COLUMN_NAME AS name, "
|
||||
+ " case DATA_TYPE"
|
||||
+ " when '12' then 'varchar'"
|
||||
+ " when '-5' then 'integer'"
|
||||
+ " when '8' then 'double'"
|
||||
+ " end AS dataType"
|
||||
+ " FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA ='%s' AND TABLE_NAME = '%s'";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDbMetaQueryTpl() {
|
||||
return "SELECT DISTINCT TABLE_SCHEMA as name FROM INFORMATION_SCHEMA.TABLES WHERE STORAGE_TYPE = 'MEMORY'";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTableMetaQueryTpl() {
|
||||
return "SELECT TABLE_NAME as name FROM INFORMATION_SCHEMA.TABLES "
|
||||
+ "WHERE STORAGE_TYPE = 'MEMORY' AND TABLE_SCHEMA = '%s'";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String functionNameCorrector(String sql) {
|
||||
return sql;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.tencent.supersonic.headless.core.adaptor.db;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
|
||||
|
||||
public class MysqlAdaptor extends DbAdaptor {
|
||||
|
||||
|
||||
/**
|
||||
* transform YYYYMMDD to YYYY-MM-DD YYYY-MM YYYY-MM-DD(MONDAY)
|
||||
*/
|
||||
@Override
|
||||
public String getDateFormat(String dateType, String dateFormat, String column) {
|
||||
if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT_INT)) {
|
||||
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
|
||||
return "DATE_FORMAT(%s, '%Y-%m')".replace("%s", column);
|
||||
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
|
||||
return "DATE_FORMAT(DATE_SUB(%s, INTERVAL (DAYOFWEEK(%s) - 2) DAY), '%Y-%m-%d')".replace("%s", column);
|
||||
} else {
|
||||
return "date_format(str_to_date(%s, '%Y%m%d'),'%Y-%m-%d')".replace("%s", column);
|
||||
}
|
||||
} else if (dateFormat.equalsIgnoreCase(Constants.DAY_FORMAT)) {
|
||||
if (TimeDimensionEnum.MONTH.name().equalsIgnoreCase(dateType)) {
|
||||
return "DATE_FORMAT(%s, '%Y-%m') ".replace("%s", column);
|
||||
} else if (TimeDimensionEnum.WEEK.name().equalsIgnoreCase(dateType)) {
|
||||
return "DATE_FORMAT(DATE_SUB(%s, INTERVAL (DAYOFWEEK(%s) - 2) DAY), '%Y-%m-%d')".replace("%s", column);
|
||||
} else {
|
||||
return column;
|
||||
}
|
||||
}
|
||||
return column;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDbMetaQueryTpl() {
|
||||
return "select distinct TABLE_SCHEMA as name from information_schema.tables "
|
||||
+ "where TABLE_SCHEMA not in ('information_schema','mysql','performance_schema','sys');";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTableMetaQueryTpl() {
|
||||
return "select TABLE_NAME as name from information_schema.tables where TABLE_SCHEMA = '%s';";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String functionNameCorrector(String sql) {
|
||||
return sql;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getColumnMetaQueryTpl() {
|
||||
return "SELECT COLUMN_NAME as name, DATA_TYPE as dataType, COLUMN_COMMENT as comment "
|
||||
+ "FROM information_schema.columns WHERE table_schema ='%s' AND table_name = '%s'";
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.annotation;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
@Target({ElementType.PARAMETER, ElementType.METHOD})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ApiHeaderCheck {
|
||||
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.annotation;
|
||||
|
||||
import java.lang.annotation.Target;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.Documented;
|
||||
|
||||
@Target(ElementType.METHOD)
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Documented
|
||||
public @interface S2SQLDataPermission {
|
||||
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.annotation;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
@Target({ElementType.PARAMETER, ElementType.METHOD})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface StructDataPermission {
|
||||
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.aspect;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Pair;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.common.util.SignatureUtils;
|
||||
import com.tencent.supersonic.headless.api.enums.AppStatus;
|
||||
import com.tencent.supersonic.headless.api.response.AppDetailResp;
|
||||
import com.tencent.supersonic.headless.server.service.AppService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.aspectj.lang.ProceedingJoinPoint;
|
||||
import org.aspectj.lang.annotation.Around;
|
||||
import org.aspectj.lang.annotation.Aspect;
|
||||
import org.aspectj.lang.annotation.Pointcut;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
@Component
|
||||
@Aspect
|
||||
@Order(1)
|
||||
@Slf4j
|
||||
public class ApiHeaderCheckAspect {
|
||||
|
||||
public static final String APPID = "appId";
|
||||
|
||||
private static final String TIMESTAMP = "timestamp";
|
||||
|
||||
private static final String SIGNATURE = "signature";
|
||||
|
||||
@Autowired
|
||||
private AppService appService;
|
||||
|
||||
@Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck)")
|
||||
private void apiPermissionCheck() {
|
||||
}
|
||||
|
||||
@Around("apiPermissionCheck()")
|
||||
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||
Object[] objects = joinPoint.getArgs();
|
||||
HttpServletRequest request = (HttpServletRequest) objects[1];
|
||||
checkHeader(request);
|
||||
return joinPoint.proceed();
|
||||
}
|
||||
|
||||
private void checkHeader(HttpServletRequest request) {
|
||||
String timestampStr = request.getHeader(TIMESTAMP);
|
||||
String signature = request.getHeader(SIGNATURE);
|
||||
String appId = request.getHeader(APPID);
|
||||
if (StringUtils.isBlank(timestampStr)) {
|
||||
throw new InvalidArgumentException("header中timestamp不可为空");
|
||||
}
|
||||
if (StringUtils.isBlank(signature)) {
|
||||
throw new InvalidArgumentException("header中signature不可为空");
|
||||
}
|
||||
if (StringUtils.isBlank(appId)) {
|
||||
throw new InvalidArgumentException("header中appId不可为空");
|
||||
}
|
||||
AppDetailResp appDetailResp = appService.getApp(Integer.parseInt(appId));
|
||||
if (appDetailResp == null) {
|
||||
throw new InvalidArgumentException("该appId对应的应用不存在");
|
||||
}
|
||||
if (!AppStatus.ONLINE.equals(appDetailResp.getAppStatus())) {
|
||||
throw new InvalidArgumentException("该应用暂时为非在线状态");
|
||||
}
|
||||
Pair<Boolean, String> checkResult = SignatureUtils.isValidSignature(appId, appDetailResp.getAppSecret(),
|
||||
Long.parseLong(timestampStr), signature);
|
||||
if (!checkResult.first) {
|
||||
throw new InvalidArgumentException(checkResult.second);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,272 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.aspect;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
|
||||
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
|
||||
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
|
||||
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
||||
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
||||
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.QueryAuthorization;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class AuthCheckBaseAspect {
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat(
|
||||
new SimpleDateFormat(Constants.DAY_FORMAT));
|
||||
@Autowired
|
||||
private AuthService authService;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
@Autowired
|
||||
private MetricService metricService;
|
||||
|
||||
@Autowired
|
||||
private ModelService modelService;
|
||||
|
||||
public boolean doModelAdmin(User user, List<Long> modelIds) {
|
||||
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
|
||||
if (CollectionUtils.isEmpty(modelListAdmin)) {
|
||||
return false;
|
||||
} else {
|
||||
Set<Long> modelAdmins = modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet());
|
||||
return !CollectionUtils.isEmpty(modelAdmins) && modelAdmins.containsAll(modelIds);
|
||||
}
|
||||
}
|
||||
|
||||
public void doModelVisible(User user, List<Long> modelIds) {
|
||||
Boolean visible = true;
|
||||
List<ModelResp> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE);
|
||||
if (CollectionUtils.isEmpty(modelListVisible)) {
|
||||
visible = false;
|
||||
} else {
|
||||
Set<Long> modelVisibles = modelListVisible.stream().map(ModelResp::getId).collect(Collectors.toSet());
|
||||
if (!CollectionUtils.isEmpty(modelVisibles) && !modelVisibles.containsAll(modelIds)) {
|
||||
visible = false;
|
||||
}
|
||||
}
|
||||
if (!visible) {
|
||||
ModelResp modelResp = modelService.getModel(modelIds.get(0));
|
||||
String modelName = modelResp.getName();
|
||||
List<String> admins = modelService.getModelAdmin(modelResp.getId());
|
||||
String message = String.format("您没有模型[%s]权限，请联系管理员%s开通", modelName, admins);
|
||||
throw new InvalidPermissionException(message);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public Set<String> getHighSensitiveColsByModelId(List<Long> modelIds) {
|
||||
Set<String> highSensitiveCols = new HashSet<>();
|
||||
MetaFilter metaFilter = new MetaFilter();
|
||||
metaFilter.setModelIds(modelIds);
|
||||
metaFilter.setSensitiveLevel(SensitiveLevelEnum.HIGH.getCode());
|
||||
List<DimensionResp> highSensitiveDimensions = dimensionService.getDimensions(metaFilter);
|
||||
List<MetricResp> highSensitiveMetrics = metricService.getMetrics(metaFilter);
|
||||
if (!CollectionUtils.isEmpty(highSensitiveDimensions)) {
|
||||
highSensitiveDimensions.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(highSensitiveMetrics)) {
|
||||
highSensitiveMetrics.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
|
||||
}
|
||||
return highSensitiveCols;
|
||||
}
|
||||
|
||||
public AuthorizedResourceResp getAuthorizedResource(User user, List<Long> modelIds,
|
||||
Set<String> sensitiveResReq) {
|
||||
List<AuthRes> resourceReqList = new ArrayList<>();
|
||||
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelIds.get(0), res)));
|
||||
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
|
||||
queryAuthResReq.setResources(resourceReqList);
|
||||
queryAuthResReq.setModelIds(modelIds);
|
||||
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
|
||||
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelIds,
|
||||
authorizedResource);
|
||||
return authorizedResource;
|
||||
}
|
||||
|
||||
private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) {
|
||||
log.info("queryAuthResReq:{}", queryAuthResReq);
|
||||
return authService.queryAuthorizedResources(queryAuthResReq, user);
|
||||
}
|
||||
|
||||
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, List<Long> modelIds) {
|
||||
Set<String> resAuthName = new HashSet<>();
|
||||
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
|
||||
authResGrpList.stream().forEach(authResGrp -> {
|
||||
List<AuthRes> cols = authResGrp.getGroup();
|
||||
if (!CollectionUtils.isEmpty(cols)) {
|
||||
cols.stream().filter(col -> modelIds.contains(col.getModelId()))
|
||||
.forEach(col -> resAuthName.add(col.getName()));
|
||||
}
|
||||
|
||||
});
|
||||
log.info("resAuthName:{}", resAuthName);
|
||||
return resAuthName;
|
||||
}
|
||||
|
||||
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
|
||||
if (resAuthSet.containsAll(sensitiveResReq)) {
|
||||
return true;
|
||||
}
|
||||
log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet);
|
||||
return false;
|
||||
}
|
||||
|
||||
public QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns,
|
||||
List<Long> modelIds,
|
||||
AuthorizedResourceResp authResource) {
|
||||
addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet());
|
||||
return resultWithColumns;
|
||||
}
|
||||
|
||||
public QueryResultWithSchemaResp desensitizationData(QueryResultWithSchemaResp raw, Set<String> need2Apply) {
|
||||
log.debug("start desensitizationData logic");
|
||||
if (CollectionUtils.isEmpty(need2Apply)) {
|
||||
log.info("user has all sensitiveRes");
|
||||
return raw;
|
||||
}
|
||||
|
||||
List<QueryColumn> columns = raw.getColumns();
|
||||
|
||||
boolean doDesensitization = false;
|
||||
for (QueryColumn queryColumn : columns) {
|
||||
for (String sensitiveCol : need2Apply) {
|
||||
if (queryColumn.getNameEn().contains(sensitiveCol)) {
|
||||
doDesensitization = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!doDesensitization) {
|
||||
return raw;
|
||||
}
|
||||
|
||||
QueryResultWithSchemaResp queryResultWithColumns = raw;
|
||||
try {
|
||||
queryResultWithColumns = deepCopyResult(raw);
|
||||
} catch (Exception e) {
|
||||
log.warn("deepCopyResult: ", e);
|
||||
}
|
||||
addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply);
|
||||
desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply);
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
private void addAuthorizedSchemaInfo(List<QueryColumn> columns, Set<String> need2Apply) {
|
||||
if (CollectionUtils.isEmpty(need2Apply)) {
|
||||
return;
|
||||
}
|
||||
columns.stream().forEach(col -> {
|
||||
if (need2Apply.contains(col.getNameEn())) {
|
||||
col.setAuthorized(false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void desensitizationInternal(List<Map<String, Object>> result, Set<String> need2Apply) {
|
||||
log.info("start desensitizationInternal logic");
|
||||
for (int i = 0; i < result.size(); i++) {
|
||||
Map<String, Object> row = result.get(i);
|
||||
Map<String, Object> newRow = new HashMap<>();
|
||||
for (String col : row.keySet()) {
|
||||
boolean sensitive = false;
|
||||
for (String sensitiveCol : need2Apply) {
|
||||
if (col.contains(sensitiveCol)) {
|
||||
sensitive = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (sensitive) {
|
||||
newRow.put(col, "******");
|
||||
} else {
|
||||
newRow.put(col, row.get(col));
|
||||
}
|
||||
}
|
||||
result.set(i, newRow);
|
||||
}
|
||||
}
|
||||
|
||||
private QueryResultWithSchemaResp deepCopyResult(QueryResultWithSchemaResp raw) throws Exception {
|
||||
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
|
||||
BeanUtils.copyProperties(raw, queryResultWithColumns);
|
||||
|
||||
List<QueryColumn> columns = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(raw.getColumns())) {
|
||||
String columnsStr = MAPPER.writeValueAsString(raw.getColumns());
|
||||
columns = MAPPER.readValue(columnsStr, new TypeReference<List<QueryColumn>>() {
|
||||
});
|
||||
queryResultWithColumns.setColumns(columns);
|
||||
}
|
||||
queryResultWithColumns.setColumns(columns);
|
||||
|
||||
List<Map<String, Object>> resultData = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(raw.getResultList())) {
|
||||
for (Map<String, Object> line : raw.getResultList()) {
|
||||
Map<String, Object> newLine = new HashMap<>();
|
||||
newLine.putAll(line);
|
||||
resultData.add(newLine);
|
||||
}
|
||||
}
|
||||
queryResultWithColumns.setResultList(resultData);
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
public void addPromptInfoInfo(List<Long> modelIds, QueryResultWithSchemaResp queryResultWithColumns,
|
||||
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
|
||||
List<DimensionFilter> filters = authorizedResource.getFilters();
|
||||
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
|
||||
return;
|
||||
}
|
||||
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
|
||||
if (!CollectionUtils.isEmpty(need2Apply)) {
|
||||
String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins);
|
||||
queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(filters)) {
|
||||
log.debug("dimensionFilters:{}", filters);
|
||||
ModelResp modelResp = modelService.getModel(modelIds.get(0));
|
||||
List<String> exprList = new ArrayList<>();
|
||||
List<String> descList = new ArrayList<>();
|
||||
filters.stream().forEach(filter -> {
|
||||
descList.add(filter.getDescription());
|
||||
exprList.add(filter.getExpressions().toString());
|
||||
});
|
||||
String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s";
|
||||
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
|
||||
|
||||
queryResultWithColumns.setQueryAuthorization(
|
||||
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
|
||||
log.info("queryResultWithColumns:{}", queryResultWithColumns);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,312 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.aspect;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.aspectj.lang.ProceedingJoinPoint;
|
||||
import org.aspectj.lang.annotation.Around;
|
||||
import org.aspectj.lang.annotation.Aspect;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Aspect
|
||||
@Component
|
||||
@Slf4j
|
||||
public class DimValueAspect {
|
||||
|
||||
@Value("${dimension.value.map.enable:true}")
|
||||
private Boolean dimensionValueMapEnable;
|
||||
|
||||
@Value("${dimension.value.map.sql.enable:true}")
|
||||
private Boolean dimensionValueMapSqlEnable;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
|
||||
@Around("execution(* com.tencent.supersonic.headless.core.service.impl.QueryServiceImpl.queryBySql(..))")
|
||||
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||
if (!dimensionValueMapSqlEnable) {
|
||||
log.debug("sql dimensionValueMapEnable is false, skip dimensionValueMap");
|
||||
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
Object[] args = joinPoint.getArgs();
|
||||
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) args[0];
|
||||
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelIds()));
|
||||
String sql = queryS2SQLReq.getSql();
|
||||
log.info("correctorSql before replacing:{}", sql);
|
||||
// if dimensionvalue is alias,consider the true dimensionvalue.
|
||||
List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
|
||||
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
|
||||
Set<String> fieldNames = dimensions.stream().map(o -> o.getName()).collect(Collectors.toSet());
|
||||
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
|
||||
fieldExpressionList.stream().forEach(expression -> {
|
||||
if (fieldNames.contains(expression.getFieldName())) {
|
||||
dimensions.stream().forEach(dimension -> {
|
||||
if (expression.getFieldName().equals(dimension.getName())
|
||||
&& !CollectionUtils.isEmpty(dimension.getDimValueMaps())) {
|
||||
// consider '=' filter
|
||||
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())) {
|
||||
dimension.getDimValueMaps().stream().forEach(dimValue -> {
|
||||
if (!CollectionUtils.isEmpty(dimValue.getAlias())
|
||||
&& dimValue.getAlias().contains(expression.getFieldValue().toString())) {
|
||||
getFiledNameToValueMap(filedNameToValueMap, expression.getFieldValue().toString(),
|
||||
dimValue.getTechName(), expression.getFieldName());
|
||||
}
|
||||
});
|
||||
}
|
||||
// consider 'in' filter,each element needs to judge.
|
||||
replaceInCondition(expression, dimension, filedNameToValueMap);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
log.info("filedNameToValueMap:{}", filedNameToValueMap);
|
||||
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
|
||||
log.info("correctorSql after replacing:{}", sql);
|
||||
queryS2SQLReq.setSql(sql);
|
||||
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
|
||||
|
||||
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
|
||||
if (Objects.nonNull(queryResultWithColumns)) {
|
||||
rewriteDimValue(queryResultWithColumns, techNameToBizName);
|
||||
}
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
public void replaceInCondition(FieldExpression expression, DimensionResp dimension,
|
||||
Map<String, Map<String, String>> filedNameToValueMap) {
|
||||
if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) {
|
||||
String fieldValue = JsonUtil.toString(expression.getFieldValue());
|
||||
fieldValue = fieldValue.replace("'", "");
|
||||
List<String> values = JsonUtil.toList(fieldValue, String.class);
|
||||
List<String> revisedValues = new ArrayList<>();
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
Boolean flag = new Boolean(false);
|
||||
for (DimValueMap dimValueMap : dimension.getDimValueMaps()) {
|
||||
if (!CollectionUtils.isEmpty(dimValueMap.getAlias())
|
||||
&& dimValueMap.getAlias().contains(values.get(i))) {
|
||||
flag = true;
|
||||
revisedValues.add(dimValueMap.getTechName());
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!flag) {
|
||||
revisedValues.add(values.get(i));
|
||||
}
|
||||
}
|
||||
if (!revisedValues.equals(values)) {
|
||||
getFiledNameToValueMap(filedNameToValueMap, JsonUtil.toString(values),
|
||||
JsonUtil.toString(revisedValues), expression.getFieldName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void getFiledNameToValueMap(Map<String, Map<String, String>> filedNameToValueMap,
|
||||
String oldValue, String newValue, String fieldName) {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
map.put(oldValue, newValue);
|
||||
filedNameToValueMap.put(fieldName, map);
|
||||
}
|
||||
|
||||
@Around("execution(* com.tencent.supersonic.headless.core.rest.QueryController.queryByStruct(..))"
|
||||
+ " || execution(* com.tencent.supersonic.headless.core.service.QueryService.queryByStruct(..))"
|
||||
+ " || execution(* com.tencent.supersonic.headless.core.service.QueryService.queryByStructWithAuth(..))")
|
||||
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||
|
||||
if (!dimensionValueMapEnable) {
|
||||
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
|
||||
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
Object[] args = joinPoint.getArgs();
|
||||
QueryStructReq queryStructReq = (QueryStructReq) args[0];
|
||||
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryStructReq.getModelIds()));
|
||||
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
|
||||
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions);
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions);
|
||||
|
||||
rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair);
|
||||
|
||||
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
|
||||
if (Objects.nonNull(queryResultWithColumns)) {
|
||||
rewriteDimValue(queryResultWithColumns, dimAndTechNameAndBizNamePair);
|
||||
}
|
||||
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns,
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
|
||||
if (!selectDimValueMap(queryResultWithColumns.getColumns(), dimAndTechNameAndBizNamePair)) {
|
||||
return;
|
||||
}
|
||||
log.debug("start rewriteDimValue for resultList");
|
||||
for (Map<String, Object> line : queryResultWithColumns.getResultList()) {
|
||||
for (String bizName : line.keySet()) {
|
||||
if (dimAndTechNameAndBizNamePair.containsKey(bizName) && Objects.nonNull(line.get(bizName))) {
|
||||
String techName = line.get(bizName).toString();
|
||||
Map<String, String> techAndBizPair = dimAndTechNameAndBizNamePair.get(bizName);
|
||||
if (!CollectionUtils.isEmpty(techAndBizPair) && techAndBizPair.containsKey(techName)) {
|
||||
String bizValueName = techAndBizPair.get(techName);
|
||||
if (Strings.isNotEmpty(bizValueName)) {
|
||||
line.put(bizName, bizValueName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean selectDimValueMap(List<QueryColumn> columns, Map<String,
|
||||
Map<String, String>> dimAndTechNameAndBizNamePair) {
|
||||
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)
|
||||
|| CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (QueryColumn queryColumn : columns) {
|
||||
if (dimAndTechNameAndBizNamePair.containsKey(queryColumn.getNameEn())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void rewriteFilter(List<Filter> dimensionFilters, Map<String, Map<String, String>> aliasAndTechNamePair) {
|
||||
for (Filter filter : dimensionFilters) {
|
||||
if (Objects.isNull(filter)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (CollectionUtils.isEmpty(filter.getChildren())) {
|
||||
Object value = filter.getValue();
|
||||
String bizName = filter.getBizName();
|
||||
if (aliasAndTechNamePair.containsKey(bizName)) {
|
||||
Map<String, String> aliasPair = aliasAndTechNamePair.get(bizName);
|
||||
if (Objects.nonNull(value)) {
|
||||
if (value instanceof List) {
|
||||
List<String> values = (List) value;
|
||||
List<String> valuesNew = new ArrayList<>();
|
||||
for (String valueSingle : values) {
|
||||
if (aliasPair.containsKey(valueSingle)) {
|
||||
valuesNew.add(aliasPair.get(valueSingle));
|
||||
} else {
|
||||
valuesNew.add(valueSingle);
|
||||
}
|
||||
}
|
||||
filter.setValue(valuesNew);
|
||||
}
|
||||
if (value instanceof String) {
|
||||
if (aliasPair.containsKey(value)) {
|
||||
filter.setValue(aliasPair.get(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
rewriteFilter(filter.getChildren(), aliasAndTechNamePair);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Map<String, String>> getAliasAndBizNameToTechName(List<DimensionResp> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensions)) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
Map<String, Map<String, String>> result = new HashMap<>();
|
||||
for (DimensionResp dimension : dimensions) {
|
||||
if (needSkipDimension(dimension)) {
|
||||
continue;
|
||||
}
|
||||
String bizName = dimension.getBizName();
|
||||
List<DimValueMap> dimValueMaps = dimension.getDimValueMaps();
|
||||
Map<String, String> aliasAndBizNameToTechName = new HashMap<>();
|
||||
|
||||
for (DimValueMap dimValueMap : dimValueMaps) {
|
||||
if (needSkipDimValue(dimValueMap)) {
|
||||
continue;
|
||||
}
|
||||
if (Strings.isNotEmpty(dimValueMap.getBizName())) {
|
||||
aliasAndBizNameToTechName.put(dimValueMap.getBizName(), dimValueMap.getTechName());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(dimValueMap.getAlias())) {
|
||||
dimValueMap.getAlias().stream().forEach(alias -> {
|
||||
if (Strings.isNotEmpty(alias)) {
|
||||
aliasAndBizNameToTechName.put(alias, dimValueMap.getTechName());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!CollectionUtils.isEmpty(aliasAndBizNameToTechName)) {
|
||||
result.put(bizName, aliasAndBizNameToTechName);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean needSkipDimValue(DimValueMap dimValueMap) {
|
||||
return Objects.isNull(dimValueMap) || Strings.isEmpty(dimValueMap.getTechName());
|
||||
}
|
||||
|
||||
private Map<String, Map<String, String>> getTechNameToBizName(List<DimensionResp> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensions)) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
Map<String, Map<String, String>> result = new HashMap<>();
|
||||
for (DimensionResp dimension : dimensions) {
|
||||
if (needSkipDimension(dimension)) {
|
||||
continue;
|
||||
}
|
||||
String bizName = dimension.getBizName();
|
||||
List<DimValueMap> dimValueMaps = dimension.getDimValueMaps();
|
||||
Map<String, String> techNameToBizName = new HashMap<>();
|
||||
|
||||
for (DimValueMap dimValueMap : dimValueMaps) {
|
||||
if (needSkipDimValue(dimValueMap)) {
|
||||
continue;
|
||||
}
|
||||
if (StringUtils.isNotEmpty(dimValueMap.getBizName())) {
|
||||
techNameToBizName.put(dimValueMap.getTechName(), dimValueMap.getBizName());
|
||||
}
|
||||
}
|
||||
|
||||
if (!CollectionUtils.isEmpty(techNameToBizName)) {
|
||||
result.put(bizName, techNameToBizName);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean needSkipDimension(DimensionResp dimension) {
|
||||
return Objects.isNull(dimension) || Strings.isEmpty(dimension.getBizName()) || CollectionUtils.isEmpty(
|
||||
dimension.getDimValueMaps());
|
||||
}
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.aspect;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import net.sf.jsqlparser.JSQLParserException;
|
||||
import net.sf.jsqlparser.expression.Expression;
|
||||
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.aspectj.lang.ProceedingJoinPoint;
|
||||
import org.aspectj.lang.annotation.Around;
|
||||
import org.aspectj.lang.annotation.Aspect;
|
||||
import org.aspectj.lang.annotation.Pointcut;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.StringJoiner;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
|
||||
|
||||
@Component
|
||||
@Aspect
|
||||
@Order(1)
|
||||
@Slf4j
|
||||
public class S2SQLDataAspect extends AuthCheckBaseAspect {
|
||||
|
||||
@Autowired
|
||||
private QueryStructUtils queryStructUtils;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
@Autowired
|
||||
private ModelService modelService;
|
||||
@Value("${permission.data.enable:true}")
|
||||
private Boolean permissionDataEnable;
|
||||
|
||||
@Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.S2SQLDataPermission)")
|
||||
private void s2SQLPermissionCheck() {
|
||||
}
|
||||
|
||||
@Around("s2SQLPermissionCheck()")
|
||||
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||
log.info("s2SQL permission check!");
|
||||
Object[] objects = joinPoint.getArgs();
|
||||
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) objects[0];
|
||||
User user = (User) objects[1];
|
||||
if (!permissionDataEnable) {
|
||||
log.info("not to check s2SQL permission!");
|
||||
return joinPoint.proceed();
|
||||
}
|
||||
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
|
||||
throw new RuntimeException("please provide user information");
|
||||
}
|
||||
List<Long> modelIds = queryS2SQLReq.getModelIds();
|
||||
|
||||
//1. determine whether admin of the model
|
||||
if (doModelAdmin(user, modelIds)) {
|
||||
log.info("determine whether admin of the model!");
|
||||
return joinPoint.proceed();
|
||||
}
|
||||
// 2. determine whether the subject field is visible
|
||||
doModelVisible(user, modelIds);
|
||||
// 3. fetch data permission meta information
|
||||
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2SQLReq, user);
|
||||
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
|
||||
|
||||
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
|
||||
Set<String> sensitiveResReq = res4Privilege.parallelStream()
|
||||
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
|
||||
log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq);
|
||||
|
||||
// query user privilege info
|
||||
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIds, sensitiveResReq);
|
||||
// get sensitiveRes that user has privilege
|
||||
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIds);
|
||||
|
||||
// 4.if sensitive fields without permission are involved in filter, thrown an exception
|
||||
doFilterCheckLogic(queryS2SQLReq, resAuthSet, sensitiveResReq);
|
||||
|
||||
// 5.row permission pre-filter
|
||||
doRowPermission(queryS2SQLReq, authorizedResource);
|
||||
|
||||
// 6.proceed
|
||||
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
|
||||
|
||||
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
|
||||
// if sensitiveRes is empty
|
||||
log.info("sensitiveResReq is empty");
|
||||
return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
|
||||
}
|
||||
|
||||
// 6.if the column has no permission, hit *
|
||||
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
|
||||
.collect(Collectors.toSet());
|
||||
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
|
||||
QueryResultWithSchemaResp queryResultAfterDesensitization =
|
||||
desensitizationData(queryResultWithColumns, need2Apply);
|
||||
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
|
||||
|
||||
return queryResultAfterDesensitization;
|
||||
}
|
||||
|
||||
private void doRowPermission(QueryS2SQLReq queryS2SQLReq, AuthorizedResourceResp authorizedResource) {
|
||||
log.debug("start doRowPermission logic");
|
||||
StringJoiner joiner = new StringJoiner(" OR ");
|
||||
List<String> dimensionFilters = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(authorizedResource.getFilters())) {
|
||||
authorizedResource.getFilters().stream()
|
||||
.forEach(filter -> dimensionFilters.addAll(filter.getExpressions()));
|
||||
}
|
||||
|
||||
if (CollectionUtils.isEmpty(dimensionFilters)) {
|
||||
log.debug("dimensionFilters is empty");
|
||||
return;
|
||||
}
|
||||
|
||||
dimensionFilters.stream().forEach(filter -> {
|
||||
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
|
||||
joiner.add(" ( " + filter + " ) ");
|
||||
}
|
||||
});
|
||||
try {
|
||||
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
|
||||
if (StringUtils.isNotEmpty(joiner.toString())) {
|
||||
String sql = SqlParserAddHelper.addWhere(queryS2SQLReq.getSql(), expression);
|
||||
log.info("before doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
|
||||
queryS2SQLReq.setSql(sql);
|
||||
log.info("after doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
|
||||
}
|
||||
} catch (JSQLParserException jsqlParserException) {
|
||||
log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void doFilterCheckLogic(QueryS2SQLReq queryS2SQLReq, Set<String> resAuthName,
|
||||
Set<String> sensitiveResReq) {
|
||||
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2SQLReq);
|
||||
Set<String> need2Apply = resFilterSet.stream()
|
||||
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
|
||||
Set<String> nameCnSet = new HashSet<>();
|
||||
|
||||
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds());
|
||||
ModelFilter modelFilter = new ModelFilter();
|
||||
modelFilter.setModelIds(modelIds);
|
||||
List<ModelResp> modelInfos = modelService.getModelList(modelFilter);
|
||||
String modelNameCn = Constants.EMPTY;
|
||||
if (!CollectionUtils.isEmpty(modelInfos)) {
|
||||
modelNameCn = modelInfos.get(0).getName();
|
||||
}
|
||||
MetaFilter metaFilter = new MetaFilter(modelIds);
|
||||
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(metaFilter);
|
||||
String finalDomainNameCn = modelNameCn;
|
||||
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
|
||||
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));
|
||||
|
||||
if (!CollectionUtils.isEmpty(need2Apply)) {
|
||||
ModelResp modelResp = modelInfos.get(0);
|
||||
List<String> admins = modelService.getModelAdmin(modelResp.getId());
|
||||
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
|
||||
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
|
||||
throw new InvalidPermissionException(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.aspect;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.aspectj.lang.ProceedingJoinPoint;
|
||||
import org.aspectj.lang.annotation.Around;
|
||||
import org.aspectj.lang.annotation.Aspect;
|
||||
import org.aspectj.lang.annotation.Pointcut;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.StringJoiner;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
|
||||
|
||||
@Component
|
||||
@Aspect
|
||||
@Slf4j
|
||||
public class StructDataAspect extends AuthCheckBaseAspect {
|
||||
@Autowired
|
||||
private QueryStructUtils queryStructUtils;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
@Autowired
|
||||
private ModelService modelService;
|
||||
@Value("${permission.data.enable:true}")
|
||||
private Boolean permissionDataEnable;
|
||||
|
||||
@Pointcut("@annotation(com.tencent.supersonic.headless.core.annotation.StructDataPermission)")
|
||||
public void dataPermissionAOP() {
|
||||
}
|
||||
|
||||
@Around(value = "dataPermissionAOP()")
|
||||
public Object around(ProceedingJoinPoint point) throws Throwable {
|
||||
Object[] args = point.getArgs();
|
||||
QueryStructReq queryStructReq = (QueryStructReq) args[0];
|
||||
User user = (User) args[1];
|
||||
|
||||
if (!permissionDataEnable) {
|
||||
log.info("permissionDataEnable is false");
|
||||
return point.proceed();
|
||||
}
|
||||
|
||||
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
|
||||
throw new RuntimeException("lease provide user information");
|
||||
}
|
||||
//1. determine whether admin of the model
|
||||
if (doModelAdmin(user, queryStructReq.getModelIds())) {
|
||||
return point.proceed();
|
||||
}
|
||||
|
||||
// 2. determine whether the subject field is visible
|
||||
doModelVisible(user, queryStructReq.getModelIds());
|
||||
|
||||
// 3. fetch data permission meta information
|
||||
List<Long> modelIds = queryStructReq.getModelIds();
|
||||
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
|
||||
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
|
||||
|
||||
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
|
||||
Set<String> sensitiveResReq = res4Privilege.parallelStream()
|
||||
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
|
||||
log.info("this query domainId:{}, sensitiveResReq:{}", modelIds, sensitiveResReq);
|
||||
|
||||
// query user privilege info
|
||||
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
|
||||
modelIds, sensitiveResReq);
|
||||
// get sensitiveRes that user has privilege
|
||||
Set<String> resAuthSet = getAuthResNameSet(authorizedResource,
|
||||
queryStructReq.getModelIds());
|
||||
|
||||
// 4.if sensitive fields without permission are involved in filter, thrown an exception
|
||||
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
|
||||
|
||||
// 5.row permission pre-filter
|
||||
doRowPermission(queryStructReq, authorizedResource);
|
||||
|
||||
// 6.proceed
|
||||
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) point.proceed();
|
||||
|
||||
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
|
||||
// if sensitiveRes is empty
|
||||
log.info("sensitiveResReq is empty");
|
||||
return getQueryResultWithColumns(queryResultWithColumns, modelIds, authorizedResource);
|
||||
}
|
||||
|
||||
// 6.if the column has no permission, hit *
|
||||
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
|
||||
.collect(Collectors.toSet());
|
||||
QueryResultWithSchemaResp queryResultAfterDesensitization =
|
||||
desensitizationData(queryResultWithColumns, need2Apply);
|
||||
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
|
||||
|
||||
return queryResultAfterDesensitization;
|
||||
|
||||
}
|
||||
|
||||
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
|
||||
if (resAuthSet.containsAll(sensitiveResReq)) {
|
||||
return true;
|
||||
}
|
||||
log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet);
|
||||
return false;
|
||||
}
|
||||
|
||||
private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
|
||||
log.debug("start doRowPermission logic");
|
||||
StringJoiner joiner = new StringJoiner(" OR ");
|
||||
List<String> dimensionFilters = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(authorizedResource.getFilters())) {
|
||||
authorizedResource.getFilters().stream()
|
||||
.forEach(filter -> dimensionFilters.addAll(filter.getExpressions()));
|
||||
}
|
||||
|
||||
if (CollectionUtils.isEmpty(dimensionFilters)) {
|
||||
log.debug("dimensionFilters is empty");
|
||||
return;
|
||||
}
|
||||
|
||||
dimensionFilters.stream().forEach(filter -> {
|
||||
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
|
||||
joiner.add(" ( " + filter + " ) ");
|
||||
}
|
||||
});
|
||||
|
||||
if (StringUtils.isNotEmpty(joiner.toString())) {
|
||||
log.info("before doRowPermission, queryStructReq:{}", queryStructReq);
|
||||
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString());
|
||||
List<Filter> filters = Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>()
|
||||
: queryStructReq.getOriginalFilter();
|
||||
filters.add(filter);
|
||||
queryStructReq.setDimensionFilters(filters);
|
||||
log.info("after doRowPermission, queryStructReq:{}", queryStructReq);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void doFilterCheckLogic(QueryStructReq queryStructReq, Set<String> resAuthName,
|
||||
Set<String> sensitiveResReq) {
|
||||
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
|
||||
Set<String> need2Apply = resFilterSet.stream()
|
||||
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
|
||||
Set<String> nameCnSet = new HashSet<>();
|
||||
|
||||
Map<Long, ModelResp> modelRespMap = modelService.getModelMap();
|
||||
List<Long> modelIds = Lists.newArrayList(queryStructReq.getModelIds());
|
||||
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIds));
|
||||
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
|
||||
.forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName()));
|
||||
|
||||
if (!CollectionUtils.isEmpty(need2Apply)) {
|
||||
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
|
||||
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
|
||||
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
|
||||
throw new InvalidPermissionException(message);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,9 +2,8 @@ package com.tencent.supersonic.headless.core.executor;
|
||||
|
||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.server.utils.SqlUtils;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.utils.SqlUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.stereotype.Component;
|
||||
@@ -25,13 +24,13 @@ public class JdbcExecutor implements QueryExecutor {
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResultWithSchemaResp execute(Catalog catalog, QueryStatement queryStatement) {
|
||||
public QueryResultWithSchemaResp execute(QueryStatement queryStatement) {
|
||||
if (Strings.isEmpty(queryStatement.getSourceId())) {
|
||||
log.warn("data base id is empty");
|
||||
return null;
|
||||
}
|
||||
log.info("query SQL: {}", queryStatement.getSql());
|
||||
DatabaseResp databaseResp = catalog.getDatabase(Long.parseLong(queryStatement.getSourceId()));
|
||||
DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp();
|
||||
log.info("database info:{}", databaseResp);
|
||||
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
|
||||
SqlUtils sqlUtils = this.sqlUtils.init(databaseResp);
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
package com.tencent.supersonic.headless.core.executor;
|
||||
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
|
||||
public interface QueryExecutor {
|
||||
|
||||
boolean accept(QueryStatement queryStatement);
|
||||
|
||||
QueryResultWithSchemaResp execute(Catalog catalog, QueryStatement queryStatement);
|
||||
QueryResultWithSchemaResp execute(QueryStatement queryStatement);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.tencent.supersonic.headless.core.manager;
|
||||
|
||||
|
||||
import com.tencent.supersonic.headless.api.enums.IdentifyType;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DimensionYamlManager {
|
||||
|
||||
public static List<DimensionYamlTpl> convert2DimensionYaml(List<DimensionResp> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensions)) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
return dimensions.stream()
|
||||
.filter(dimension -> !dimension.getType().equalsIgnoreCase(IdentifyType.primary.name()))
|
||||
.map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) {
|
||||
DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl();
|
||||
BeanUtils.copyProperties(dimension, dimensionYamlTpl);
|
||||
dimensionYamlTpl.setName(dimension.getBizName());
|
||||
dimensionYamlTpl.setOwners(dimension.getCreatedBy());
|
||||
return dimensionYamlTpl;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.tencent.supersonic.headless.core.manager;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.headless.api.pojo.Measure;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricTypeParams;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class MetricYamlManager {
|
||||
|
||||
public static List<MetricYamlTpl> convert2YamlObj(List<MetricResp> metrics) {
|
||||
|
||||
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
|
||||
for (MetricResp metric : metrics) {
|
||||
MetricYamlTpl metricYamlTpl = convert2MetricYamlTpl(metric);
|
||||
metricYamlTpls.add(metricYamlTpl);
|
||||
}
|
||||
return metricYamlTpls;
|
||||
}
|
||||
|
||||
public static MetricYamlTpl convert2MetricYamlTpl(MetricResp metric) {
|
||||
MetricYamlTpl metricYamlTpl = new MetricYamlTpl();
|
||||
BeanUtils.copyProperties(metric, metricYamlTpl);
|
||||
metricYamlTpl.setName(metric.getBizName());
|
||||
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
|
||||
MetricTypeParams exprMetricTypeParams = metric.getTypeParams();
|
||||
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
|
||||
metricTypeParamsYamlTpl.setExpr(exprMetricTypeParams.getExpr());
|
||||
List<Measure> measures = exprMetricTypeParams.getMeasures();
|
||||
metricTypeParamsYamlTpl.setMeasures(
|
||||
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
|
||||
metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl);
|
||||
return metricYamlTpl;
|
||||
}
|
||||
|
||||
public static MeasureYamlTpl convert(Measure measure) {
|
||||
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
|
||||
measureYamlTpl.setName(measure.getBizName());
|
||||
return measureYamlTpl;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package com.tencent.supersonic.headless.core.manager;
|
||||
|
||||
import com.tencent.supersonic.headless.api.enums.DatasourceQuery;
|
||||
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
|
||||
import com.tencent.supersonic.headless.api.pojo.Dim;
|
||||
import com.tencent.supersonic.headless.api.pojo.Identify;
|
||||
import com.tencent.supersonic.headless.api.pojo.Measure;
|
||||
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
|
||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
|
||||
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
|
||||
import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ModelYamlManager {
|
||||
|
||||
public static DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) {
|
||||
ModelDetail modelDetail = modelResp.getModelDetail();
|
||||
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
|
||||
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
|
||||
addInterCntMetric(modelResp.getBizName(), modelDetail);
|
||||
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
|
||||
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);
|
||||
dataModelYamlTpl.setIdentifiers(modelDetail.getIdentifiers().stream().map(ModelYamlManager::convert)
|
||||
.collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setDimensions(modelDetail.getDimensions().stream().map(ModelYamlManager::convert)
|
||||
.collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setMeasures(modelDetail.getMeasures().stream().map(ModelYamlManager::convert)
|
||||
.collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setName(modelResp.getBizName());
|
||||
dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
|
||||
dataModelYamlTpl.setModelSourceTypeEnum(ModelSourceType.of(modelResp.getSourceType()));
|
||||
if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) {
|
||||
dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery());
|
||||
} else {
|
||||
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
|
||||
}
|
||||
return dataModelYamlTpl;
|
||||
}
|
||||
|
||||
public static DimensionYamlTpl convert(Dim dim) {
|
||||
DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl();
|
||||
BeanUtils.copyProperties(dim, dimensionYamlTpl);
|
||||
dimensionYamlTpl.setName(dim.getBizName());
|
||||
if (Objects.isNull(dimensionYamlTpl.getExpr())) {
|
||||
dimensionYamlTpl.setExpr(dim.getBizName());
|
||||
}
|
||||
if (dim.getTypeParams() != null) {
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl = new DimensionTimeTypeParamsTpl();
|
||||
dimensionTimeTypeParamsTpl.setIsPrimary(dim.getTypeParams().getIsPrimary());
|
||||
dimensionTimeTypeParamsTpl.setTimeGranularity(dim.getTypeParams().getTimeGranularity());
|
||||
dimensionYamlTpl.setTypeParams(dimensionTimeTypeParamsTpl);
|
||||
}
|
||||
return dimensionYamlTpl;
|
||||
}
|
||||
|
||||
public static MeasureYamlTpl convert(Measure measure) {
|
||||
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
|
||||
BeanUtils.copyProperties(measure, measureYamlTpl);
|
||||
measureYamlTpl.setName(measure.getBizName());
|
||||
return measureYamlTpl;
|
||||
}
|
||||
|
||||
public static IdentifyYamlTpl convert(Identify identify) {
|
||||
IdentifyYamlTpl identifyYamlTpl = new IdentifyYamlTpl();
|
||||
identifyYamlTpl.setName(identify.getBizName());
|
||||
identifyYamlTpl.setType(identify.getType());
|
||||
return identifyYamlTpl;
|
||||
}
|
||||
|
||||
private static void addInterCntMetric(String datasourceEnName, ModelDetail datasourceDetail) {
|
||||
Measure measure = new Measure();
|
||||
measure.setExpr("1");
|
||||
if (!CollectionUtils.isEmpty(datasourceDetail.getIdentifiers())) {
|
||||
measure.setExpr(datasourceDetail.getIdentifiers().get(0).getBizName());
|
||||
}
|
||||
measure.setAgg("count");
|
||||
measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt"));
|
||||
measure.setCreateMetric("true");
|
||||
datasourceDetail.getMeasures().add(measure);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,14 +2,14 @@ package com.tencent.supersonic.headless.core.optimizer;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Component("DetailQuery")
|
||||
public class DetailQuery implements QueryOptimizer {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.tencent.supersonic.headless.core.optimizer;
|
||||
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
|
||||
public interface QueryOptimizer {
|
||||
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
package com.tencent.supersonic.headless.core.parser;
|
||||
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
|
||||
public interface HeadlessConverter {
|
||||
|
||||
boolean accept(QueryStatement queryStatement);
|
||||
|
||||
void converter(Catalog catalog, QueryStatement queryStatement) throws Exception;
|
||||
void convert(QueryStatement queryStatement) throws Exception;
|
||||
|
||||
}
|
||||
|
||||
@@ -6,9 +6,8 @@ import com.tencent.supersonic.headless.api.pojo.MetricTable;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
@@ -25,13 +24,6 @@ import java.util.stream.Collectors;
|
||||
@Primary
|
||||
public class QueryParser {
|
||||
|
||||
|
||||
private final Catalog catalog;
|
||||
|
||||
public QueryParser(Catalog catalog) {
|
||||
this.catalog = catalog;
|
||||
}
|
||||
|
||||
public QueryStatement logicSql(QueryStatement queryStatement) throws Exception {
|
||||
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
|
||||
if (Objects.isNull(queryStatement.getParseSqlReq())) {
|
||||
@@ -44,7 +36,7 @@ public class QueryParser {
|
||||
for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) {
|
||||
if (headlessConverter.accept(queryStatement)) {
|
||||
log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
|
||||
headlessConverter.converter(catalog, queryStatement);
|
||||
headlessConverter.convert(queryStatement);
|
||||
}
|
||||
}
|
||||
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(),
|
||||
@@ -58,24 +50,26 @@ public class QueryParser {
|
||||
|
||||
}
|
||||
|
||||
public QueryStatement parser(ParseSqlReq sqlCommend, QueryStatement queryStatement) {
|
||||
log.info("parser MetricReq [{}] ", sqlCommend);
|
||||
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) {
|
||||
log.info("parser MetricReq [{}] ", parseSqlReq);
|
||||
try {
|
||||
if (!CollectionUtils.isEmpty(sqlCommend.getTables())) {
|
||||
if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) {
|
||||
List<String[]> tables = new ArrayList<>();
|
||||
String sourceId = "";
|
||||
for (MetricTable metricTable : sqlCommend.getTables()) {
|
||||
for (MetricTable metricTable : parseSqlReq.getTables()) {
|
||||
MetricQueryReq metricReq = new MetricQueryReq();
|
||||
metricReq.setMetrics(metricTable.getMetrics());
|
||||
metricReq.setDimensions(metricTable.getDimensions());
|
||||
metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
|
||||
metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
|
||||
metricReq.setRootPath(sqlCommend.getRootPath());
|
||||
metricReq.setRootPath(parseSqlReq.getRootPath());
|
||||
QueryStatement tableSql = new QueryStatement();
|
||||
tableSql.setIsS2SQL(false);
|
||||
tableSql.setMetricReq(metricReq);
|
||||
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
|
||||
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
|
||||
tableSql.setModelIds(queryStatement.getModelIds());
|
||||
tableSql.setHeadlessModel(queryStatement.getHeadlessModel());
|
||||
tableSql = parser(tableSql, metricTable.getAggOption());
|
||||
if (!tableSql.isOk()) {
|
||||
queryStatement.setErrMsg(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
|
||||
@@ -88,20 +82,20 @@ public class QueryParser {
|
||||
|
||||
if (!tables.isEmpty()) {
|
||||
String sql = "";
|
||||
if (sqlCommend.isSupportWith()) {
|
||||
if (parseSqlReq.isSupportWith()) {
|
||||
sql = "with " + String.join(",",
|
||||
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
|
||||
Collectors.toList())) + "\n" + sqlCommend.getSql();
|
||||
Collectors.toList())) + "\n" + parseSqlReq.getSql();
|
||||
} else {
|
||||
sql = sqlCommend.getSql();
|
||||
sql = parseSqlReq.getSql();
|
||||
for (String[] tb : tables) {
|
||||
sql = StringUtils.replace(sql, tb[0],
|
||||
"(" + tb[1] + ") " + (sqlCommend.isWithAlias() ? "" : tb[0]), -1);
|
||||
"(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1);
|
||||
}
|
||||
}
|
||||
queryStatement.setSql(sql);
|
||||
queryStatement.setSourceId(sourceId);
|
||||
queryStatement.setParseSqlReq(sqlCommend);
|
||||
queryStatement.setParseSqlReq(parseSqlReq);
|
||||
return queryStatement;
|
||||
}
|
||||
}
|
||||
@@ -117,18 +111,18 @@ public class QueryParser {
|
||||
}
|
||||
|
||||
public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
|
||||
MetricQueryReq metricCommand = queryStatement.getMetricReq();
|
||||
log.info("parser MetricReq [{}] isAgg [{}]", metricCommand, isAgg);
|
||||
if (metricCommand.getRootPath().isEmpty()) {
|
||||
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
|
||||
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg);
|
||||
if (metricQueryReq.getRootPath().isEmpty()) {
|
||||
queryStatement.setErrMsg("rootPath empty");
|
||||
return queryStatement;
|
||||
}
|
||||
try {
|
||||
queryStatement = ComponentFactory.getSqlParser().explain(queryStatement, isAgg, catalog);
|
||||
queryStatement = ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
|
||||
return queryStatement;
|
||||
} catch (Exception e) {
|
||||
queryStatement.setErrMsg(e.getMessage());
|
||||
log.error("parser error MetricCommand[{}] error [{}]", metricCommand, e);
|
||||
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
|
||||
}
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
package com.tencent.supersonic.headless.core.parser;
|
||||
|
||||
import com.tencent.supersonic.headless.api.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
|
||||
public interface SqlParser {
|
||||
|
||||
QueryStatement explain(QueryStatement queryStatement, AggOption aggOption, Catalog catalog) throws Exception;
|
||||
QueryStatement explain(QueryStatement queryStatement, AggOption aggOption) throws Exception;
|
||||
}
|
||||
|
||||
@@ -7,24 +7,16 @@ import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.RuntimeOptions;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component("CalciteSqlParser")
|
||||
public class CalciteSqlParser implements SqlParser {
|
||||
|
||||
private final HeadlessSchemaManager headlessSchemaManager;
|
||||
|
||||
public CalciteSqlParser(
|
||||
HeadlessSchemaManager headlessSchemaManager) {
|
||||
this.headlessSchemaManager = headlessSchemaManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg, Catalog catalog) throws Exception {
|
||||
public QueryStatement explain(QueryStatement queryStatement, AggOption isAgg) throws Exception {
|
||||
MetricQueryReq metricReq = queryStatement.getMetricReq();
|
||||
HeadlessModel headlessModel = headlessSchemaManager.get(metricReq.getRootPath());
|
||||
HeadlessModel headlessModel = queryStatement.getHeadlessModel();
|
||||
if (headlessModel == null) {
|
||||
queryStatement.setErrMsg("semanticSchema not found");
|
||||
return queryStatement;
|
||||
|
||||
@@ -1,336 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.parser.calcite;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.tencent.supersonic.common.pojo.ModelRela;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.tuple.Triple;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class HeadlessSchemaManager {
|
||||
|
||||
@Autowired
|
||||
private LoadingCache<String, HeadlessModel> loadingCache;
|
||||
private final Catalog catalog;
|
||||
|
||||
public HeadlessSchemaManager(Catalog catalog) {
|
||||
this.catalog = catalog;
|
||||
}
|
||||
|
||||
public HeadlessModel reload(String rootPath) {
|
||||
HeadlessModel headlessModel = new HeadlessModel();
|
||||
headlessModel.setRootPath(rootPath);
|
||||
Set<Long> modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim()))
|
||||
.collect(Collectors.toSet());
|
||||
if (modelIds.isEmpty()) {
|
||||
log.error("get modelIds empty {}", rootPath);
|
||||
return headlessModel;
|
||||
}
|
||||
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
|
||||
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
|
||||
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
|
||||
Map<Long, String> modelIdName = new HashMap<>();
|
||||
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
|
||||
List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
|
||||
if (!CollectionUtils.isEmpty(modelRelas)) {
|
||||
headlessModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
|
||||
}
|
||||
if (!dataModelYamlTpls.isEmpty()) {
|
||||
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(d -> getDatasource(d))
|
||||
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
|
||||
headlessModel.setDatasourceMap(dataSourceMap);
|
||||
}
|
||||
if (!dimensionYamlTpls.isEmpty()) {
|
||||
Map<String, List<Dimension>> dimensionMap = new HashMap<>();
|
||||
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
|
||||
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
|
||||
}
|
||||
headlessModel.setDimensionMap(dimensionMap);
|
||||
}
|
||||
if (!metricYamlTpls.isEmpty()) {
|
||||
headlessModel.setMetrics(getMetrics(metricYamlTpls));
|
||||
}
|
||||
return headlessModel;
|
||||
}
|
||||
|
||||
//private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>();
|
||||
public HeadlessModel get(String rootPath) throws Exception {
|
||||
rootPath = formatKey(rootPath);
|
||||
HeadlessModel schema = loadingCache.get(rootPath);
|
||||
if (schema == null) {
|
||||
return null;
|
||||
}
|
||||
return schema;
|
||||
}
|
||||
|
||||
public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
|
||||
return getMetricsByMetricYamlTpl(t);
|
||||
}
|
||||
|
||||
public static List<Dimension> getDimensions(final List<DimensionYamlTpl> t) {
|
||||
return getDimension(t);
|
||||
}
|
||||
|
||||
public static DataSource getDatasource(final DataModelYamlTpl d) {
|
||||
DataSource datasource = DataSource.builder().sourceId(d.getSourceId()).sqlQuery(d.getSqlQuery())
|
||||
.name(d.getName()).tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
|
||||
.measures(getMeasures(d.getMeasures())).dimensions(getDimensions(d.getDimensions())).build();
|
||||
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
|
||||
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
|
||||
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
|
||||
}
|
||||
return datasource;
|
||||
}
|
||||
|
||||
private static String getDataSourceAggTime(List<Dimension> dimensions) {
|
||||
Optional<Dimension> timeDimension = dimensions.stream()
|
||||
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).findFirst();
|
||||
if (timeDimension.isPresent() && Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
|
||||
return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
|
||||
}
|
||||
return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE;
|
||||
}
|
||||
|
||||
private static List<Metric> getMetricsByMetricYamlTpl(List<MetricYamlTpl> metricYamlTpls) {
|
||||
List<Metric> metrics = new ArrayList<>();
|
||||
for (MetricYamlTpl metricYamlTpl : metricYamlTpls) {
|
||||
Metric metric = new Metric();
|
||||
metric.setMetricTypeParams(getMetricTypeParams(metricYamlTpl.getTypeParams()));
|
||||
metric.setOwners(metricYamlTpl.getOwners());
|
||||
metric.setType(metricYamlTpl.getType());
|
||||
metric.setName(metricYamlTpl.getName());
|
||||
metrics.add(metric);
|
||||
}
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static MetricTypeParams getMetricTypeParams(MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
|
||||
MetricTypeParams metricTypeParams = new MetricTypeParams();
|
||||
metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
|
||||
metricTypeParams.setMeasures(getMeasures(metricTypeParamsYamlTpl.getMeasures()));
|
||||
return metricTypeParams;
|
||||
}
|
||||
|
||||
private static List<Measure> getMeasures(List<MeasureYamlTpl> measureYamlTpls) {
|
||||
List<Measure> measures = new ArrayList<>();
|
||||
for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) {
|
||||
Measure measure = new Measure();
|
||||
measure.setCreateMetric(measureYamlTpl.getCreateMetric());
|
||||
measure.setExpr(measureYamlTpl.getExpr());
|
||||
measure.setAgg(measureYamlTpl.getAgg());
|
||||
measure.setName(measureYamlTpl.getName());
|
||||
measure.setAlias(measureYamlTpl.getAlias());
|
||||
measure.setConstraint(measureYamlTpl.getConstraint());
|
||||
measures.add(measure);
|
||||
}
|
||||
return measures;
|
||||
}
|
||||
|
||||
private static List<Dimension> getDimension(List<DimensionYamlTpl> dimensionYamlTpls) {
|
||||
List<Dimension> dimensions = new ArrayList<>();
|
||||
for (DimensionYamlTpl dimensionYamlTpl : dimensionYamlTpls) {
|
||||
Dimension dimension = Dimension.builder().build();
|
||||
dimension.setType(dimensionYamlTpl.getType());
|
||||
dimension.setExpr(dimensionYamlTpl.getExpr());
|
||||
dimension.setName(dimensionYamlTpl.getName());
|
||||
dimension.setOwners(dimensionYamlTpl.getOwners());
|
||||
if (Objects.nonNull(dimensionYamlTpl.getDataType())) {
|
||||
dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType()));
|
||||
}
|
||||
if (Objects.isNull(dimension.getDataType())) {
|
||||
dimension.setDataType(DataType.UNKNOWN);
|
||||
}
|
||||
dimension.setDimensionTimeTypeParams(getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams()));
|
||||
dimensions.add(dimension);
|
||||
}
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
private static DimensionTimeTypeParams getDimensionTimeTypeParams(
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl) {
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||
if (dimensionTimeTypeParamsTpl != null) {
|
||||
dimensionTimeTypeParams.setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity());
|
||||
dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary());
|
||||
}
|
||||
return dimensionTimeTypeParams;
|
||||
}
|
||||
|
||||
private static List<Identify> getIdentify(List<IdentifyYamlTpl> identifyYamlTpls) {
|
||||
List<Identify> identifies = new ArrayList<>();
|
||||
for (IdentifyYamlTpl identifyYamlTpl : identifyYamlTpls) {
|
||||
Identify identify = new Identify();
|
||||
identify.setType(identifyYamlTpl.getType());
|
||||
identify.setName(identifyYamlTpl.getName());
|
||||
identifies.add(identify);
|
||||
}
|
||||
return identifies;
|
||||
}
|
||||
|
||||
private static List<JoinRelation> getJoinRelation(List<ModelRela> modelRelas, Map<Long, String> modelIdName) {
|
||||
List<JoinRelation> joinRelations = new ArrayList<>();
|
||||
modelRelas.stream().forEach(r -> {
|
||||
if (modelIdName.containsKey(r.getFromModelId()) && modelIdName.containsKey(r.getToModelId())) {
|
||||
JoinRelation joinRelation = JoinRelation.builder().left(modelIdName.get(r.getFromModelId()))
|
||||
.right(modelIdName.get(r.getToModelId())).joinType(r.getJoinType()).build();
|
||||
List<Triple<String, String, String>> conditions = new ArrayList<>();
|
||||
r.getJoinConditions().stream().forEach(rr -> {
|
||||
if (FilterOperatorEnum.isValueCompare(rr.getOperator())) {
|
||||
conditions.add(Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField()));
|
||||
}
|
||||
});
|
||||
joinRelation.setJoinCondition(conditions);
|
||||
joinRelations.add(joinRelation);
|
||||
}
|
||||
});
|
||||
return joinRelations;
|
||||
}
|
||||
|
||||
public static void update(HeadlessSchema schema, List<Metric> metric) throws Exception {
|
||||
if (schema != null) {
|
||||
updateMetric(metric, schema.getMetrics());
|
||||
}
|
||||
}
|
||||
|
||||
public static void update(HeadlessSchema schema, DataSource datasourceYamlTpl) throws Exception {
|
||||
if (schema != null) {
|
||||
String dataSourceName = datasourceYamlTpl.getName();
|
||||
Optional<Entry<String, DataSource>> datasourceYamlTplMap = schema.getDatasource().entrySet().stream()
|
||||
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
|
||||
if (datasourceYamlTplMap.isPresent()) {
|
||||
datasourceYamlTplMap.get().setValue(datasourceYamlTpl);
|
||||
} else {
|
||||
schema.getDatasource().put(dataSourceName, datasourceYamlTpl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void update(HeadlessSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
|
||||
throws Exception {
|
||||
if (schema != null) {
|
||||
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema.getDimension().entrySet()
|
||||
.stream().filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
|
||||
if (datasourceYamlTplMap.isPresent()) {
|
||||
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
|
||||
} else {
|
||||
List<Dimension> dimensions = new ArrayList<>();
|
||||
updateDimension(dimensionYamlTpls, dimensions);
|
||||
schema.getDimension().put(datasourceBizName, dimensions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void updateDimension(List<Dimension> dimensionYamlTpls, List<Dimension> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensionYamlTpls)) {
|
||||
return;
|
||||
}
|
||||
Set<String> toAdd = dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
|
||||
Iterator<Dimension> iterator = dimensions.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Dimension cur = iterator.next();
|
||||
if (toAdd.contains(cur.getName())) {
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
dimensions.addAll(dimensionYamlTpls);
|
||||
}
|
||||
|
||||
private static void updateMetric(List<Metric> metricYamlTpls, List<Metric> metrics) {
|
||||
if (CollectionUtils.isEmpty(metricYamlTpls)) {
|
||||
return;
|
||||
}
|
||||
Set<String> toAdd = metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
|
||||
Iterator<Metric> iterator = metrics.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Metric cur = iterator.next();
|
||||
if (toAdd.contains(cur.getName())) {
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
metrics.addAll(metricYamlTpls);
|
||||
}
|
||||
|
||||
public static String formatKey(String key) {
|
||||
key = key.trim();
|
||||
if (key.startsWith("/")) {
|
||||
key = key.substring(1);
|
||||
}
|
||||
if (key.endsWith("/")) {
|
||||
key = key.substring(0, key.length() - 1);
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
@Configuration
|
||||
@EnableCaching
|
||||
public class GuavaCacheConfig {
|
||||
|
||||
@Value("${parser.cache.saveMinute:1}")
|
||||
private Integer saveMinutes = 1;
|
||||
@Value("${parser.cache.maximumSize:1000}")
|
||||
private Integer maximumSize = 1000;
|
||||
|
||||
@Bean
|
||||
public LoadingCache<String, HeadlessModel> getCache() {
|
||||
LoadingCache<String, HeadlessModel> cache
|
||||
= CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(saveMinutes, TimeUnit.MINUTES)
|
||||
.initialCapacity(10)
|
||||
.maximumSize(maximumSize).build(
|
||||
new CacheLoader<String, HeadlessModel>() {
|
||||
@Override
|
||||
public HeadlessModel load(String key) {
|
||||
log.info("load SemanticSchema [{}]", key);
|
||||
return HeadlessSchemaManager.this.reload(key);
|
||||
}
|
||||
}
|
||||
);
|
||||
return cache;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -4,27 +4,20 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner;
|
||||
import com.tencent.supersonic.headless.api.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.Renderer;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.TableView;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.DataSourceNode;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSqlDialect;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.optimizer.FilterToGroupScanRule;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.FilterRender;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.OutputRender;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.render.SourceRender;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.ListIterator;
|
||||
import java.util.Objects;
|
||||
import java.util.Stack;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.calcite.plan.RelOptPlanner;
|
||||
import org.apache.calcite.plan.hep.HepPlanner;
|
||||
@@ -36,6 +29,13 @@ import org.apache.calcite.sql.validate.SqlValidator;
|
||||
import org.apache.calcite.sql.validate.SqlValidatorScope;
|
||||
import org.apache.calcite.sql2rel.SqlToRelConverter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.ListIterator;
|
||||
import java.util.Objects;
|
||||
import java.util.Stack;
|
||||
|
||||
@Slf4j
|
||||
public class AggPlanner implements Planner {
|
||||
|
||||
|
||||
@@ -1,338 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.parser.calcite.planner;
|
||||
|
||||
import com.tencent.supersonic.common.util.calcite.SqlParseUtils;
|
||||
import com.tencent.supersonic.headless.api.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.Configuration;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MaterializationElement;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.TimeRange;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.calcite.adapter.enumerable.EnumerableRules;
|
||||
import org.apache.calcite.config.CalciteConnectionConfigImpl;
|
||||
import org.apache.calcite.jdbc.CalciteSchema;
|
||||
import org.apache.calcite.plan.ConventionTraitDef;
|
||||
import org.apache.calcite.plan.RelOptCluster;
|
||||
import org.apache.calcite.plan.RelOptMaterialization;
|
||||
import org.apache.calcite.plan.RelOptPlanner;
|
||||
import org.apache.calcite.plan.RelOptTable;
|
||||
import org.apache.calcite.plan.hep.HepPlanner;
|
||||
import org.apache.calcite.plan.hep.HepProgramBuilder;
|
||||
import org.apache.calcite.prepare.CalciteCatalogReader;
|
||||
import org.apache.calcite.rel.RelDistributionTraitDef;
|
||||
import org.apache.calcite.rel.RelHomogeneousShuttle;
|
||||
import org.apache.calcite.rel.RelNode;
|
||||
import org.apache.calcite.rel.RelShuttle;
|
||||
import org.apache.calcite.rel.core.RelFactories;
|
||||
import org.apache.calcite.rel.core.TableScan;
|
||||
import org.apache.calcite.rel.rules.materialize.MaterializedViewRules;
|
||||
import org.apache.calcite.rex.RexBuilder;
|
||||
import org.apache.calcite.rex.RexNode;
|
||||
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
|
||||
import org.apache.calcite.sql.parser.SqlParseException;
|
||||
import org.apache.calcite.tools.RelBuilder;
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public class MaterializationPlanner implements Planner {
|
||||
|
||||
protected HeadlessSchema schema;
|
||||
protected CalciteSchema viewSchema;
|
||||
protected HepProgramBuilder hepProgramBuilder;
|
||||
protected RelOptPlanner relOptPlanner;
|
||||
protected RelBuilder relBuilder;
|
||||
protected CalciteCatalogReader calciteCatalogReader;
|
||||
|
||||
protected Comparator materializationSort = new Comparator<Entry<Long, Set<String>>>() {
|
||||
@Override
|
||||
public int compare(Entry<Long, Set<String>> o1, Entry<Long, Set<String>> o2) {
|
||||
if (o1.getValue().size() == o2.getValue().size()) {
|
||||
Optional<Materialization> o1Lever = schema.getMaterializationList().stream()
|
||||
.filter(m -> m.getMaterializationId().equals(o1.getKey())).findFirst();
|
||||
Optional<Materialization> o2Lever = schema.getMaterializationList().stream()
|
||||
.filter(m -> m.getMaterializationId().equals(o2.getKey())).findFirst();
|
||||
if (o1Lever.isPresent() && o2Lever.isPresent()) {
|
||||
return o2Lever.get().getLevel() - o1Lever.get().getLevel();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
return o2.getValue().size() - o1.getValue().size();
|
||||
}
|
||||
};
|
||||
|
||||
public MaterializationPlanner(HeadlessSchema schema) {
|
||||
this.schema = schema;
|
||||
init();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void explain(QueryStatement queryStatement, AggOption isAgg) throws Exception {
|
||||
// findMatchMaterialization
|
||||
// checkValid field + time
|
||||
if (CollectionUtils.isEmpty(queryStatement.getTimeRanges())) {
|
||||
//has no matchMaterialization time info
|
||||
return;
|
||||
}
|
||||
Set<String> fields = new HashSet<>();
|
||||
MetricQueryReq metricCommand = queryStatement.getMetricReq();
|
||||
if (!Objects.isNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) {
|
||||
fields.addAll(SqlParseUtils.getFilterField(metricCommand.getWhere()));
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(metricCommand.getMetrics())) {
|
||||
fields.addAll(metricCommand.getMetrics());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(metricCommand.getDimensions())) {
|
||||
fields.addAll(metricCommand.getDimensions());
|
||||
}
|
||||
Map<Long, Set<String>> matchMaterialization = new HashMap<>();
|
||||
Map<Long, Long> materializationDataBase = schema.getMaterializationList().stream()
|
||||
.collect(Collectors.toMap(Materialization::getMaterializationId, Materialization::getDataBase));
|
||||
for (String elem : fields) {
|
||||
boolean checkOk = false;
|
||||
for (Materialization materialization : schema.getMaterializationList()) {
|
||||
if (check(metricCommand, materialization, elem, queryStatement.getTimeRanges())) {
|
||||
if (!matchMaterialization.containsKey(materialization.getMaterializationId())) {
|
||||
matchMaterialization.put(materialization.getMaterializationId(), new HashSet<>());
|
||||
}
|
||||
matchMaterialization.get(materialization.getMaterializationId()).add(elem);
|
||||
checkOk = true;
|
||||
}
|
||||
}
|
||||
if (!checkOk) {
|
||||
log.info("check fail [{}]", elem);
|
||||
}
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(matchMaterialization)) {
|
||||
List<Entry<Long, Set<String>>> sortedMaterialization = new ArrayList<>(matchMaterialization.entrySet());
|
||||
sortedMaterialization.stream().collect(Collectors.toList()).sort(materializationSort);
|
||||
for (Entry<Long, Set<String>> m : sortedMaterialization) {
|
||||
Optional<Materialization> materialization = schema.getMaterializationList().stream()
|
||||
.filter(mz -> mz.getMaterializationId().equals(m.getKey())).findFirst();
|
||||
if (!materialization.isPresent()) {
|
||||
continue;
|
||||
}
|
||||
Set<String> viewField = new HashSet<>(m.getValue());
|
||||
viewField.add(materialization.get().getEntities());
|
||||
viewField.add(materialization.get().getDateInfo());
|
||||
if (materialization.get().getTimePartType().equals(TimePartType.ZIPPER)) {
|
||||
viewField.add(Constants.MATERIALIZATION_ZIPPER_START + materialization.get().getDateInfo());
|
||||
viewField.add(Constants.MATERIALIZATION_ZIPPER_END + materialization.get().getDateInfo());
|
||||
}
|
||||
if (viewField.containsAll(fields)) {
|
||||
addDataSource(materialization.get());
|
||||
break;
|
||||
}
|
||||
List<Entry<Long, Set<String>>> linkMaterialization = new ArrayList<>();
|
||||
for (Entry<Long, Set<String>> mm : sortedMaterialization) {
|
||||
if (mm.getKey().equals(m.getKey())) {
|
||||
continue;
|
||||
}
|
||||
if (materializationDataBase.get(mm.getKey()).equals(materializationDataBase.get(m.getKey()))) {
|
||||
linkMaterialization.add(mm);
|
||||
}
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(linkMaterialization)) {
|
||||
linkMaterialization.sort(materializationSort);
|
||||
for (Entry<Long, Set<String>> mm : linkMaterialization) {
|
||||
Set<String> linkField = new HashSet<>(mm.getValue());
|
||||
linkField.addAll(viewField);
|
||||
if (linkField.containsAll(fields)) {
|
||||
Optional<Materialization> linkMaterial = schema.getMaterializationList().stream()
|
||||
.filter(mz -> mz.getMaterializationId().equals(mm.getKey())).findFirst();
|
||||
if (linkMaterial.isPresent()) {
|
||||
addDataSource(materialization.get());
|
||||
addDataSource(linkMaterial.get());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addDataSource(Materialization materialization) {
|
||||
Identify identify = new Identify();
|
||||
identify.setName(materialization.getEntities());
|
||||
List<Measure> metrics = materialization.getMetrics().stream()
|
||||
.map(m -> Measure.builder().name(m.getName()).expr(m.getName()).build()).collect(
|
||||
Collectors.toList());
|
||||
List<Dimension> dimensions = materialization.getDimensions().stream()
|
||||
.map(d -> Dimension.builder().name(d.getName()).expr(d.getName()).build()).collect(
|
||||
Collectors.toList());
|
||||
if (materialization.getTimePartType().equals(TimePartType.ZIPPER)) {
|
||||
dimensions.add(
|
||||
Dimension.builder().name(Constants.MATERIALIZATION_ZIPPER_START + materialization.getDateInfo())
|
||||
.type(Constants.DIMENSION_TYPE_TIME)
|
||||
.expr(Constants.MATERIALIZATION_ZIPPER_START + materialization.getDateInfo()).build());
|
||||
dimensions.add(
|
||||
Dimension.builder().name(Constants.MATERIALIZATION_ZIPPER_END + materialization.getDateInfo())
|
||||
.type(Constants.DIMENSION_TYPE_TIME)
|
||||
.expr(Constants.MATERIALIZATION_ZIPPER_END + materialization.getDateInfo()).build());
|
||||
} else {
|
||||
dimensions.add(Dimension.builder().name(materialization.getDateInfo()).expr(materialization.getDateInfo())
|
||||
.type(Constants.DIMENSION_TYPE_TIME)
|
||||
.build());
|
||||
}
|
||||
|
||||
DataSource dataSource = DataSource.builder().sourceId(materialization.getDataBase())
|
||||
.tableQuery(materialization.getDestinationTable())
|
||||
.timePartType(materialization.getTimePartType())
|
||||
.name("v_" + String.valueOf(materialization.getMaterializationId()))
|
||||
.identifiers(Arrays.asList(identify))
|
||||
.measures(metrics)
|
||||
.dimensions(dimensions)
|
||||
.build();
|
||||
schema.getDatasource().put(dataSource.getName(), dataSource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSql() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSourceId() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HeadlessSchema findBest() {
|
||||
return schema;
|
||||
}
|
||||
|
||||
private void init() {
|
||||
viewSchema = SchemaBuilder.getMaterializationSchema();
|
||||
hepProgramBuilder = new HepProgramBuilder();
|
||||
hepProgramBuilder.addRuleInstance(MaterializedViewRules.PROJECT_FILTER);
|
||||
relOptPlanner = new HepPlanner(hepProgramBuilder.build());
|
||||
calciteCatalogReader = new CalciteCatalogReader(
|
||||
CalciteSchema.from(viewSchema.plus()),
|
||||
CalciteSchema.from(viewSchema.plus()).path(null),
|
||||
Configuration.typeFactory,
|
||||
new CalciteConnectionConfigImpl(new Properties()));
|
||||
|
||||
relOptPlanner.addRelTraitDef(ConventionTraitDef.INSTANCE);
|
||||
relOptPlanner.addRelTraitDef(RelDistributionTraitDef.INSTANCE);
|
||||
EnumerableRules.rules().forEach(relOptPlanner::addRule);
|
||||
|
||||
RexBuilder rexBuilder = new RexBuilder(Configuration.typeFactory);
|
||||
RelOptCluster relOptCluster = RelOptCluster.create(relOptPlanner, rexBuilder);
|
||||
relBuilder = RelFactories.LOGICAL_BUILDER.create(relOptCluster, calciteCatalogReader);
|
||||
}
|
||||
|
||||
private RexNode getRexNode(List<ImmutablePair<String, String>> timeRanges, String viewField) {
|
||||
RexNode rexNode = null;
|
||||
for (ImmutablePair<String, String> timeRange : timeRanges) {
|
||||
if (rexNode == null) {
|
||||
rexNode = getRexNodeByTimeRange(TimeRange.builder().start(timeRange.left).end(timeRange.right).build(),
|
||||
viewField);
|
||||
continue;
|
||||
}
|
||||
rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode,
|
||||
getRexNodeByTimeRange(TimeRange.builder().start(timeRange.left).end(timeRange.right).build(),
|
||||
viewField));
|
||||
}
|
||||
return rexNode;
|
||||
}
|
||||
|
||||
private RexNode getRexNode(Materialization materialization, String elem, String viewField) {
|
||||
Optional<MaterializationElement> dim = materialization.getDimensions()
|
||||
.stream().filter(d -> d.getName().equalsIgnoreCase(elem)).findFirst();
|
||||
if (!dim.isPresent()) {
|
||||
dim = materialization.getMetrics().stream().filter(m -> m.getName().equalsIgnoreCase(elem)).findFirst();
|
||||
}
|
||||
RexNode rexNode = null;
|
||||
if (dim.isPresent()) {
|
||||
for (TimeRange timeRange : dim.get().getTimeRangeList()) {
|
||||
if (rexNode == null) {
|
||||
rexNode = getRexNodeByTimeRange(timeRange, viewField);
|
||||
continue;
|
||||
}
|
||||
rexNode = relBuilder.call(SqlStdOperatorTable.OR, rexNode, getRexNodeByTimeRange(timeRange, viewField));
|
||||
}
|
||||
}
|
||||
return rexNode;
|
||||
}
|
||||
|
||||
private RexNode getRexNodeByTimeRange(TimeRange timeRange, String field) {
|
||||
return relBuilder.call(SqlStdOperatorTable.AND,
|
||||
relBuilder.call(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, relBuilder.field(field),
|
||||
relBuilder.literal(timeRange.getStart())),
|
||||
relBuilder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, relBuilder.field(field),
|
||||
relBuilder.literal(timeRange.getEnd())));
|
||||
}
|
||||
|
||||
public boolean check(MetricQueryReq metricCommand, Materialization materialization, String elem,
|
||||
List<ImmutablePair<String, String>> timeRanges)
|
||||
throws SqlParseException {
|
||||
boolean isMatch = false;
|
||||
try {
|
||||
relBuilder.clear();
|
||||
if (!CollectionUtils.isEmpty(relOptPlanner.getMaterializations())) {
|
||||
relOptPlanner.clear();
|
||||
}
|
||||
String db = SchemaBuilder.MATERIALIZATION_SYS_DB;
|
||||
RelBuilder viewBuilder = relBuilder.scan(Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_SOURCE));
|
||||
RexNode viewFilter = getRexNode(materialization, elem, SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE);
|
||||
if (viewFilter == null) {
|
||||
return false;
|
||||
}
|
||||
RelNode viewRel = viewBuilder.filter(viewFilter).project(relBuilder.fields()).build();
|
||||
log.debug("view {}", viewRel.explain());
|
||||
List<String> view = Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_VIEW);
|
||||
RelNode replacement = relBuilder.scan(view).build();
|
||||
RelOptMaterialization relOptMaterialization = new RelOptMaterialization(replacement, viewRel, null, view);
|
||||
relOptPlanner.addMaterialization(relOptMaterialization);
|
||||
|
||||
RelNode checkRel = relBuilder.scan(Arrays.asList(db, SchemaBuilder.MATERIALIZATION_SYS_SOURCE))
|
||||
.filter(getRexNode(timeRanges, SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE))
|
||||
.project(relBuilder.field(SchemaBuilder.MATERIALIZATION_SYS_FIELD_DATE)).build();
|
||||
|
||||
relOptPlanner.setRoot(checkRel);
|
||||
RelNode optRel = relOptPlanner.findBestExp();
|
||||
log.debug("findBestExp {}", optRel.explain());
|
||||
isMatch = !extractTableNames(optRel).contains(SchemaBuilder.MATERIALIZATION_SYS_SOURCE);
|
||||
} catch (Exception e) {
|
||||
log.error("check error {}", e);
|
||||
}
|
||||
return isMatch;
|
||||
}
|
||||
|
||||
public static Set<String> extractTableNames(RelNode relNode) {
|
||||
Set<String> tableNames = new HashSet<>();
|
||||
RelShuttle shuttle = new RelHomogeneousShuttle() {
|
||||
public RelNode visit(TableScan scan) {
|
||||
RelOptTable table = scan.getTable();
|
||||
tableNames.addAll(table.getQualifiedName());
|
||||
return scan;
|
||||
}
|
||||
};
|
||||
relNode.accept(shuttle);
|
||||
return tableNames;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.planner;
|
||||
|
||||
|
||||
import com.tencent.supersonic.headless.api.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
|
||||
public interface Planner {
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class DataSource {
|
||||
|
||||
private Long id;
|
||||
|
||||
private String name;
|
||||
|
||||
private Long sourceId;
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
package com.tencent.supersonic.headless.core.parser.calcite.s2sql;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.ItemDateResp;
|
||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.Data;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Data
|
||||
public class HeadlessModel {
|
||||
@@ -15,4 +20,15 @@ public class HeadlessModel {
|
||||
private Map<String, List<Dimension>> dimensionMap = new HashMap<>();
|
||||
private List<Materialization> materializationList = new ArrayList<>();
|
||||
private List<JoinRelation> joinRelations;
|
||||
private ItemDateResp dataDate;
|
||||
private DatabaseResp databaseResp;
|
||||
|
||||
public List<Dimension> getDimensions() {
|
||||
return dimensionMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public Map<Long, DataSource> getModelMap() {
|
||||
return datasourceMap.values().stream().collect(Collectors.toMap(DataSource::getId, dataSource -> dataSource));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.parser.convert;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Component("DefaultDimValueConverter")
|
||||
public class DefaultDimValueConverter implements HeadlessConverter {
|
||||
|
||||
@Override
|
||||
public boolean accept(QueryStatement queryStatement) {
|
||||
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void converter(Catalog catalog, QueryStatement queryStatement) {
|
||||
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
|
||||
MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds());
|
||||
List<DimensionResp> dimensionResps = catalog.getDimensions(metaFilter);
|
||||
//dimension which has default values
|
||||
dimensionResps = dimensionResps.stream()
|
||||
.filter(dimensionResp -> !CollectionUtils.isEmpty(dimensionResp.getDefaultValues()))
|
||||
.collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(dimensionResps)) {
|
||||
return;
|
||||
}
|
||||
log.info("dimension with default values:{}, queryStruct:{}", dimensionResps, queryStructCmd);
|
||||
//add dimension default value to filter
|
||||
List<String> dimensionFilterBizName = queryStructCmd.getDimensionFilters().stream()
|
||||
.map(Filter::getBizName).collect(Collectors.toList());
|
||||
for (DimensionResp dimensionResp : dimensionResps) {
|
||||
if (!dimensionFilterBizName.contains(dimensionResp.getBizName())) {
|
||||
Filter filter = new Filter();
|
||||
filter.setBizName(dimensionResp.getBizName());
|
||||
filter.setValue(dimensionResp.getDefaultValues());
|
||||
filter.setOperator(FilterOperatorEnum.IN);
|
||||
filter.setName(dimensionResp.getName());
|
||||
queryStructCmd.getDimensionFilters().add(filter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.parser.convert;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Component("MetricCheckConverter")
|
||||
@Slf4j
|
||||
public class MetricCheckConverter implements HeadlessConverter {
|
||||
|
||||
@Override
|
||||
public boolean accept(QueryStatement queryStatement) {
|
||||
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
|
||||
return false;
|
||||
}
|
||||
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
|
||||
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
|
||||
return false;
|
||||
}
|
||||
return !CollectionUtils.isEmpty(queryStructCmd.getAggregators());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void converter(Catalog catalog, QueryStatement queryStatement) {
|
||||
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
|
||||
MetaFilter metaFilter = new MetaFilter(queryStructReq.getModelIds());
|
||||
List<MetricResp> metricResps = catalog.getMetrics(metaFilter);
|
||||
List<DimensionResp> dimensionResps = catalog.getDimensions(metaFilter);
|
||||
Map<Long, DimensionResp> dimensionMap = dimensionResps.stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getId, d -> d));
|
||||
List<String> metricBizNames = queryStructReq.getMetrics();
|
||||
List<String> dimensionFilterBizNames = queryStructReq.getDimensionFilters().stream()
|
||||
.map(Filter::getBizName).collect(Collectors.toList());
|
||||
List<MetricResp> metricToQuery = metricResps.stream().filter(metricResp ->
|
||||
metricBizNames.contains(metricResp.getBizName())).collect(Collectors.toList());
|
||||
List<Long> dimensionToFilter = dimensionResps.stream().filter(dimensionResp ->
|
||||
dimensionFilterBizNames.contains(dimensionResp.getBizName()))
|
||||
.map(DimensionResp::getId).collect(Collectors.toList());
|
||||
for (MetricResp metricResp : metricToQuery) {
|
||||
Set<Long> necessaryDimensionIds = metricResp.getNecessaryDimensionIds();
|
||||
if (CollectionUtils.isEmpty(necessaryDimensionIds)) {
|
||||
continue;
|
||||
}
|
||||
DimensionResp dimensionResp = null;
|
||||
for (Long dimensionId : necessaryDimensionIds) {
|
||||
dimensionResp = dimensionMap.get(dimensionId);
|
||||
if (dimensionResp != null) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (dimensionResp == null) {
|
||||
continue;
|
||||
}
|
||||
String message = String.format("该指标必须配合维度[%s]来进行过滤查询", dimensionResp.getName());
|
||||
if (CollectionUtils.isEmpty(dimensionToFilter)) {
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
boolean flag = false;
|
||||
for (Long dimensionId : dimensionToFilter) {
|
||||
if (necessaryDimensionIds.contains(dimensionId)) {
|
||||
flag = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!flag) {
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.parser.convert;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.ColumnOrder;
|
||||
import com.tencent.supersonic.headless.api.pojo.Param;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component("ParserDefaultConverter")
|
||||
@Slf4j
|
||||
public class ParserDefaultConverter implements HeadlessConverter {
|
||||
|
||||
|
||||
private final CalculateAggConverter calculateCoverterAgg;
|
||||
private final QueryStructUtils queryStructUtils;
|
||||
|
||||
public ParserDefaultConverter(
|
||||
CalculateAggConverter calculateCoverterAgg,
|
||||
QueryStructUtils queryStructUtils) {
|
||||
this.calculateCoverterAgg = calculateCoverterAgg;
|
||||
this.queryStructUtils = queryStructUtils;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accept(QueryStatement queryStatement) {
|
||||
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
|
||||
return false;
|
||||
}
|
||||
return !calculateCoverterAgg.accept(queryStatement);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception {
|
||||
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
|
||||
MetricQueryReq metricCommand = queryStatement.getMetricReq();
|
||||
MetricQueryReq metricReq = generateSqlCommand(catalog, queryStructCmd);
|
||||
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructCmd));
|
||||
BeanUtils.copyProperties(metricReq, metricCommand);
|
||||
}
|
||||
|
||||
public MetricQueryReq generateSqlCommand(Catalog catalog, QueryStructReq queryStructCmd) {
|
||||
MetricQueryReq sqlCommend = new MetricQueryReq();
|
||||
sqlCommend.setMetrics(queryStructCmd.getMetrics());
|
||||
sqlCommend.setDimensions(queryStructCmd.getGroups());
|
||||
String where = queryStructUtils.generateWhere(queryStructCmd);
|
||||
log.info("in generateSqlCommend, complete where:{}", where);
|
||||
|
||||
sqlCommend.setWhere(where);
|
||||
sqlCommend.setOrder(queryStructCmd.getOrders().stream()
|
||||
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
|
||||
sqlCommend.setVariables(queryStructCmd.getParams().stream()
|
||||
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
|
||||
sqlCommend.setLimit(queryStructCmd.getLimit());
|
||||
String rootPath = catalog.getModelFullPath(queryStructCmd.getModelIds());
|
||||
sqlCommend.setRootPath(rootPath);
|
||||
|
||||
// todo tmp delete
|
||||
// support detail query
|
||||
if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
|
||||
for (Long modelId : queryStructCmd.getModelIds()) {
|
||||
String internalMetricName = queryStructUtils.generateInternalMetricName(
|
||||
modelId, queryStructCmd.getGroups());
|
||||
sqlCommend.getMetrics().add(internalMetricName);
|
||||
}
|
||||
}
|
||||
|
||||
return sqlCommend;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.parser.convert;
|
||||
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.headless.api.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricTable;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
|
||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
|
||||
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptor;
|
||||
import com.tencent.supersonic.headless.server.adaptor.db.DbAdaptorFactory;
|
||||
import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class QueryReqConverter {
|
||||
|
||||
@Autowired
|
||||
private ModelService domainService;
|
||||
@Autowired
|
||||
private HeadlessQueryEngine parserService;
|
||||
@Autowired
|
||||
private QueryStructUtils queryStructUtils;
|
||||
|
||||
@Autowired
|
||||
private Catalog catalog;
|
||||
|
||||
public QueryStatement convert(QueryS2SQLReq databaseReq, List<ModelSchemaResp> modelSchemaResps) throws Exception {
|
||||
|
||||
if (CollectionUtils.isEmpty(modelSchemaResps)) {
|
||||
return new QueryStatement();
|
||||
}
|
||||
//1.convert name to bizName
|
||||
convertNameToBizName(databaseReq, modelSchemaResps);
|
||||
//2.functionName corrector
|
||||
functionNameCorrector(databaseReq);
|
||||
//3.correct tableName
|
||||
correctTableName(databaseReq);
|
||||
|
||||
String tableName = SqlParserSelectHelper.getTableName(databaseReq.getSql());
|
||||
if (StringUtils.isEmpty(tableName)) {
|
||||
return new QueryStatement();
|
||||
}
|
||||
//4.build MetricTables
|
||||
List<String> allFields = SqlParserSelectHelper.getAllFields(databaseReq.getSql());
|
||||
List<String> metrics = getMetrics(modelSchemaResps, allFields);
|
||||
QueryStructReq queryStructCmd = new QueryStructReq();
|
||||
MetricTable metricTable = new MetricTable();
|
||||
metricTable.setMetrics(metrics);
|
||||
|
||||
Set<String> dimensions = getDimensions(modelSchemaResps, allFields);
|
||||
|
||||
metricTable.setDimensions(new ArrayList<>(dimensions));
|
||||
|
||||
metricTable.setAlias(tableName.toLowerCase());
|
||||
// if metric empty , fill model default
|
||||
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
|
||||
metricTable.setMetrics(new ArrayList<>());
|
||||
for (Long modelId : databaseReq.getModelIds()) {
|
||||
metricTable.getMetrics().add(queryStructUtils.generateInternalMetricName(modelId,
|
||||
metricTable.getDimensions()));
|
||||
}
|
||||
} else {
|
||||
queryStructCmd.setAggregators(
|
||||
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
|
||||
Collectors.toList()));
|
||||
}
|
||||
AggOption aggOption = getAggOption(databaseReq);
|
||||
metricTable.setAggOption(aggOption);
|
||||
List<MetricTable> tables = new ArrayList<>();
|
||||
tables.add(metricTable);
|
||||
//4.build ParseSqlReq
|
||||
ParseSqlReq result = new ParseSqlReq();
|
||||
BeanUtils.copyProperties(databaseReq, result);
|
||||
|
||||
result.setRootPath(catalog.getModelFullPath(databaseReq.getModelIds()));
|
||||
result.setTables(tables);
|
||||
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0));
|
||||
if (!queryStructUtils.isSupportWith(EngineTypeEnum.valueOf(database.getType().toUpperCase()),
|
||||
database.getVersion())) {
|
||||
result.setSupportWith(false);
|
||||
result.setWithAlias(false);
|
||||
}
|
||||
//5.physicalSql by ParseSqlReq
|
||||
queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql()));
|
||||
queryStructCmd.setModelIds(databaseReq.getModelIds().stream().collect(Collectors.toSet()));
|
||||
queryStructCmd.setQueryType(getQueryType(aggOption));
|
||||
log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd);
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq(queryStructCmd);
|
||||
queryStatement.setParseSqlReq(result);
|
||||
queryStatement.setIsS2SQL(true);
|
||||
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructCmd));
|
||||
queryStatement = parserService.plan(queryStatement);
|
||||
queryStatement.setSql(String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql()));
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
private AggOption getAggOption(QueryS2SQLReq databaseReq) {
|
||||
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
|
||||
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
|
||||
String sql = databaseReq.getSql();
|
||||
if (!SqlParserSelectHelper.hasGroupBy(sql)
|
||||
|| SqlParserSelectFunctionHelper.hasFunction(sql, "count")
|
||||
|| SqlParserSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
|
||||
return AggOption.NATIVE;
|
||||
}
|
||||
return AggOption.DEFAULT;
|
||||
}
|
||||
|
||||
private void convertNameToBizName(QueryS2SQLReq databaseReq, List<ModelSchemaResp> modelSchemaResps) {
|
||||
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResps);
|
||||
String sql = databaseReq.getSql();
|
||||
log.info("convert name to bizName before:{}", sql);
|
||||
String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
|
||||
log.info("convert name to bizName after:{}", replaceFields);
|
||||
databaseReq.setSql(replaceFields);
|
||||
}
|
||||
|
||||
private Set<String> getDimensions(List<ModelSchemaResp> modelSchemaResps, List<String> allFields) {
|
||||
Map<String, String> dimensionLowerToNameMap = modelSchemaResps.stream()
|
||||
.flatMap(modelSchemaResp -> modelSchemaResp.getDimensions().stream())
|
||||
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName,
|
||||
(k1, k2) -> k1));
|
||||
Map<String, String> internalLowerToNameMap = QueryStructUtils.internalCols.stream()
|
||||
.collect(Collectors.toMap(String::toLowerCase, a -> a));
|
||||
dimensionLowerToNameMap.putAll(internalLowerToNameMap);
|
||||
return allFields.stream()
|
||||
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
|
||||
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private List<String> getMetrics(List<ModelSchemaResp> modelSchemaResps, List<String> allFields) {
|
||||
Map<String, String> metricLowerToNameMap = modelSchemaResps.stream()
|
||||
.flatMap(modelSchemaResp -> modelSchemaResp.getMetrics().stream())
|
||||
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName));
|
||||
return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
|
||||
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private void functionNameCorrector(QueryS2SQLReq databaseReq) {
|
||||
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0));
|
||||
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
|
||||
return;
|
||||
}
|
||||
String type = database.getType();
|
||||
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase());
|
||||
log.info("type:{},engineAdaptor:{}", type, engineAdaptor);
|
||||
if (Objects.nonNull(engineAdaptor)) {
|
||||
String functionNameCorrector = engineAdaptor.functionNameCorrector(databaseReq.getSql());
|
||||
log.info("sql:{} ,after corrector", databaseReq.getSql(), functionNameCorrector);
|
||||
databaseReq.setSql(functionNameCorrector);
|
||||
}
|
||||
}
|
||||
|
||||
protected Map<String, String> getFieldNameToBizNameMap(List<ModelSchemaResp> modelSchemaResps) {
|
||||
// support fieldName and field alias to bizName
|
||||
Map<String, String> dimensionResults = modelSchemaResps.stream().flatMap(modelSchemaResp
|
||||
-> modelSchemaResp.getDimensions().stream())
|
||||
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
|
||||
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
|
||||
|
||||
Map<String, String> metricResults = modelSchemaResps.stream().flatMap(modelSchemaResp
|
||||
-> modelSchemaResp.getMetrics().stream())
|
||||
.flatMap(entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
|
||||
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
|
||||
|
||||
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
|
||||
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
|
||||
dimensionResults.putAll(metricResults);
|
||||
return dimensionResults;
|
||||
}
|
||||
|
||||
private Stream<Pair<String, String>> getPairStream(String aliasStr, String name, String bizName) {
|
||||
Set<Pair<String, String>> elements = new HashSet<>();
|
||||
elements.add(Pair.of(name, bizName));
|
||||
if (StringUtils.isNotBlank(aliasStr)) {
|
||||
List<String> aliasList = SchemaItem.getAliasList(aliasStr);
|
||||
for (String alias : aliasList) {
|
||||
elements.add(Pair.of(alias, bizName));
|
||||
}
|
||||
}
|
||||
return elements.stream();
|
||||
}
|
||||
|
||||
public void correctTableName(QueryS2SQLReq databaseReq) {
|
||||
String sql = databaseReq.getSql();
|
||||
for (Long modelId : databaseReq.getModelIds()) {
|
||||
sql = SqlParserReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + modelId);
|
||||
}
|
||||
databaseReq.setSql(sql);
|
||||
}
|
||||
|
||||
private QueryType getQueryType(AggOption aggOption) {
|
||||
boolean isAgg = AggOption.isAgg(aggOption);
|
||||
QueryType queryType = QueryType.TAG;
|
||||
if (isAgg) {
|
||||
queryType = QueryType.METRIC;
|
||||
}
|
||||
return queryType;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.parser.convert;
|
||||
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* correct the Query parameters when the model source type is zipper
|
||||
*/
|
||||
@Component("ZipperModelConverter")
|
||||
@Slf4j
|
||||
public class ZipperModelConverter implements HeadlessConverter {
|
||||
|
||||
private final QueryStructUtils queryStructUtils;
|
||||
private final Catalog catalog;
|
||||
|
||||
public ZipperModelConverter(QueryStructUtils queryStructUtils,
|
||||
Catalog catalog) {
|
||||
this.queryStructUtils = queryStructUtils;
|
||||
this.catalog = catalog;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accept(QueryStatement queryStatement) {
|
||||
if (Objects.isNull(queryStatement.getQueryStructReq())) {
|
||||
return false;
|
||||
}
|
||||
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
|
||||
List<ModelResp> modelRespList = catalog.getModelList(queryStructCmd.getModelIds());
|
||||
if (!CollectionUtils.isEmpty(modelRespList)) {
|
||||
// all data sources are zipper tables
|
||||
long zipperCnt = modelRespList.stream().filter(m -> ModelSourceType.isZipper(m.getSourceType()))
|
||||
.count();
|
||||
return modelRespList.size() == zipperCnt;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception {
|
||||
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
|
||||
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq();
|
||||
MetricQueryReq metricCommand = queryStatement.getMetricReq();
|
||||
doSingleZipperSource(queryStructCmd, sqlCommend, metricCommand);
|
||||
}
|
||||
|
||||
protected void doSingleZipperSource(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend,
|
||||
MetricQueryReq metricCommand) {
|
||||
// all data sources are zipper tables
|
||||
// request time field rewrite to start_ end_
|
||||
if (!sqlCommend.getSql().isEmpty()) {
|
||||
String sqlNew = queryStructUtils.generateZipperWhere(queryStructCmd, sqlCommend);
|
||||
log.info("doSingleZipperSource before[{}] after[{}]", sqlCommend.getSql(), sqlNew);
|
||||
sqlCommend.setSql(sqlNew);
|
||||
} else {
|
||||
String where = queryStructUtils.generateZipperWhere(queryStructCmd);
|
||||
if (!where.isEmpty() && Objects.nonNull(metricCommand)) {
|
||||
log.info("doSingleZipperSource before[{}] after[{}]", metricCommand.getWhere(), where);
|
||||
metricCommand.setWhere(where);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
package com.tencent.supersonic.headless.core.parser.convert;
|
||||
|
||||
package com.tencent.supersonic.headless.core.parser.converter;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
@@ -7,20 +6,15 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateModeUtils;
|
||||
import com.tencent.supersonic.headless.api.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.api.enums.EngineType;
|
||||
import com.tencent.supersonic.headless.api.pojo.MetricTable;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryStructUtils;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
|
||||
import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@@ -35,24 +29,11 @@ import java.util.stream.Collectors;
|
||||
@Slf4j
|
||||
public class CalculateAggConverter implements HeadlessConverter {
|
||||
|
||||
|
||||
private final HeadlessQueryEngine parserService;
|
||||
private final QueryStructUtils queryStructUtils;
|
||||
private final SqlGenerateUtils sqlGenerateUtils;
|
||||
private final Catalog catalog;
|
||||
|
||||
@Value("${metricParser.agg.default:sum}")
|
||||
private String metricAggDefault;
|
||||
|
||||
|
||||
public CalculateAggConverter(
|
||||
HeadlessQueryEngine parserService,
|
||||
@Lazy QueryStructUtils queryStructUtils,
|
||||
SqlGenerateUtils sqlGenerateUtils, Catalog catalog) {
|
||||
this.parserService = parserService;
|
||||
this.queryStructUtils = queryStructUtils;
|
||||
public CalculateAggConverter(SqlGenerateUtils sqlGenerateUtils) {
|
||||
this.sqlGenerateUtils = sqlGenerateUtils;
|
||||
this.catalog = catalog;
|
||||
}
|
||||
|
||||
public interface EngineSql {
|
||||
@@ -60,34 +41,36 @@ public class CalculateAggConverter implements HeadlessConverter {
|
||||
String sql(QueryStructReq queryStructCmd, boolean isOver, boolean asWith, String metricSql);
|
||||
}
|
||||
|
||||
public ParseSqlReq generateSqlCommend(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum, String version)
|
||||
public ParseSqlReq generateSqlCommend(QueryStatement queryStatement, EngineType engineTypeEnum, String version)
|
||||
throws Exception {
|
||||
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
|
||||
// 同环比
|
||||
if (isRatioAccept(queryStructCmd)) {
|
||||
return generateRatioSqlCommand(queryStructCmd, engineTypeEnum, version);
|
||||
if (isRatioAccept(queryStructReq)) {
|
||||
return generateRatioSqlCommand(queryStatement, engineTypeEnum, version);
|
||||
}
|
||||
ParseSqlReq sqlCommand = new ParseSqlReq();
|
||||
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds()));
|
||||
sqlCommand.setRootPath(queryStructReq.getModelIdStr());
|
||||
String metricTableName = "v_metric_tb_tmp";
|
||||
MetricTable metricTable = new MetricTable();
|
||||
metricTable.setAlias(metricTableName);
|
||||
metricTable.setMetrics(queryStructCmd.getMetrics());
|
||||
metricTable.setDimensions(queryStructCmd.getGroups());
|
||||
String where = queryStructUtils.generateWhere(queryStructCmd);
|
||||
metricTable.setMetrics(queryStructReq.getMetrics());
|
||||
metricTable.setDimensions(queryStructReq.getGroups());
|
||||
String where = sqlGenerateUtils.generateWhere(queryStructReq,
|
||||
queryStatement.getHeadlessModel().getDataDate());
|
||||
log.info("in generateSqlCommand, complete where:{}", where);
|
||||
metricTable.setWhere(where);
|
||||
metricTable.setAggOption(AggOption.AGGREGATION);
|
||||
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
|
||||
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
|
||||
String sql = String.format("select %s from %s %s %s %s", sqlGenerateUtils.getSelect(queryStructReq),
|
||||
metricTableName,
|
||||
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
|
||||
sqlGenerateUtils.getLimit(queryStructCmd));
|
||||
if (!queryStructUtils.isSupportWith(engineTypeEnum, version)) {
|
||||
sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq),
|
||||
sqlGenerateUtils.getLimit(queryStructReq));
|
||||
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
|
||||
sqlCommand.setSupportWith(false);
|
||||
sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructCmd),
|
||||
sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(queryStructReq),
|
||||
metricTableName,
|
||||
sqlGenerateUtils.getGroupBy(queryStructCmd), sqlGenerateUtils.getOrderBy(queryStructCmd),
|
||||
sqlGenerateUtils.getLimit(queryStructCmd));
|
||||
sqlGenerateUtils.getGroupBy(queryStructReq), sqlGenerateUtils.getOrderBy(queryStructReq),
|
||||
sqlGenerateUtils.getLimit(queryStructReq));
|
||||
}
|
||||
sqlCommand.setSql(sql);
|
||||
return sqlCommand;
|
||||
@@ -122,12 +105,11 @@ public class CalculateAggConverter implements HeadlessConverter {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void converter(Catalog catalog, QueryStatement queryStatement) throws Exception {
|
||||
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
|
||||
public void convert(QueryStatement queryStatement) throws Exception {
|
||||
ParseSqlReq sqlCommend = queryStatement.getParseSqlReq();
|
||||
DatabaseResp databaseResp = catalog.getDatabaseByModelId(queryStructCmd.getModelIds().get(0));
|
||||
ParseSqlReq parseSqlReq = generateSqlCommend(queryStructCmd,
|
||||
EngineTypeEnum.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
|
||||
DatabaseResp databaseResp = queryStatement.getHeadlessModel().getDatabaseResp();
|
||||
ParseSqlReq parseSqlReq = generateSqlCommend(queryStatement,
|
||||
EngineType.valueOf(databaseResp.getType().toUpperCase()), databaseResp.getVersion());
|
||||
sqlCommend.setSql(parseSqlReq.getSql());
|
||||
sqlCommend.setTables(parseSqlReq.getTables());
|
||||
sqlCommend.setRootPath(parseSqlReq.getRootPath());
|
||||
@@ -149,38 +131,39 @@ public class CalculateAggConverter implements HeadlessConverter {
|
||||
return false;
|
||||
}
|
||||
|
||||
public ParseSqlReq generateRatioSqlCommand(QueryStructReq queryStructCmd, EngineTypeEnum engineTypeEnum,
|
||||
public ParseSqlReq generateRatioSqlCommand(QueryStatement queryStatement, EngineType engineTypeEnum,
|
||||
String version)
|
||||
throws Exception {
|
||||
check(queryStructCmd);
|
||||
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
|
||||
check(queryStructReq);
|
||||
ParseSqlReq sqlCommand = new ParseSqlReq();
|
||||
sqlCommand.setRootPath(catalog.getModelFullPath(queryStructCmd.getModelIds()));
|
||||
sqlCommand.setRootPath(queryStructReq.getModelIdStr());
|
||||
String metricTableName = "v_metric_tb_tmp";
|
||||
MetricTable metricTable = new MetricTable();
|
||||
metricTable.setAlias(metricTableName);
|
||||
metricTable.setMetrics(queryStructCmd.getMetrics());
|
||||
metricTable.setDimensions(queryStructCmd.getGroups());
|
||||
String where = queryStructUtils.generateWhere(queryStructCmd);
|
||||
metricTable.setMetrics(queryStructReq.getMetrics());
|
||||
metricTable.setDimensions(queryStructReq.getGroups());
|
||||
String where = sqlGenerateUtils.generateWhere(queryStructReq, queryStatement.getHeadlessModel().getDataDate());
|
||||
log.info("in generateSqlCommend, complete where:{}", where);
|
||||
metricTable.setWhere(where);
|
||||
metricTable.setAggOption(AggOption.AGGREGATION);
|
||||
sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable)));
|
||||
boolean isOver = isOverRatio(queryStructCmd);
|
||||
boolean isOver = isOverRatio(queryStructReq);
|
||||
String sql = "";
|
||||
switch (engineTypeEnum) {
|
||||
case H2:
|
||||
sql = new H2EngineSql().sql(queryStructCmd, isOver, true, metricTableName);
|
||||
sql = new H2EngineSql().sql(queryStructReq, isOver, true, metricTableName);
|
||||
break;
|
||||
case MYSQL:
|
||||
case DORIS:
|
||||
case CLICKHOUSE:
|
||||
if (!queryStructUtils.isSupportWith(engineTypeEnum, version)) {
|
||||
if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) {
|
||||
sqlCommand.setSupportWith(false);
|
||||
}
|
||||
if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) {
|
||||
sql = new MysqlEngineSql().sql(queryStructCmd, isOver, sqlCommand.isSupportWith(), metricTableName);
|
||||
sql = new MysqlEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName);
|
||||
} else {
|
||||
sql = new CkEngineSql().sql(queryStructCmd, isOver, sqlCommand.isSupportWith(), metricTableName);
|
||||
sql = new CkEngineSql().sql(queryStructReq, isOver, sqlCommand.isSupportWith(), metricTableName);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
@@ -0,0 +1,82 @@
|
||||
package com.tencent.supersonic.headless.core.parser.converter;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.ColumnOrder;
|
||||
import com.tencent.supersonic.headless.api.pojo.Param;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component("ParserDefaultConverter")
|
||||
@Slf4j
|
||||
public class ParserDefaultConverter implements HeadlessConverter {
|
||||
|
||||
private final SqlGenerateUtils sqlGenerateUtils;
|
||||
|
||||
private final CalculateAggConverter calculateConverterAgg;
|
||||
|
||||
public ParserDefaultConverter(CalculateAggConverter calculateConverterAgg,
|
||||
SqlGenerateUtils sqlGenerateUtils) {
|
||||
this.calculateConverterAgg = calculateConverterAgg;
|
||||
this.sqlGenerateUtils = sqlGenerateUtils;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accept(QueryStatement queryStatement) {
|
||||
if (Objects.isNull(queryStatement.getQueryStructReq()) || queryStatement.getIsS2SQL()) {
|
||||
return false;
|
||||
}
|
||||
return !calculateConverterAgg.accept(queryStatement);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void convert(QueryStatement queryStatement) throws Exception {
|
||||
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
|
||||
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
|
||||
MetricQueryReq metricReq = generateSqlCommand(queryStructReq, queryStatement);
|
||||
queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryStructReq,
|
||||
queryStatement.getHeadlessModel().getDataDate()));
|
||||
BeanUtils.copyProperties(metricReq, metricQueryReq);
|
||||
}
|
||||
|
||||
public MetricQueryReq generateSqlCommand(QueryStructReq queryStructReq, QueryStatement queryStatement) {
|
||||
MetricQueryReq metricQueryReq = new MetricQueryReq();
|
||||
metricQueryReq.setMetrics(queryStructReq.getMetrics());
|
||||
metricQueryReq.setDimensions(queryStructReq.getGroups());
|
||||
String where = sqlGenerateUtils.generateWhere(queryStructReq,
|
||||
queryStatement.getHeadlessModel().getDataDate());
|
||||
log.info("in generateSqlCommend, complete where:{}", where);
|
||||
|
||||
metricQueryReq.setWhere(where);
|
||||
metricQueryReq.setOrder(queryStructReq.getOrders().stream()
|
||||
.map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
|
||||
metricQueryReq.setVariables(queryStructReq.getParams().stream()
|
||||
.collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
|
||||
metricQueryReq.setLimit(queryStructReq.getLimit());
|
||||
String rootPath = queryStructReq.getModelIdStr();
|
||||
metricQueryReq.setRootPath(rootPath);
|
||||
|
||||
// support detail query
|
||||
if (queryStructReq.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(metricQueryReq.getMetrics())) {
|
||||
Map<Long, DataSource> dataSourceMap = queryStatement.getHeadlessModel().getModelMap();
|
||||
for (Long modelId : queryStructReq.getModelIds()) {
|
||||
String modelBizName = dataSourceMap.get(modelId).getName();
|
||||
String internalMetricName = sqlGenerateUtils.generateInternalMetricName(modelBizName);
|
||||
metricQueryReq.getMetrics().add(internalMetricName);
|
||||
}
|
||||
}
|
||||
|
||||
return metricQueryReq;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.persistence.mapper;
|
||||
|
||||
|
||||
import com.tencent.supersonic.headless.api.pojo.QueryStat;
|
||||
import com.tencent.supersonic.headless.api.request.ItemUseReq;
|
||||
import java.util.List;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface StatMapper {
|
||||
|
||||
Boolean createRecord(QueryStat queryStatInfo);
|
||||
|
||||
List<QueryStat> getStatInfo(ItemUseReq itemUseCommend);
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.persistence.pojo;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class DataDownload {
|
||||
|
||||
List<List<String>> headers;
|
||||
|
||||
List<List<String>> data;
|
||||
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.persistence.repository;
|
||||
|
||||
import com.tencent.supersonic.headless.api.pojo.QueryStat;
|
||||
import com.tencent.supersonic.headless.api.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.response.ItemUseResp;
|
||||
import java.util.List;
|
||||
|
||||
public interface StatRepository {
|
||||
|
||||
Boolean createRecord(QueryStat queryStatInfo);
|
||||
|
||||
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
|
||||
|
||||
List<QueryStat> getQueryStatInfoWithoutCache(ItemUseReq itemUseCommend);
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.persistence.repository;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.headless.api.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.QueryStat;
|
||||
import com.tencent.supersonic.headless.core.persistence.mapper.StatMapper;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
|
||||
|
||||
@Slf4j
|
||||
@Repository
|
||||
public class StatRepositoryImpl implements StatRepository {
|
||||
|
||||
private final StatMapper statMapper;
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
public StatRepositoryImpl(StatMapper statMapper) {
|
||||
this.statMapper = statMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean createRecord(QueryStat queryStatInfo) {
|
||||
return statMapper.createRecord(queryStatInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
|
||||
List<ItemUseResp> result = new ArrayList<>();
|
||||
List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
|
||||
Map<String, Long> map = new ConcurrentHashMap<>();
|
||||
statInfos.stream().forEach(stat -> {
|
||||
String dimensions = stat.getDimensions();
|
||||
String metrics = stat.getMetrics();
|
||||
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.getName(), stat.getModelId());
|
||||
updateStatMapInfo(map, metrics, TypeEnums.METRIC.getName(), stat.getModelId());
|
||||
});
|
||||
|
||||
map.forEach((k, v) -> {
|
||||
Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]);
|
||||
String type = k.split(AT_SYMBOL + AT_SYMBOL)[1];
|
||||
String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2];
|
||||
result.add(new ItemUseResp(classId, type, nameEn, v));
|
||||
});
|
||||
|
||||
return result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<QueryStat> getQueryStatInfoWithoutCache(ItemUseReq itemUseCommend) {
|
||||
return statMapper.getStatInfo(itemUseCommend);
|
||||
}
|
||||
|
||||
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type, Long modelId) {
|
||||
if (Strings.isNotEmpty(dimensions)) {
|
||||
try {
|
||||
List<String> dimensionList = mapper.readValue(dimensions, new TypeReference<List<String>>() {
|
||||
});
|
||||
dimensionList.stream().forEach(dimension -> {
|
||||
String key = modelId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension;
|
||||
if (map.containsKey(key)) {
|
||||
map.put(key, map.get(key) + 1);
|
||||
} else {
|
||||
map.put(key, 1L);
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.warn("e:{}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void updateStatMapInfo(Map<String, Long> map, Long modelId, String type) {
|
||||
if (Objects.nonNull(modelId)) {
|
||||
String key = type + AT_SYMBOL + AT_SYMBOL + modelId;
|
||||
if (map.containsKey(key)) {
|
||||
map.put(key, map.get(key) + 1);
|
||||
} else {
|
||||
map.put(key, 1L);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.tencent.supersonic.headless.core.pojo;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class ConnectInfo {
|
||||
|
||||
|
||||
private String url;
|
||||
|
||||
private String userName;
|
||||
|
||||
private String password;
|
||||
|
||||
private String database;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.tencent.supersonic.headless.core.pojo;
|
||||
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.RecordInfo;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class Database extends RecordInfo {
|
||||
|
||||
|
||||
private Long id;
|
||||
|
||||
private Long domainId;
|
||||
|
||||
private String name;
|
||||
|
||||
private String description;
|
||||
|
||||
private String version;
|
||||
|
||||
/**
|
||||
* mysql,clickhouse
|
||||
*/
|
||||
private String type;
|
||||
|
||||
private ConnectInfo connectInfo;
|
||||
|
||||
private List<String> admins = Lists.newArrayList();
|
||||
|
||||
private List<String> viewers = Lists.newArrayList();
|
||||
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package com.tencent.supersonic.headless.core.pojo;
|
||||
|
||||
import com.alibaba.druid.filter.Filter;
|
||||
import com.alibaba.druid.pool.DruidDataSource;
|
||||
import com.alibaba.druid.wall.WallConfig;
|
||||
import com.alibaba.druid.wall.WallFilter;
|
||||
import com.tencent.supersonic.headless.api.enums.DataType;
|
||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.DependsOn;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.STATISTIC;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class JdbcDataSource {
|
||||
|
||||
private static final Object lockLock = new Object();
|
||||
private static volatile Map<String, DruidDataSource> dataSourceMap = new ConcurrentHashMap<>();
|
||||
private static volatile Map<String, Lock> dataSourceLockMap = new ConcurrentHashMap<>();
|
||||
@Value("${source.lock-time:30}")
|
||||
@Getter
|
||||
protected Long lockTime;
|
||||
|
||||
@Value("${source.max-active:2}")
|
||||
@Getter
|
||||
protected int maxActive;
|
||||
|
||||
@Value("${source.initial-size:0}")
|
||||
@Getter
|
||||
protected int initialSize;
|
||||
|
||||
@Value("${source.min-idle:1}")
|
||||
@Getter
|
||||
protected int minIdle;
|
||||
|
||||
@Value("${source.max-wait:60000}")
|
||||
@Getter
|
||||
protected long maxWait;
|
||||
|
||||
@Value("${source.time-between-eviction-runs-millis:2000}")
|
||||
@Getter
|
||||
protected long timeBetweenEvictionRunsMillis;
|
||||
|
||||
@Value("${source.min-evictable-idle-time-millis:600000}")
|
||||
@Getter
|
||||
protected long minEvictableIdleTimeMillis;
|
||||
|
||||
@Value("${source.max-evictable-idle-time-millis:900000}")
|
||||
@Getter
|
||||
protected long maxEvictableIdleTimeMillis;
|
||||
|
||||
@Value("${source.time-between-connect-error-millis:60000}")
|
||||
@Getter
|
||||
protected long timeBetweenConnectErrorMillis;
|
||||
|
||||
@Value("${source.test-while-idle:true}")
|
||||
@Getter
|
||||
protected boolean testWhileIdle;
|
||||
|
||||
@Value("${source.test-on-borrow:false}")
|
||||
@Getter
|
||||
protected boolean testOnBorrow;
|
||||
|
||||
@Value("${source.test-on-return:false}")
|
||||
@Getter
|
||||
protected boolean testOnReturn;
|
||||
|
||||
@Value("${source.break-after-acquire-failure:true}")
|
||||
@Getter
|
||||
protected boolean breakAfterAcquireFailure;
|
||||
|
||||
@Value("${source.connection-error-retry-attempts:1}")
|
||||
@Getter
|
||||
protected int connectionErrorRetryAttempts;
|
||||
|
||||
@Value("${source.keep-alive:false}")
|
||||
@Getter
|
||||
protected boolean keepAlive;
|
||||
|
||||
@Value("${source.validation-query-timeout:5}")
|
||||
@Getter
|
||||
protected int validationQueryTimeout;
|
||||
|
||||
@Value("${source.validation-query:'select 1'}")
|
||||
@Getter
|
||||
protected String validationQuery;
|
||||
|
||||
@Value("${source.filters:'stat'}")
|
||||
@Getter
|
||||
protected String filters;
|
||||
@Autowired
|
||||
WallFilter wallFilter;
|
||||
|
||||
@Bean(name = "wallConfig")
|
||||
WallConfig wallConfig() {
|
||||
WallConfig config = new WallConfig();
|
||||
config.setDeleteAllow(false);
|
||||
config.setUpdateAllow(false);
|
||||
config.setInsertAllow(false);
|
||||
config.setReplaceAllow(false);
|
||||
config.setMergeAllow(false);
|
||||
config.setTruncateAllow(false);
|
||||
config.setCreateTableAllow(false);
|
||||
config.setAlterTableAllow(false);
|
||||
config.setDropTableAllow(false);
|
||||
config.setCommentAllow(true);
|
||||
config.setUseAllow(false);
|
||||
config.setDescribeAllow(false);
|
||||
config.setShowAllow(false);
|
||||
config.setSelectWhereAlwayTrueCheck(false);
|
||||
config.setSelectHavingAlwayTrueCheck(false);
|
||||
config.setSelectUnionCheck(false);
|
||||
config.setConditionDoubleConstAllow(true);
|
||||
config.setConditionAndAlwayTrueAllow(true);
|
||||
config.setConditionAndAlwayFalseAllow(true);
|
||||
return config;
|
||||
}
|
||||
|
||||
@Bean(name = "wallFilter")
|
||||
@DependsOn("wallConfig")
|
||||
WallFilter wallFilter(WallConfig wallConfig) {
|
||||
WallFilter wfilter = new WallFilter();
|
||||
wfilter.setConfig(wallConfig);
|
||||
return wfilter;
|
||||
}
|
||||
|
||||
private Lock getDataSourceLock(String key) {
|
||||
if (dataSourceLockMap.containsKey(key)) {
|
||||
return dataSourceLockMap.get(key);
|
||||
}
|
||||
|
||||
synchronized (lockLock) {
|
||||
if (dataSourceLockMap.containsKey(key)) {
|
||||
return dataSourceLockMap.get(key);
|
||||
}
|
||||
Lock lock = new ReentrantLock();
|
||||
dataSourceLockMap.put(key, lock);
|
||||
return lock;
|
||||
}
|
||||
}
|
||||
|
||||
public void removeDatasource(DatabaseResp jdbcSourceInfo) {
|
||||
|
||||
String key = getDataSourceKey(jdbcSourceInfo);
|
||||
|
||||
Lock lock = getDataSourceLock(key);
|
||||
|
||||
if (!lock.tryLock()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
DruidDataSource druidDataSource = dataSourceMap.remove(key);
|
||||
if (druidDataSource != null) {
|
||||
druidDataSource.close();
|
||||
}
|
||||
|
||||
dataSourceLockMap.remove(key);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public DruidDataSource getDataSource(DatabaseResp jdbcSourceInfo) throws RuntimeException {
|
||||
|
||||
String name = jdbcSourceInfo.getName();
|
||||
String type = jdbcSourceInfo.getType();
|
||||
String jdbcUrl = jdbcSourceInfo.getUrl();
|
||||
String username = jdbcSourceInfo.getUsername();
|
||||
String password = jdbcSourceInfo.getPassword();
|
||||
|
||||
String key = getDataSourceKey(jdbcSourceInfo);
|
||||
|
||||
DruidDataSource druidDataSource = dataSourceMap.get(key);
|
||||
if (druidDataSource != null && !druidDataSource.isClosed()) {
|
||||
return druidDataSource;
|
||||
}
|
||||
|
||||
Lock lock = getDataSourceLock(key);
|
||||
|
||||
try {
|
||||
if (!lock.tryLock(lockTime, TimeUnit.SECONDS)) {
|
||||
druidDataSource = dataSourceMap.get(key);
|
||||
if (druidDataSource != null && !druidDataSource.isClosed()) {
|
||||
return druidDataSource;
|
||||
}
|
||||
throw new RuntimeException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
|
||||
}
|
||||
|
||||
druidDataSource = dataSourceMap.get(key);
|
||||
if (druidDataSource != null && !druidDataSource.isClosed()) {
|
||||
lock.unlock();
|
||||
return druidDataSource;
|
||||
}
|
||||
|
||||
druidDataSource = new DruidDataSource();
|
||||
|
||||
try {
|
||||
String className = JdbcDataSourceUtils.getDriverClassName(jdbcUrl);
|
||||
try {
|
||||
Class.forName(className);
|
||||
} catch (ClassNotFoundException e) {
|
||||
throw new RuntimeException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
|
||||
}
|
||||
|
||||
druidDataSource.setDriverClassName(className);
|
||||
|
||||
druidDataSource.setName(name);
|
||||
druidDataSource.setUrl(jdbcUrl);
|
||||
druidDataSource.setUsername(username);
|
||||
|
||||
if (!jdbcUrl.toLowerCase().contains(DataType.PRESTO.getFeature())) {
|
||||
druidDataSource.setPassword(password);
|
||||
}
|
||||
|
||||
druidDataSource.setInitialSize(initialSize);
|
||||
druidDataSource.setMinIdle(minIdle);
|
||||
druidDataSource.setMaxActive(maxActive);
|
||||
druidDataSource.setMaxWait(maxWait);
|
||||
druidDataSource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
|
||||
druidDataSource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
|
||||
druidDataSource.setMaxEvictableIdleTimeMillis(maxEvictableIdleTimeMillis);
|
||||
druidDataSource.setTimeBetweenConnectErrorMillis(timeBetweenConnectErrorMillis);
|
||||
druidDataSource.setTestWhileIdle(testWhileIdle);
|
||||
druidDataSource.setTestOnBorrow(testOnBorrow);
|
||||
druidDataSource.setTestOnReturn(testOnReturn);
|
||||
druidDataSource.setConnectionErrorRetryAttempts(connectionErrorRetryAttempts);
|
||||
druidDataSource.setBreakAfterAcquireFailure(breakAfterAcquireFailure);
|
||||
druidDataSource.setKeepAlive(keepAlive);
|
||||
druidDataSource.setValidationQueryTimeout(validationQueryTimeout);
|
||||
druidDataSource.setRemoveAbandoned(true);
|
||||
druidDataSource.setRemoveAbandonedTimeout(3600 + 5 * 60);
|
||||
druidDataSource.setLogAbandoned(true);
|
||||
|
||||
// default validation query
|
||||
String driverName = druidDataSource.getDriverClassName();
|
||||
if (driverName.indexOf("sqlserver") != -1 || driverName.indexOf("mysql") != -1
|
||||
|| driverName.indexOf("h2") != -1 || driverName.indexOf("moonbox") != -1) {
|
||||
druidDataSource.setValidationQuery("select 1");
|
||||
}
|
||||
|
||||
if (driverName.indexOf("oracle") != -1) {
|
||||
druidDataSource.setValidationQuery("select 1 from dual");
|
||||
}
|
||||
|
||||
if (driverName.indexOf("elasticsearch") != -1) {
|
||||
druidDataSource.setValidationQuery(null);
|
||||
}
|
||||
|
||||
// druid wall filter not support some database so set type mysql
|
||||
if (DataType.MOONBOX == DataType.urlOf(jdbcUrl)
|
||||
|| DataType.MONGODB == DataType.urlOf(jdbcUrl)
|
||||
|| DataType.ELASTICSEARCH == DataType.urlOf(jdbcUrl)
|
||||
|| DataType.CASSANDRA == DataType.urlOf(jdbcUrl)
|
||||
|| DataType.VERTICA == DataType.urlOf(jdbcUrl)
|
||||
|| DataType.KYLIN == DataType.urlOf(jdbcUrl)
|
||||
|| DataType.HANA == DataType.urlOf(jdbcUrl)
|
||||
|| DataType.IMPALA == DataType.urlOf(jdbcUrl)
|
||||
|| DataType.TDENGINE == DataType.urlOf(jdbcUrl)) {
|
||||
wallFilter.setDbType(DataType.MYSQL.getFeature());
|
||||
}
|
||||
|
||||
Properties properties = new Properties();
|
||||
if (driverName.indexOf("mysql") != -1) {
|
||||
properties.setProperty("druid.mysql.usePingMethod", "false");
|
||||
}
|
||||
|
||||
druidDataSource.setConnectProperties(properties);
|
||||
|
||||
try {
|
||||
// statistic and ck source don't need wall filter
|
||||
if (!STATISTIC.equals(name) && !DataType.CLICKHOUSE.getFeature().equalsIgnoreCase(type)) {
|
||||
druidDataSource.setProxyFilters(Arrays.asList(new Filter[]{wallFilter}));
|
||||
}
|
||||
druidDataSource.setFilters(filters);
|
||||
druidDataSource.init();
|
||||
} catch (Exception e) {
|
||||
log.error("Exception during pool initialization", e);
|
||||
throw new RuntimeException(e.getMessage());
|
||||
}
|
||||
|
||||
dataSourceMap.put(key, druidDataSource);
|
||||
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
|
||||
return druidDataSource;
|
||||
}
|
||||
|
||||
private String getDataSourceKey(DatabaseResp jdbcSourceInfo) {
|
||||
return JdbcDataSourceUtils.getKey(jdbcSourceInfo.getName(),
|
||||
jdbcSourceInfo.getUrl(),
|
||||
jdbcSourceInfo.getUsername(),
|
||||
jdbcSourceInfo.getPassword(), "", false);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.headless.core.persistence.pojo;
|
||||
package com.tencent.supersonic.headless.core.pojo;
|
||||
|
||||
public class ParserSvrResponse<T> {
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
package com.tencent.supersonic.headless.core.persistence.pojo;
|
||||
package com.tencent.supersonic.headless.core.pojo;
|
||||
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
|
||||
import lombok.Data;
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.apache.commons.lang3.tuple.Triple;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.commons.lang3.tuple.Triple;
|
||||
|
||||
@Data
|
||||
public class QueryStatement {
|
||||
@@ -26,6 +27,8 @@ public class QueryStatement {
|
||||
private Boolean enableOptimize = true;
|
||||
private Triple<String, String, String> minMaxTime;
|
||||
|
||||
private HeadlessModel headlessModel;
|
||||
|
||||
public boolean isOk() {
|
||||
this.ok = "".equals(errMsg) && !"".equals(sql);
|
||||
return ok;
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||
|
||||
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@Data
|
||||
public class DataModelYamlTpl {
|
||||
|
||||
private Long id;
|
||||
|
||||
private String name;
|
||||
|
||||
private Long sourceId;
|
||||
|
||||
private String sqlQuery;
|
||||
|
||||
private String tableQuery;
|
||||
|
||||
private List<IdentifyYamlTpl> identifiers;
|
||||
|
||||
private List<DimensionYamlTpl> dimensions;
|
||||
|
||||
private List<MeasureYamlTpl> measures;
|
||||
|
||||
private ModelSourceType modelSourceTypeEnum;
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
|
||||
@Data
|
||||
public class DimensionTimeTypeParamsTpl {
|
||||
|
||||
private String isPrimary;
|
||||
|
||||
private String timeGranularity;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
|
||||
import lombok.Data;
|
||||
|
||||
|
||||
@Data
|
||||
public class DimensionYamlTpl {
|
||||
|
||||
private String name;
|
||||
|
||||
private String owners;
|
||||
|
||||
private String type;
|
||||
|
||||
private String expr;
|
||||
|
||||
private DimensionTimeTypeParamsTpl typeParams;
|
||||
|
||||
private DataTypeEnums dataType;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class IdentifyYamlTpl {
|
||||
|
||||
private String name;
|
||||
|
||||
/**
|
||||
* 主键 primary 外键 foreign
|
||||
*/
|
||||
private String type;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class MeasureYamlTpl {
|
||||
|
||||
private String name;
|
||||
|
||||
private String agg;
|
||||
|
||||
private String expr;
|
||||
|
||||
private String constraint;
|
||||
|
||||
private String alias;
|
||||
|
||||
private String createMetric;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class MetricTypeParamsYamlTpl {
|
||||
|
||||
private List<MeasureYamlTpl> measures;
|
||||
|
||||
private String expr;
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
||||
@Data
|
||||
public class MetricYamlTpl {
|
||||
|
||||
private String name;
|
||||
|
||||
private List<String> owners;
|
||||
|
||||
private String type;
|
||||
|
||||
private MetricTypeParamsYamlTpl typeParams;
|
||||
|
||||
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.rest;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
|
||||
import com.tencent.supersonic.headless.api.request.DownloadStructReq;
|
||||
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryItemReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
|
||||
import com.tencent.supersonic.headless.api.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.enums.QueryType;
|
||||
import com.tencent.supersonic.headless.api.response.ExplainResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.SqlParserResp;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.service.DownloadService;
|
||||
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
|
||||
import com.tencent.supersonic.headless.core.service.QueryService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/semantic/query")
|
||||
@Slf4j
|
||||
public class QueryController {
|
||||
|
||||
@Autowired
|
||||
private QueryService queryService;
|
||||
@Autowired
|
||||
private HeadlessQueryEngine headlessQueryEngine;
|
||||
|
||||
@Autowired
|
||||
private DownloadService downloadService;
|
||||
|
||||
@PostMapping("/sql")
|
||||
public Object queryBySql(@RequestBody QueryS2SQLReq queryS2SQLReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
Object queryBySql = queryService.queryBySql(queryS2SQLReq, user);
|
||||
log.info("queryBySql:{}", queryBySql);
|
||||
return queryBySql;
|
||||
}
|
||||
|
||||
@PostMapping("/struct")
|
||||
public Object queryByStruct(@RequestBody QueryStructReq queryStructReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryService.queryByStructWithAuth(queryStructReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/metricDataQueryById")
|
||||
public ItemQueryResultResp metricDataQueryById(@RequestBody QueryItemReq queryApiReq,
|
||||
HttpServletRequest request) throws Exception {
|
||||
return queryService.metricDataQueryById(queryApiReq, request);
|
||||
}
|
||||
|
||||
@PostMapping("/download/struct")
|
||||
public void downloadByStruct(@RequestBody DownloadStructReq downloadStructReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
downloadService.downloadByStruct(downloadStructReq, user, response);
|
||||
}
|
||||
|
||||
@PostMapping("/download/batch")
|
||||
public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
downloadService.batchDownload(batchDownloadReq, user, response);
|
||||
}
|
||||
|
||||
@PostMapping("/queryStatement")
|
||||
public Object queryStatement(@RequestBody QueryStatement queryStatement) throws Exception {
|
||||
return queryService.queryByQueryStatement(queryStatement);
|
||||
}
|
||||
|
||||
@PostMapping("/struct/parse")
|
||||
public SqlParserResp parseByStruct(@RequestBody ParseSqlReq parseSqlReq) throws Exception {
|
||||
QueryStructReq queryStructCmd = new QueryStructReq();
|
||||
QueryStatement queryStatement = headlessQueryEngine.physicalSql(queryStructCmd, parseSqlReq);
|
||||
SqlParserResp sqlParserResp = new SqlParserResp();
|
||||
BeanUtils.copyProperties(queryStatement, sqlParserResp);
|
||||
return sqlParserResp;
|
||||
}
|
||||
|
||||
/**
|
||||
* queryByMultiStruct
|
||||
*/
|
||||
@PostMapping("/multiStruct")
|
||||
public Object queryByMultiStruct(@RequestBody QueryMultiStructReq queryMultiStructReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryService.queryByMultiStruct(queryMultiStructReq, user);
|
||||
}
|
||||
|
||||
/**
|
||||
* getStatInfo
|
||||
* query the used frequency of the metric/dimension
|
||||
*
|
||||
* @param itemUseReq
|
||||
*/
|
||||
@PostMapping("/stat")
|
||||
public List<ItemUseResp> getStatInfo(@RequestBody ItemUseReq itemUseReq) {
|
||||
return queryService.getStatInfo(itemUseReq);
|
||||
}
|
||||
|
||||
@PostMapping("/queryDimValue")
|
||||
public QueryResultWithSchemaResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryService.queryDimValue(queryDimValueReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/explain")
|
||||
public <T> ExplainResp explain(@RequestBody ExplainSqlReq<T> explainSqlReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
|
||||
User user = UserHolder.findUser(request, response);
|
||||
String queryReqJson = JsonUtil.toString(explainSqlReq.getQueryReq());
|
||||
QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum();
|
||||
|
||||
if (QueryType.SQL.equals(queryTypeEnum)) {
|
||||
QueryS2SQLReq queryS2SQLReq = JsonUtil.toObject(queryReqJson, QueryS2SQLReq.class);
|
||||
ExplainSqlReq<QueryS2SQLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2SQLReq>builder()
|
||||
.queryReq(queryS2SQLReq)
|
||||
.queryTypeEnum(queryTypeEnum).build();
|
||||
return queryService.explain(explainSqlReqNew, user);
|
||||
}
|
||||
if (QueryType.STRUCT.equals(queryTypeEnum)) {
|
||||
QueryStructReq queryStructReq = JsonUtil.toObject(queryReqJson, QueryStructReq.class);
|
||||
ExplainSqlReq<QueryStructReq> explainSqlReqNew = ExplainSqlReq.<QueryStructReq>builder()
|
||||
.queryReq(queryStructReq)
|
||||
.queryTypeEnum(queryTypeEnum).build();
|
||||
return queryService.explain(explainSqlReqNew, user);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.rest;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.headless.api.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.headless.api.request.PageMetricReq;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.DomainResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.service.SchemaService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/semantic/schema")
|
||||
public class SchemaController {
|
||||
|
||||
@Autowired
|
||||
private SchemaService schemaService;
|
||||
|
||||
@PostMapping
|
||||
public List<ModelSchemaResp> fetchModelSchema(@RequestBody ModelSchemaFilterReq filter,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return schemaService.fetchModelSchema(filter, user);
|
||||
}
|
||||
|
||||
@GetMapping("/domain/list")
|
||||
public List<DomainResp> getDomainList(HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return schemaService.getDomainList(user);
|
||||
}
|
||||
|
||||
@GetMapping("/model/list")
|
||||
public List<ModelResp> getModelList(@RequestParam("domainId") Long domainId,
|
||||
@RequestParam("authType") String authType,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return schemaService.getModelList(user, AuthType.valueOf(authType), domainId);
|
||||
}
|
||||
|
||||
@PostMapping("/dimension/page")
|
||||
public PageInfo<DimensionResp> queryDimension(@RequestBody PageDimensionReq pageDimensionCmd,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return schemaService.queryDimension(pageDimensionCmd, user);
|
||||
}
|
||||
|
||||
@PostMapping("/metric/page")
|
||||
public PageInfo<MetricResp> queryMetric(@RequestBody PageMetricReq pageMetricCmd,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return schemaService.queryMetric(pageMetricCmd, user);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.service;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
|
||||
import com.tencent.supersonic.headless.api.request.DownloadStructReq;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
public interface DownloadService {
|
||||
|
||||
void downloadByStruct(DownloadStructReq downloadStructReq,
|
||||
User user, HttpServletResponse response) throws Exception;
|
||||
|
||||
void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response) throws Exception;
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.service;
|
||||
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
||||
|
||||
public interface HeadlessQueryEngine {
|
||||
|
||||
QueryStatement plan(QueryStatement queryStatement) throws Exception;
|
||||
|
||||
QueryExecutor route(QueryStatement queryStatement);
|
||||
|
||||
QueryResultWithSchemaResp execute(QueryStatement queryStatement);
|
||||
|
||||
QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception;
|
||||
|
||||
QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq sqlCommend) throws Exception;
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.service;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.headless.api.request.QueryItemReq;
|
||||
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
|
||||
import com.tencent.supersonic.headless.api.response.ExplainResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.List;
|
||||
|
||||
public interface QueryService {
|
||||
|
||||
Object queryBySql(QueryS2SQLReq querySqlCmd, User user) throws Exception;
|
||||
|
||||
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;
|
||||
|
||||
QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user)
|
||||
throws Exception;
|
||||
|
||||
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception;
|
||||
|
||||
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
|
||||
|
||||
Object queryByQueryStatement(QueryStatement queryStatement);
|
||||
|
||||
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
|
||||
|
||||
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
|
||||
|
||||
@ApiHeaderCheck
|
||||
ItemQueryResultResp metricDataQueryById(QueryItemReq queryApiReq,
|
||||
HttpServletRequest request) throws Exception;
|
||||
|
||||
QueryStatement parseMetricReq(MetricQueryReq metricReq) throws Exception;
|
||||
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.service;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.headless.api.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.headless.api.request.PageMetricReq;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.response.DomainResp;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface SchemaService {
|
||||
|
||||
List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq filter, User user);
|
||||
|
||||
PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq, User user);
|
||||
|
||||
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
|
||||
|
||||
List<DomainResp> getDomainList(User user);
|
||||
|
||||
List<ModelResp> getModelList(User user, AuthType authType, Long domainId);
|
||||
}
|
||||
@@ -1,306 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.service.impl;
|
||||
|
||||
import com.alibaba.excel.EasyExcel;
|
||||
import com.alibaba.excel.ExcelWriter;
|
||||
import com.alibaba.excel.util.FileUtils;
|
||||
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.DataDownload;
|
||||
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
|
||||
import com.tencent.supersonic.headless.api.request.DownloadStructReq;
|
||||
import com.tencent.supersonic.headless.api.enums.SemanticType;
|
||||
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.service.QueryService;
|
||||
import com.tencent.supersonic.headless.core.utils.DataTransformUtils;
|
||||
import com.tencent.supersonic.headless.core.service.DownloadService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DownloadServiceImpl implements DownloadService {
|
||||
|
||||
private static final String internMetricCol = "指标名称";
|
||||
|
||||
private static final long downloadSize = 10000;
|
||||
|
||||
private ModelService modelService;
|
||||
|
||||
private QueryService queryService;
|
||||
|
||||
public DownloadServiceImpl(ModelService modelService, QueryService queryService) {
|
||||
this.modelService = modelService;
|
||||
this.queryService = queryService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void downloadByStruct(DownloadStructReq downloadStructReq,
|
||||
User user, HttpServletResponse response) throws Exception {
|
||||
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
|
||||
File file = FileUtils.createTmpFile(fileName);
|
||||
try {
|
||||
QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
|
||||
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
|
||||
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
|
||||
} catch (RuntimeException e) {
|
||||
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
|
||||
.doWrite(buildErrMessageData(e.getMessage()));
|
||||
return;
|
||||
}
|
||||
downloadFile(response, file, fileName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchDownload(BatchDownloadReq batchDownloadReq, User user,
|
||||
HttpServletResponse response) throws Exception {
|
||||
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
|
||||
File file = FileUtils.createTmpFile(fileName);
|
||||
List<Long> metricIds = batchDownloadReq.getMetricIds();
|
||||
if (CollectionUtils.isEmpty(metricIds)) {
|
||||
return;
|
||||
}
|
||||
batchDownload(batchDownloadReq, user, file);
|
||||
downloadFile(response, file, fileName);
|
||||
}
|
||||
|
||||
public void batchDownload(BatchDownloadReq batchDownloadReq, User user, File file) throws Exception {
|
||||
List<Long> metricIds = batchDownloadReq.getMetricIds();
|
||||
List<ModelSchemaResp> modelSchemaRespList = modelService.fetchModelSchema(new ModelSchemaFilterReq());
|
||||
Map<String, List<MetricSchemaResp>> metricSchemaMap = getMetricSchemaMap(modelSchemaRespList, metricIds);
|
||||
Map<Long, DimSchemaResp> dimensionRespMap = getDimensionMap(modelSchemaRespList);
|
||||
ExcelWriter excelWriter = EasyExcel.write(file).build();
|
||||
int sheetCount = 1;
|
||||
for (List<MetricSchemaResp> metrics : metricSchemaMap.values()) {
|
||||
if (CollectionUtils.isEmpty(metrics)) {
|
||||
continue;
|
||||
}
|
||||
MetricSchemaResp metricSchemaResp = metrics.get(0);
|
||||
List<DimSchemaResp> dimensions = getMetricRelaDimensions(metricSchemaResp, dimensionRespMap);
|
||||
for (MetricSchemaResp metric : metrics) {
|
||||
try {
|
||||
DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq);
|
||||
QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
|
||||
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
|
||||
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
|
||||
.head(dataDownload.getHeaders()).build();
|
||||
excelWriter.write(dataDownload.getData(), writeSheet);
|
||||
} catch (RuntimeException e) {
|
||||
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
|
||||
.doWrite(buildErrMessageData(e.getMessage()));
|
||||
return;
|
||||
}
|
||||
}
|
||||
sheetCount++;
|
||||
}
|
||||
excelWriter.finish();
|
||||
}
|
||||
|
||||
private List<List<String>> buildErrMessageHead() {
|
||||
List<List<String>> headers = Lists.newArrayList();
|
||||
headers.add(Lists.newArrayList("异常提示"));
|
||||
return headers;
|
||||
}
|
||||
|
||||
private List<List<String>> buildErrMessageData(String errMsg) {
|
||||
List<List<String>> data = Lists.newArrayList();
|
||||
data.add(Lists.newArrayList(errMsg));
|
||||
return data;
|
||||
}
|
||||
|
||||
private List<List<String>> buildHeader(QueryResultWithSchemaResp queryResultWithSchemaResp) {
|
||||
List<List<String>> header = Lists.newArrayList();
|
||||
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
|
||||
header.add(Lists.newArrayList(column.getName()));
|
||||
}
|
||||
return header;
|
||||
}
|
||||
|
||||
private List<List<String>> buildHeader(List<QueryColumn> queryColumns, List<String> dateList) {
|
||||
List<List<String>> headers = Lists.newArrayList();
|
||||
for (QueryColumn queryColumn : queryColumns) {
|
||||
if (SemanticType.DATE.name().equals(queryColumn.getShowType())) {
|
||||
continue;
|
||||
}
|
||||
headers.add(Lists.newArrayList(queryColumn.getName()));
|
||||
}
|
||||
for (String date : dateList) {
|
||||
headers.add(Lists.newArrayList(date));
|
||||
}
|
||||
headers.add(Lists.newArrayList(internMetricCol));
|
||||
return headers;
|
||||
}
|
||||
|
||||
private List<List<String>> buildData(QueryResultWithSchemaResp queryResultWithSchemaResp) {
|
||||
List<List<String>> data = new ArrayList<>();
|
||||
for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
|
||||
List<String> rowData = new ArrayList<>();
|
||||
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
|
||||
rowData.add(String.valueOf(row.get(column.getNameEn())));
|
||||
}
|
||||
data.add(rowData);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
private List<List<String>> buildData(List<List<String>> headers, Map<String, String> nameMap,
|
||||
List<Map<String, Object>> dataTransformed, String metricName) {
|
||||
List<List<String>> data = Lists.newArrayList();
|
||||
for (Map<String, Object> map : dataTransformed) {
|
||||
List<String> row = Lists.newArrayList();
|
||||
for (List<String> header : headers) {
|
||||
String head = header.get(0);
|
||||
if (internMetricCol.equals(head)) {
|
||||
continue;
|
||||
}
|
||||
Object object = map.getOrDefault(nameMap.getOrDefault(head, head), "");
|
||||
if (object == null) {
|
||||
row.add("");
|
||||
} else {
|
||||
row.add(String.valueOf(object));
|
||||
}
|
||||
}
|
||||
row.add(metricName);
|
||||
data.add(row);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
private DataDownload buildDataDownload(QueryResultWithSchemaResp queryResult, DownloadStructReq downloadStructReq) {
|
||||
List<QueryColumn> metricColumns = queryResult.getMetricColumns();
|
||||
List<QueryColumn> dimensionColumns = queryResult.getDimensionColumns();
|
||||
if (downloadStructReq.isTransform() && !CollectionUtils.isEmpty(metricColumns)) {
|
||||
QueryColumn metric = metricColumns.get(0);
|
||||
List<String> groups = downloadStructReq.getGroups();
|
||||
List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(),
|
||||
metric.getNameEn(), groups, downloadStructReq.getDateInfo());
|
||||
List<List<String>> headers = buildHeader(dimensionColumns, downloadStructReq.getDateInfo().getDateList());
|
||||
List<List<String>> data = buildData(headers, getDimensionNameMap(dimensionColumns),
|
||||
dataTransformed, metric.getName());
|
||||
return DataDownload.builder().headers(headers).data(data).build();
|
||||
} else {
|
||||
List<List<String>> data = buildData(queryResult);
|
||||
List<List<String>> header = buildHeader(queryResult);
|
||||
return DataDownload.builder().data(data).headers(header).build();
|
||||
}
|
||||
}
|
||||
|
||||
private DownloadStructReq buildDownloadStructReq(List<DimSchemaResp> dimensionResps, MetricResp metricResp,
|
||||
BatchDownloadReq batchDownloadReq) {
|
||||
DateConf dateConf = batchDownloadReq.getDateInfo();
|
||||
Set<Long> modelIds = dimensionResps.stream().map(DimSchemaResp::getModelId).collect(Collectors.toSet());
|
||||
modelIds.add(metricResp.getModelId());
|
||||
DownloadStructReq downloadStructReq = new DownloadStructReq();
|
||||
downloadStructReq.setGroups(dimensionResps.stream()
|
||||
.map(DimSchemaResp::getBizName).collect(Collectors.toList()));
|
||||
downloadStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricResp.getBizName());
|
||||
downloadStructReq.setAggregators(Lists.newArrayList(aggregator));
|
||||
downloadStructReq.setDateInfo(dateConf);
|
||||
downloadStructReq.setModelIds(modelIds);
|
||||
downloadStructReq.setLimit(downloadSize);
|
||||
downloadStructReq.setIsTransform(batchDownloadReq.isTransform());
|
||||
return downloadStructReq;
|
||||
}
|
||||
|
||||
private String getTimeDimension(DateConf dateConf) {
|
||||
if (Constants.MONTH.equals(dateConf.getPeriod())) {
|
||||
return TimeDimensionEnum.MONTH.getName();
|
||||
} else if (Constants.WEEK.equals(dateConf.getPeriod())) {
|
||||
return TimeDimensionEnum.WEEK.getName();
|
||||
} else {
|
||||
return TimeDimensionEnum.DAY.getName();
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, List<MetricSchemaResp>> getMetricSchemaMap(List<ModelSchemaResp> modelSchemaRespList,
|
||||
List<Long> metricIds) {
|
||||
return modelSchemaRespList.stream().flatMap(modelSchemaResp
|
||||
-> modelSchemaResp.getMetrics().stream())
|
||||
.filter(metricSchemaResp -> metricIds.contains(metricSchemaResp.getId()))
|
||||
.collect(Collectors.groupingBy(MetricSchemaResp::getRelaDimensionIdKey));
|
||||
}
|
||||
|
||||
private Map<Long, DimSchemaResp> getDimensionMap(List<ModelSchemaResp> modelSchemaRespList) {
|
||||
return modelSchemaRespList.stream().flatMap(modelSchemaResp
|
||||
-> modelSchemaResp.getDimensions().stream())
|
||||
.collect(Collectors.toMap(DimensionResp::getId, dimensionResp -> dimensionResp));
|
||||
}
|
||||
|
||||
private Map<String, String> getDimensionNameMap(List<QueryColumn> queryColumns) {
|
||||
return queryColumns.stream().collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn));
|
||||
}
|
||||
|
||||
private List<DimSchemaResp> getMetricRelaDimensions(MetricSchemaResp metricSchemaResp,
|
||||
Map<Long, DimSchemaResp> dimensionRespMap) {
|
||||
if (metricSchemaResp.getRelateDimension() == null
|
||||
|| CollectionUtils.isEmpty(metricSchemaResp.getRelateDimension().getDrillDownDimensions())) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
return metricSchemaResp.getRelateDimension().getDrillDownDimensions()
|
||||
.stream().map(drillDownDimension -> dimensionRespMap.get(drillDownDimension.getDimensionId()))
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private void downloadFile(HttpServletResponse response, File file, String filename) {
|
||||
try {
|
||||
byte[] buffer = readFileToByteArray(file);
|
||||
response.reset();
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
|
||||
response.addHeader("Content-Length", "" + file.length());
|
||||
try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
|
||||
response.setContentType("application/octet-stream");
|
||||
outputStream.write(buffer);
|
||||
outputStream.flush();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("failed to download file", e);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] readFileToByteArray(File file) throws IOException {
|
||||
try (InputStream fis = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
|
||||
byte[] buffer = new byte[fis.available()];
|
||||
fis.read(buffer);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.service.impl;
|
||||
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
||||
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
|
||||
import com.tencent.supersonic.headless.core.parser.QueryParser;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
|
||||
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryUtils;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
|
||||
|
||||
private final QueryParser queryParser;
|
||||
private final Catalog catalog;
|
||||
private final QueryUtils queryUtils;
|
||||
|
||||
public HeadlessQueryEngineImpl(QueryParser queryParser, Catalog catalog,
|
||||
QueryUtils queryUtils) {
|
||||
this.queryParser = queryParser;
|
||||
this.catalog = catalog;
|
||||
this.queryUtils = queryUtils;
|
||||
}
|
||||
|
||||
public QueryResultWithSchemaResp execute(QueryStatement queryStatement) {
|
||||
QueryResultWithSchemaResp queryResultWithColumns = null;
|
||||
QueryExecutor queryExecutor = route(queryStatement);
|
||||
if (queryExecutor != null) {
|
||||
queryResultWithColumns = queryExecutor.execute(catalog, queryStatement);
|
||||
queryResultWithColumns.setSql(queryStatement.getSql());
|
||||
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
|
||||
queryUtils.fillItemNameInfo(queryResultWithColumns, queryStatement.getModelIds());
|
||||
}
|
||||
}
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
public QueryStatement plan(QueryStatement queryStatement) throws Exception {
|
||||
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
|
||||
queryStatement = queryParser.logicSql(queryStatement);
|
||||
queryUtils.checkSqlParse(queryStatement);
|
||||
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
|
||||
log.info("queryStatement:{}", queryStatement);
|
||||
return optimize(queryStatement.getQueryStructReq(), queryStatement);
|
||||
}
|
||||
|
||||
public QueryStatement optimize(QueryStructReq queryStructCmd, QueryStatement queryStatement) {
|
||||
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
|
||||
queryOptimizer.rewrite(queryStructCmd, queryStatement);
|
||||
}
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
public QueryExecutor route(QueryStatement queryStatement) {
|
||||
for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
|
||||
if (queryExecutor.accept(queryStatement)) {
|
||||
return queryExecutor;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq(queryStructCmd);
|
||||
queryStatement.setParseSqlReq(sqlCommend);
|
||||
queryStatement.setIsS2SQL(true);
|
||||
return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement));
|
||||
}
|
||||
|
||||
public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq(queryStructCmd);
|
||||
queryStatement.setMetricReq(metricCommand);
|
||||
queryStatement.setIsS2SQL(false);
|
||||
return queryParser.parser(queryStatement);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,428 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.service.impl;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.enums.ApiItemType;
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.common.util.cache.CacheUtils;
|
||||
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
|
||||
import com.tencent.supersonic.headless.api.pojo.Cache;
|
||||
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryItemReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
|
||||
import com.tencent.supersonic.headless.api.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.enums.QueryType;
|
||||
import com.tencent.supersonic.headless.api.pojo.Dim;
|
||||
import com.tencent.supersonic.headless.api.pojo.Item;
|
||||
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.headless.api.response.AppDetailResp;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.ExplainResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.annotation.ApiHeaderCheck;
|
||||
import com.tencent.supersonic.headless.core.annotation.S2SQLDataPermission;
|
||||
import com.tencent.supersonic.headless.core.annotation.StructDataPermission;
|
||||
import com.tencent.supersonic.headless.core.aspect.ApiHeaderCheckAspect;
|
||||
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
||||
import com.tencent.supersonic.headless.core.parser.convert.QueryReqConverter;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.service.HeadlessQueryEngine;
|
||||
import com.tencent.supersonic.headless.core.service.QueryService;
|
||||
import com.tencent.supersonic.headless.core.service.SchemaService;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryUtils;
|
||||
import com.tencent.supersonic.headless.core.utils.StatUtils;
|
||||
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
|
||||
import com.tencent.supersonic.headless.server.service.AppService;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class QueryServiceImpl implements QueryService {
|
||||
|
||||
protected final com.google.common.cache.Cache<String, List<ItemUseResp>> itemUseCache =
|
||||
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
|
||||
|
||||
private final StatUtils statUtils;
|
||||
private final CacheUtils cacheUtils;
|
||||
private final QueryUtils queryUtils;
|
||||
private final QueryReqConverter queryReqConverter;
|
||||
private final Catalog catalog;
|
||||
private final AppService appService;
|
||||
|
||||
@Value("${query.cache.enable:true}")
|
||||
private Boolean cacheEnable;
|
||||
|
||||
private final HeadlessQueryEngine headlessQueryEngine;
|
||||
|
||||
public QueryServiceImpl(
|
||||
StatUtils statUtils,
|
||||
CacheUtils cacheUtils,
|
||||
QueryUtils queryUtils,
|
||||
QueryReqConverter queryReqConverter,
|
||||
HeadlessQueryEngine headlessQueryEngine,
|
||||
Catalog catalog,
|
||||
AppService appService) {
|
||||
this.statUtils = statUtils;
|
||||
this.cacheUtils = cacheUtils;
|
||||
this.queryUtils = queryUtils;
|
||||
this.queryReqConverter = queryReqConverter;
|
||||
this.headlessQueryEngine = headlessQueryEngine;
|
||||
this.catalog = catalog;
|
||||
this.appService = appService;
|
||||
}
|
||||
|
||||
@Override
|
||||
@S2SQLDataPermission
|
||||
@SneakyThrows
|
||||
public Object queryBySql(QueryS2SQLReq queryS2SQLReq, User user) {
|
||||
statUtils.initStatInfo(queryS2SQLReq, user);
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
try {
|
||||
queryStatement = convertToQueryStatement(queryS2SQLReq, user);
|
||||
} catch (Exception e) {
|
||||
log.info("convertToQueryStatement has a exception:", e);
|
||||
}
|
||||
log.info("queryStatement:{}", queryStatement);
|
||||
QueryResultWithSchemaResp results = headlessQueryEngine.execute(queryStatement);
|
||||
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
|
||||
return results;
|
||||
}
|
||||
|
||||
public Object queryByQueryStatement(QueryStatement queryStatement) {
|
||||
return headlessQueryEngine.execute(queryStatement);
|
||||
}
|
||||
|
||||
private QueryStatement convertToQueryStatement(QueryS2SQLReq querySqlCmd, User user) throws Exception {
|
||||
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
|
||||
filter.setModelIds(querySqlCmd.getModelIds());
|
||||
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
|
||||
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user);
|
||||
QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, modelSchemaResps);
|
||||
queryStatement.setModelIds(querySqlCmd.getModelIds());
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception {
|
||||
QueryResultWithSchemaResp queryResultWithColumns = null;
|
||||
log.info("[queryStructCmd:{}]", queryStructCmd);
|
||||
try {
|
||||
statUtils.initStatInfo(queryStructCmd, user);
|
||||
String cacheKey = cacheUtils.generateCacheKey(getKeyByModelIds(queryStructCmd.getModelIds()),
|
||||
queryStructCmd.generateCommandMd5());
|
||||
handleGlobalCacheDisable(queryStructCmd);
|
||||
boolean isCache = isCache(queryStructCmd);
|
||||
if (isCache) {
|
||||
queryResultWithColumns = queryByCache(cacheKey, queryStructCmd);
|
||||
if (queryResultWithColumns != null) {
|
||||
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
}
|
||||
StatUtils.get().setUseResultCache(false);
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq(queryStructCmd);
|
||||
queryStatement.setIsS2SQL(false);
|
||||
queryStatement = headlessQueryEngine.plan(queryStatement);
|
||||
QueryExecutor queryExecutor = headlessQueryEngine.route(queryStatement);
|
||||
if (queryExecutor != null) {
|
||||
queryResultWithColumns = headlessQueryEngine.execute(queryStatement);
|
||||
if (isCache) {
|
||||
// if queryResultWithColumns is not null, update cache data
|
||||
queryUtils.cacheResultLogic(cacheKey, queryResultWithColumns);
|
||||
}
|
||||
}
|
||||
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
|
||||
return queryResultWithColumns;
|
||||
} catch (Exception e) {
|
||||
log.warn("exception in queryByStruct, e: ", e);
|
||||
statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@StructDataPermission
|
||||
@SneakyThrows
|
||||
public QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) {
|
||||
return queryByStruct(queryStructCmd, user);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user)
|
||||
throws Exception {
|
||||
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
|
||||
String cacheKey = cacheUtils.generateCacheKey(
|
||||
getKeyByModelIds(queryMultiStructReq.getQueryStructReqs().get(0).getModelIds()),
|
||||
queryMultiStructReq.generateCommandMd5());
|
||||
boolean isCache = isCache(queryMultiStructReq);
|
||||
QueryResultWithSchemaResp queryResultWithColumns;
|
||||
if (isCache) {
|
||||
queryResultWithColumns = queryByCache(cacheKey, queryMultiStructReq);
|
||||
if (queryResultWithColumns != null) {
|
||||
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
}
|
||||
log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq);
|
||||
try {
|
||||
QueryStatement sqlParser = getQueryStatementByMultiStruct(queryMultiStructReq);
|
||||
queryResultWithColumns = headlessQueryEngine.execute(sqlParser);
|
||||
if (queryResultWithColumns != null) {
|
||||
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
|
||||
queryUtils.fillItemNameInfo(queryResultWithColumns, queryMultiStructReq);
|
||||
}
|
||||
return queryResultWithColumns;
|
||||
} catch (Exception e) {
|
||||
log.warn("exception in queryByMultiStruct, e: ", e);
|
||||
statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private QueryStatement getQueryStatementByMultiStruct(QueryMultiStructReq queryMultiStructReq) throws Exception {
|
||||
List<QueryStatement> sqlParsers = new ArrayList<>();
|
||||
for (QueryStructReq queryStructCmd : queryMultiStructReq.getQueryStructReqs()) {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq(queryStructCmd);
|
||||
queryStatement.setIsS2SQL(false);
|
||||
queryStatement = headlessQueryEngine.plan(queryStatement);
|
||||
queryUtils.checkSqlParse(queryStatement);
|
||||
sqlParsers.add(queryStatement);
|
||||
}
|
||||
log.info("multi sqlParser:{}", sqlParsers);
|
||||
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
|
||||
QueryS2SQLReq queryS2SQLReq = generateDimValueQuerySql(queryDimValueReq);
|
||||
return (QueryResultWithSchemaResp) queryBySql(queryS2SQLReq, user);
|
||||
}
|
||||
|
||||
private void handleGlobalCacheDisable(QueryStructReq queryStructCmd) {
|
||||
if (!cacheEnable) {
|
||||
Cache cacheInfo = new Cache();
|
||||
cacheInfo.setCache(false);
|
||||
queryStructCmd.setCacheInfo(cacheInfo);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
|
||||
if (itemUseReq.getCacheEnable()) {
|
||||
return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> {
|
||||
List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
|
||||
itemUseCache.put(JsonUtil.toString(itemUseReq), data);
|
||||
return data;
|
||||
});
|
||||
}
|
||||
return statUtils.getStatInfo(itemUseReq);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception {
|
||||
QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum();
|
||||
T queryReq = explainSqlReq.getQueryReq();
|
||||
|
||||
if (QueryType.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2SQLReq) {
|
||||
QueryStatement queryStatement = convertToQueryStatement((QueryS2SQLReq) queryReq, user);
|
||||
return getExplainResp(queryStatement);
|
||||
}
|
||||
if (QueryType.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq((QueryStructReq) queryReq);
|
||||
queryStatement.setIsS2SQL(false);
|
||||
queryStatement = headlessQueryEngine.plan(queryStatement);
|
||||
return getExplainResp(queryStatement);
|
||||
}
|
||||
if (QueryType.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryMultiStructReq) {
|
||||
QueryMultiStructReq queryMultiStructReq = (QueryMultiStructReq) queryReq;
|
||||
QueryStatement queryStatement = getQueryStatementByMultiStruct(queryMultiStructReq);
|
||||
return getExplainResp(queryStatement);
|
||||
}
|
||||
|
||||
throw new IllegalArgumentException("Parameters are invalid, explainSqlReq: " + explainSqlReq);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ApiHeaderCheck
|
||||
public ItemQueryResultResp metricDataQueryById(QueryItemReq queryItemReq,
|
||||
HttpServletRequest request) throws Exception {
|
||||
AppDetailResp appDetailResp = getAppDetailResp(request);
|
||||
authCheck(appDetailResp, queryItemReq.getIds(), ApiItemType.METRIC);
|
||||
List<SingleItemQueryResult> results = Lists.newArrayList();
|
||||
Map<Long, Item> map = appDetailResp.getConfig().getItems().stream()
|
||||
.collect(Collectors.toMap(Item::getId, i -> i));
|
||||
for (Long id : queryItemReq.getIds()) {
|
||||
Item item = map.get(id);
|
||||
SingleItemQueryResult apiQuerySingleResult = dataQuery(appDetailResp.getId(),
|
||||
item, queryItemReq.getDateConf(), queryItemReq.getLimit());
|
||||
results.add(apiQuerySingleResult);
|
||||
}
|
||||
return ItemQueryResultResp.builder().results(results).build();
|
||||
}
|
||||
|
||||
private SingleItemQueryResult dataQuery(Integer appId, Item item, DateConf dateConf, Long limit) throws Exception {
|
||||
MetricResp metricResp = catalog.getMetric(item.getId());
|
||||
List<Item> items = item.getRelateItems();
|
||||
List<DimensionResp> dimensionResps = Lists.newArrayList();
|
||||
if (!org.springframework.util.CollectionUtils.isEmpty(items)) {
|
||||
List<Long> ids = items.stream().map(Item::getId).collect(Collectors.toList());
|
||||
DimensionFilter dimensionFilter = new DimensionFilter();
|
||||
dimensionFilter.setIds(ids);
|
||||
dimensionResps = catalog.getDimensions(dimensionFilter);
|
||||
}
|
||||
QueryStructReq queryStructReq = buildQueryStructReq(dimensionResps, metricResp, dateConf, limit);
|
||||
QueryResultWithSchemaResp queryResultWithSchemaResp =
|
||||
queryByStruct(queryStructReq, User.getAppUser(appId));
|
||||
SingleItemQueryResult apiQuerySingleResult = new SingleItemQueryResult();
|
||||
apiQuerySingleResult.setItem(item);
|
||||
apiQuerySingleResult.setResult(queryResultWithSchemaResp);
|
||||
return apiQuerySingleResult;
|
||||
}
|
||||
|
||||
private AppDetailResp getAppDetailResp(HttpServletRequest request) {
|
||||
int appId = Integer.parseInt(request.getHeader(ApiHeaderCheckAspect.APPID));
|
||||
return appService.getApp(appId);
|
||||
}
|
||||
|
||||
private QueryStructReq buildQueryStructReq(List<DimensionResp> dimensionResps,
|
||||
MetricResp metricResp, DateConf dateConf, Long limit) {
|
||||
Set<Long> modelIds = dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
|
||||
modelIds.add(metricResp.getModelId());
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setGroups(dimensionResps.stream()
|
||||
.map(DimensionResp::getBizName).collect(Collectors.toList()));
|
||||
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricResp.getBizName());
|
||||
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
|
||||
queryStructReq.setDateInfo(dateConf);
|
||||
queryStructReq.setModelIds(modelIds);
|
||||
queryStructReq.setLimit(limit);
|
||||
return queryStructReq;
|
||||
}
|
||||
|
||||
private String getTimeDimension(DateConf dateConf) {
|
||||
if (Constants.MONTH.equals(dateConf.getPeriod())) {
|
||||
return TimeDimensionEnum.MONTH.getName();
|
||||
} else if (Constants.WEEK.equals(dateConf.getPeriod())) {
|
||||
return TimeDimensionEnum.WEEK.getName();
|
||||
} else {
|
||||
return TimeDimensionEnum.DAY.getName();
|
||||
}
|
||||
}
|
||||
|
||||
private void authCheck(AppDetailResp appDetailResp, List<Long> ids, ApiItemType type) {
|
||||
Set<Long> idsInApp = appDetailResp.getConfig().getAllItems().stream()
|
||||
.filter(item -> type.equals(item.getType())).map(Item::getId).collect(Collectors.toSet());
|
||||
if (!idsInApp.containsAll(ids)) {
|
||||
throw new InvalidArgumentException("查询范围超过应用申请范围, 请检查");
|
||||
}
|
||||
}
|
||||
|
||||
private ExplainResp getExplainResp(QueryStatement queryStatement) {
|
||||
String sql = "";
|
||||
if (Objects.nonNull(queryStatement)) {
|
||||
sql = queryStatement.getSql();
|
||||
}
|
||||
return ExplainResp.builder().sql(sql).build();
|
||||
}
|
||||
|
||||
public QueryStatement parseMetricReq(MetricQueryReq metricReq) throws Exception {
|
||||
QueryStructReq queryStructCmd = new QueryStructReq();
|
||||
return headlessQueryEngine.physicalSql(queryStructCmd, metricReq);
|
||||
}
|
||||
|
||||
private boolean isCache(QueryStructReq queryStructCmd) {
|
||||
if (!cacheEnable) {
|
||||
return false;
|
||||
}
|
||||
if (queryStructCmd.getCacheInfo() != null) {
|
||||
return queryStructCmd.getCacheInfo().getCache();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean isCache(QueryMultiStructReq queryStructCmd) {
|
||||
if (!cacheEnable) {
|
||||
return false;
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(queryStructCmd.getQueryStructReqs())
|
||||
&& queryStructCmd.getQueryStructReqs().get(0).getCacheInfo() != null) {
|
||||
return queryStructCmd.getQueryStructReqs().get(0).getCacheInfo().getCache();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private QueryResultWithSchemaResp queryByCache(String key, Object queryCmd) {
|
||||
|
||||
Object resultObject = cacheUtils.get(key);
|
||||
if (Objects.nonNull(resultObject)) {
|
||||
log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString());
|
||||
statUtils.updateResultCacheKey(key);
|
||||
return (QueryResultWithSchemaResp) resultObject;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private QueryS2SQLReq generateDimValueQuerySql(QueryDimValueReq queryDimValueReq) {
|
||||
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
|
||||
List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
|
||||
DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(),
|
||||
queryDimValueReq.getModelId());
|
||||
ModelResp modelResp = modelResps.get(0);
|
||||
String sql = String.format("select distinct %s from %s", dimensionResp.getName(), modelResp.getName());
|
||||
List<Dim> timeDims = modelResp.getTimeDimension();
|
||||
if (CollectionUtils.isNotEmpty(timeDims)) {
|
||||
sql = String.format("%s where %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(),
|
||||
queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
|
||||
queryDimValueReq.getDateInfo().getEndDate());
|
||||
}
|
||||
queryS2SQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
|
||||
queryS2SQLReq.setSql(sql);
|
||||
return queryS2SQLReq;
|
||||
}
|
||||
|
||||
private String getKeyByModelIds(List<Long> modelIds) {
|
||||
return String.join(",", modelIds.stream()
|
||||
.map(Object::toString).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.service.impl;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.headless.api.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.headless.api.request.PageDimensionReq;
|
||||
import com.tencent.supersonic.headless.api.request.PageMetricReq;
|
||||
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.DomainResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.service.QueryService;
|
||||
import com.tencent.supersonic.headless.core.service.SchemaService;
|
||||
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||
import com.tencent.supersonic.headless.server.service.DomainService;
|
||||
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class SchemaServiceImpl implements SchemaService {
|
||||
|
||||
private final QueryService queryService;
|
||||
private final ModelService modelService;
|
||||
private final DimensionService dimensionService;
|
||||
private final MetricService metricService;
|
||||
private final DomainService domainService;
|
||||
|
||||
public SchemaServiceImpl(QueryService queryService,
|
||||
ModelService modelService,
|
||||
DimensionService dimensionService,
|
||||
MetricService metricService,
|
||||
DomainService domainService) {
|
||||
this.queryService = queryService;
|
||||
this.modelService = modelService;
|
||||
this.dimensionService = dimensionService;
|
||||
this.metricService = metricService;
|
||||
this.domainService = domainService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ModelSchemaResp> fetchModelSchema(ModelSchemaFilterReq filter, User user) {
|
||||
List<ModelSchemaResp> domainSchemaDescList = modelService.fetchModelSchema(filter);
|
||||
ItemUseReq itemUseCommend = new ItemUseReq();
|
||||
itemUseCommend.setModelIds(filter.getModelIds());
|
||||
|
||||
List<ItemUseResp> statInfos = queryService.getStatInfo(itemUseCommend);
|
||||
log.debug("statInfos:{}", statInfos);
|
||||
fillCnt(domainSchemaDescList, statInfos);
|
||||
return domainSchemaDescList;
|
||||
}
|
||||
|
||||
private void fillCnt(List<ModelSchemaResp> domainSchemaDescList, List<ItemUseResp> statInfos) {
|
||||
|
||||
Map<String, ItemUseResp> typeIdAndStatPair = statInfos.stream()
|
||||
.collect(Collectors.toMap(
|
||||
itemUseInfo -> itemUseInfo.getType() + AT_SYMBOL + AT_SYMBOL + itemUseInfo.getBizName(),
|
||||
itemUseInfo -> itemUseInfo,
|
||||
(item1, item2) -> item1));
|
||||
log.debug("typeIdAndStatPair:{}", typeIdAndStatPair);
|
||||
for (ModelSchemaResp domainSchemaDesc : domainSchemaDescList) {
|
||||
fillDimCnt(domainSchemaDesc, typeIdAndStatPair);
|
||||
fillMetricCnt(domainSchemaDesc, typeIdAndStatPair);
|
||||
}
|
||||
}
|
||||
|
||||
private void fillMetricCnt(ModelSchemaResp domainSchemaDesc, Map<String, ItemUseResp> typeIdAndStatPair) {
|
||||
List<MetricSchemaResp> metrics = domainSchemaDesc.getMetrics();
|
||||
if (CollectionUtils.isEmpty(domainSchemaDesc.getMetrics())) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!CollectionUtils.isEmpty(metrics)) {
|
||||
metrics.stream().forEach(metric -> {
|
||||
String key = TypeEnums.METRIC.getName() + AT_SYMBOL + AT_SYMBOL + metric.getBizName();
|
||||
if (typeIdAndStatPair.containsKey(key)) {
|
||||
metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
|
||||
}
|
||||
});
|
||||
}
|
||||
domainSchemaDesc.setMetrics(metrics);
|
||||
}
|
||||
|
||||
private void fillDimCnt(ModelSchemaResp domainSchemaDesc, Map<String, ItemUseResp> typeIdAndStatPair) {
|
||||
List<DimSchemaResp> dimensions = domainSchemaDesc.getDimensions();
|
||||
if (CollectionUtils.isEmpty(domainSchemaDesc.getDimensions())) {
|
||||
return;
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(dimensions)) {
|
||||
dimensions.stream().forEach(dim -> {
|
||||
String key = TypeEnums.DIMENSION.getName() + AT_SYMBOL + AT_SYMBOL + dim.getBizName();
|
||||
if (typeIdAndStatPair.containsKey(key)) {
|
||||
dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
|
||||
}
|
||||
});
|
||||
}
|
||||
domainSchemaDesc.setDimensions(dimensions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionCmd, User user) {
|
||||
return dimensionService.queryDimension(pageDimensionCmd);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user) {
|
||||
return metricService.queryMetric(pageMetricReq, user);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainList(User user) {
|
||||
return domainService.getDomainListWithAdminAuth(user);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ModelResp> getModelList(User user, AuthType authTypeEnum, Long domainId) {
|
||||
return modelService.getModelListWithAuth(user, domainId, authTypeEnum);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,18 +1,15 @@
|
||||
package com.tencent.supersonic.headless.core.utils;
|
||||
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
|
||||
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
||||
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
|
||||
import com.tencent.supersonic.headless.core.parser.SqlParser;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser;
|
||||
import com.tencent.supersonic.headless.core.parser.convert.DefaultDimValueConverter;
|
||||
import com.tencent.supersonic.headless.core.parser.convert.ZipperModelConverter;
|
||||
import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
|
||||
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
||||
import com.tencent.supersonic.headless.core.optimizer.DetailQuery;
|
||||
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
|
||||
import com.tencent.supersonic.headless.core.parser.convert.CalculateAggConverter;
|
||||
import com.tencent.supersonic.headless.core.parser.convert.MetricCheckConverter;
|
||||
import com.tencent.supersonic.headless.core.parser.convert.ParserDefaultConverter;
|
||||
import com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter;
|
||||
import com.tencent.supersonic.headless.core.parser.converter.ParserDefaultConverter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
@@ -22,14 +19,14 @@ import java.util.stream.Collectors;
|
||||
public class ComponentFactory {
|
||||
|
||||
private static List<HeadlessConverter> headlessConverters = new ArrayList<>();
|
||||
private static List<QueryExecutor> queryExecutors = new ArrayList<>();
|
||||
private static Map<String, QueryOptimizer> queryOptimizers = new HashMap<>();
|
||||
private static List<QueryExecutor> queryExecutors = new ArrayList<>();
|
||||
private static SqlParser sqlParser;
|
||||
|
||||
static {
|
||||
initSemanticConverter();
|
||||
initQueryExecutors();
|
||||
initQueryOptimizer();
|
||||
initQueryExecutors();
|
||||
}
|
||||
|
||||
public static List<HeadlessConverter> getSemanticConverters() {
|
||||
@@ -39,13 +36,6 @@ public class ComponentFactory {
|
||||
return headlessConverters;
|
||||
}
|
||||
|
||||
public static List<QueryExecutor> getQueryExecutors() {
|
||||
if (queryExecutors.isEmpty()) {
|
||||
initQueryExecutors();
|
||||
}
|
||||
return queryExecutors;
|
||||
}
|
||||
|
||||
public static List<QueryOptimizer> getQueryOptimizers() {
|
||||
if (queryOptimizers.isEmpty()) {
|
||||
initQueryOptimizer();
|
||||
@@ -53,6 +43,13 @@ public class ComponentFactory {
|
||||
return queryOptimizers.values().stream().collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static List<QueryExecutor> getQueryExecutors() {
|
||||
if (queryExecutors.isEmpty()) {
|
||||
initQueryExecutors();
|
||||
}
|
||||
return queryExecutors;
|
||||
}
|
||||
|
||||
public static SqlParser getSqlParser() {
|
||||
if (sqlParser == null) {
|
||||
sqlParser = ContextUtils.getContext().getBean("CalciteSqlParser", CalciteSqlParser.class);
|
||||
@@ -76,16 +73,13 @@ public class ComponentFactory {
|
||||
queryOptimizers.put("DetailQuery", getBean("DetailQuery", DetailQuery.class));
|
||||
}
|
||||
|
||||
private static void initSemanticConverter() {
|
||||
headlessConverters.add(getBean("MetricCheckConverter", MetricCheckConverter.class));
|
||||
headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
|
||||
headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
|
||||
headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
|
||||
headlessConverters.add(getBean("ZipperModelConverter", ZipperModelConverter.class));
|
||||
}
|
||||
|
||||
private static void initQueryExecutors() {
|
||||
queryExecutors.add(ContextUtils.getContext().getBean("JdbcExecutor", JdbcExecutor.class));
|
||||
}
|
||||
|
||||
private static void initSemanticConverter() {
|
||||
headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
|
||||
headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
package com.tencent.supersonic.headless.core.utils;
|
||||
|
||||
import com.alibaba.druid.util.StringUtils;
|
||||
import com.tencent.supersonic.common.util.MD5Util;
|
||||
import com.tencent.supersonic.headless.api.enums.DataType;
|
||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.core.pojo.Database;
|
||||
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.COLON;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.DOUBLE_SLASH;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.EMPTY;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.JDBC_PREFIX_FORMATTER;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.NEW_LINE_CHAR;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.PATTERN_JDBC_TYPE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
|
||||
|
||||
@Slf4j
|
||||
public class JdbcDataSourceUtils {
|
||||
|
||||
@Getter
|
||||
private static Set releaseSourceSet = new HashSet();
|
||||
private JdbcDataSource jdbcDataSource;
|
||||
|
||||
public JdbcDataSourceUtils(JdbcDataSource jdbcDataSource) {
|
||||
this.jdbcDataSource = jdbcDataSource;
|
||||
}
|
||||
|
||||
public static boolean testDatabase(Database database) {
|
||||
|
||||
try {
|
||||
Class.forName(getDriverClassName(database.getConnectInfo().getUrl()));
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.error(e.toString(), e);
|
||||
return false;
|
||||
}
|
||||
try (Connection con = DriverManager.getConnection(database.getConnectInfo().getUrl(),
|
||||
database.getConnectInfo().getUserName(), database.getConnectInfo().getPassword());) {
|
||||
return con != null;
|
||||
} catch (SQLException e) {
|
||||
log.error(e.toString(), e);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static void releaseConnection(Connection connection) {
|
||||
if (null != connection) {
|
||||
try {
|
||||
connection.close();
|
||||
} catch (Exception e) {
|
||||
log.error("Connection release error", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void closeResult(ResultSet rs) {
|
||||
if (rs != null) {
|
||||
try {
|
||||
rs.close();
|
||||
} catch (Exception e) {
|
||||
log.error("ResultSet close error", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static String isSupportedDatasource(String jdbcUrl) {
|
||||
String dataSourceName = getDataSourceName(jdbcUrl);
|
||||
if (StringUtils.isEmpty(dataSourceName)) {
|
||||
throw new RuntimeException("Not supported dataSource: jdbcUrl=" + jdbcUrl);
|
||||
}
|
||||
|
||||
if (!DataType.getAllSupportedDatasourceNameSet().contains(dataSourceName)) {
|
||||
throw new RuntimeException("Not supported dataSource: jdbcUrl=" + jdbcUrl);
|
||||
}
|
||||
|
||||
String urlPrefix = String.format(JDBC_PREFIX_FORMATTER, dataSourceName);
|
||||
String checkUrl = jdbcUrl.replaceFirst(DOUBLE_SLASH, EMPTY).replaceFirst(AT_SYMBOL, EMPTY);
|
||||
if (urlPrefix.equals(checkUrl)) {
|
||||
throw new RuntimeException("Communications link failure");
|
||||
}
|
||||
|
||||
return dataSourceName;
|
||||
}
|
||||
|
||||
public static String getDataSourceName(String jdbcUrl) {
|
||||
String dataSourceName = null;
|
||||
jdbcUrl = jdbcUrl.replaceAll(NEW_LINE_CHAR, EMPTY).replaceAll(SPACE, EMPTY).trim();
|
||||
Matcher matcher = PATTERN_JDBC_TYPE.matcher(jdbcUrl);
|
||||
if (matcher.find()) {
|
||||
dataSourceName = matcher.group().split(COLON)[1];
|
||||
}
|
||||
return dataSourceName;
|
||||
}
|
||||
|
||||
public static String getDriverClassName(String jdbcUrl) {
|
||||
|
||||
String className = null;
|
||||
|
||||
try {
|
||||
className = DriverManager.getDriver(jdbcUrl.trim()).getClass().getName();
|
||||
} catch (SQLException e) {
|
||||
log.error("e", e);
|
||||
}
|
||||
|
||||
if (!StringUtils.isEmpty(className) && !className.contains("com.sun.proxy")
|
||||
&& !className.contains("net.sf.cglib.proxy")) {
|
||||
return className;
|
||||
}
|
||||
|
||||
DataType dataTypeEnum = DataType.urlOf(jdbcUrl);
|
||||
if (dataTypeEnum != null) {
|
||||
return dataTypeEnum.getDriver();
|
||||
}
|
||||
|
||||
throw new RuntimeException("Not supported data type: jdbcUrl=" + jdbcUrl);
|
||||
}
|
||||
|
||||
public static String getKey(String name, String jdbcUrl, String username, String password, String version,
|
||||
boolean isExt) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
sb.append(StringUtils.isEmpty(name) ? "null" : name).append(COLON);
|
||||
sb.append(StringUtils.isEmpty(username) ? "null" : username).append(COLON);
|
||||
sb.append(StringUtils.isEmpty(password) ? "null" : password).append(COLON);
|
||||
sb.append(jdbcUrl.trim()).append(COLON);
|
||||
if (isExt && !StringUtils.isEmpty(version)) {
|
||||
sb.append(version);
|
||||
} else {
|
||||
sb.append("null");
|
||||
}
|
||||
|
||||
return MD5Util.getMD5(sb.toString(), true, 64);
|
||||
}
|
||||
|
||||
public DataSource getDataSource(DatabaseResp databaseResp) throws RuntimeException {
|
||||
return jdbcDataSource.getDataSource(databaseResp);
|
||||
}
|
||||
|
||||
public Connection getConnection(DatabaseResp databaseResp) throws RuntimeException {
|
||||
Connection conn = getConnectionWithRetry(databaseResp);
|
||||
if (conn == null) {
|
||||
try {
|
||||
releaseDataSource(databaseResp);
|
||||
DataSource dataSource = getDataSource(databaseResp);
|
||||
return dataSource.getConnection();
|
||||
} catch (Exception e) {
|
||||
log.error("Get connection error, jdbcUrl:{}, e:{}", databaseResp.getUrl(), e);
|
||||
throw new RuntimeException("Get connection error, jdbcUrl:" + databaseResp.getUrl()
|
||||
+ " you can try again later or reset datasource");
|
||||
}
|
||||
}
|
||||
return conn;
|
||||
}
|
||||
|
||||
private Connection getConnectionWithRetry(DatabaseResp databaseResp) {
|
||||
int rc = 1;
|
||||
for (; ; ) {
|
||||
|
||||
if (rc > 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
Connection connection = getDataSource(databaseResp).getConnection();
|
||||
if (connection != null && connection.isValid(5)) {
|
||||
return connection;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("e", e);
|
||||
}
|
||||
|
||||
try {
|
||||
Thread.sleep((long) Math.pow(2, rc) * 1000);
|
||||
} catch (InterruptedException e) {
|
||||
log.error("e", e);
|
||||
}
|
||||
|
||||
rc++;
|
||||
}
|
||||
}
|
||||
|
||||
public void releaseDataSource(DatabaseResp databaseResp) {
|
||||
jdbcDataSource.removeDatasource(databaseResp);
|
||||
}
|
||||
}
|
||||
@@ -1,493 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.utils;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
|
||||
import com.tencent.supersonic.common.pojo.ItemDateResp;
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import com.tencent.supersonic.common.util.DateModeUtils;
|
||||
import com.tencent.supersonic.common.util.SqlFilterUtils;
|
||||
import com.tencent.supersonic.common.util.StringUtil;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.service.SchemaService;
|
||||
import com.tencent.supersonic.headless.server.pojo.EngineTypeEnum;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.apache.commons.lang3.tuple.Triple;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.DAY;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class QueryStructUtils {
|
||||
|
||||
public static Set<String> internalTimeCols = new HashSet<>(
|
||||
Arrays.asList("dayno", "sys_imp_date", "sys_imp_week", "sys_imp_month"));
|
||||
public static Set<String> internalCols;
|
||||
|
||||
static {
|
||||
internalCols = new HashSet<>(Arrays.asList("plat_sys_var"));
|
||||
internalCols.addAll(internalTimeCols);
|
||||
}
|
||||
|
||||
private final DateModeUtils dateModeUtils;
|
||||
private final SqlFilterUtils sqlFilterUtils;
|
||||
private final Catalog catalog;
|
||||
@Value("${internal.metric.cnt.suffix:internal_cnt}")
|
||||
private String internalMetricNameSuffix;
|
||||
@Value("${metricParser.agg.mysql.lowVersion:5.7}")
|
||||
private String mysqlLowVersion;
|
||||
@Value("${metricParser.agg.ck.lowVersion:20.4}")
|
||||
private String ckLowVersion;
|
||||
@Autowired
|
||||
private SchemaService schemaService;
|
||||
|
||||
private String variablePrefix = "'${";
|
||||
|
||||
public QueryStructUtils(
|
||||
DateModeUtils dateModeUtils,
|
||||
SqlFilterUtils sqlFilterUtils, Catalog catalog) {
|
||||
|
||||
this.dateModeUtils = dateModeUtils;
|
||||
this.sqlFilterUtils = sqlFilterUtils;
|
||||
this.catalog = catalog;
|
||||
}
|
||||
|
||||
private List<Long> getDimensionIds(QueryStructReq queryStructCmd) {
|
||||
List<Long> dimensionIds = new ArrayList<>();
|
||||
MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds());
|
||||
List<DimensionResp> dimensions = catalog.getDimensions(metaFilter);
|
||||
Map<String, List<DimensionResp>> pair = dimensions.stream()
|
||||
.collect(Collectors.groupingBy(DimensionResp::getBizName));
|
||||
for (String group : queryStructCmd.getGroups()) {
|
||||
if (pair.containsKey(group)) {
|
||||
dimensionIds.add(pair.get(group).get(0).getId());
|
||||
}
|
||||
}
|
||||
|
||||
List<String> filtersCols = sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter());
|
||||
for (String col : filtersCols) {
|
||||
if (pair.containsKey(col)) {
|
||||
dimensionIds.add(pair.get(col).get(0).getId());
|
||||
}
|
||||
}
|
||||
|
||||
return dimensionIds;
|
||||
}
|
||||
|
||||
private List<Long> getMetricIds(QueryStructReq queryStructCmd) {
|
||||
List<Long> metricIds = new ArrayList<>();
|
||||
MetaFilter metaFilter = new MetaFilter(queryStructCmd.getModelIds());
|
||||
List<MetricResp> metrics = catalog.getMetrics(metaFilter);
|
||||
Map<String, List<MetricResp>> pair = metrics.stream().collect(Collectors.groupingBy(SchemaItem::getBizName));
|
||||
for (Aggregator agg : queryStructCmd.getAggregators()) {
|
||||
if (pair.containsKey(agg.getColumn())) {
|
||||
metricIds.add(pair.get(agg.getColumn()).get(0).getId());
|
||||
}
|
||||
}
|
||||
List<String> filtersCols = sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter());
|
||||
for (String col : filtersCols) {
|
||||
if (pair.containsKey(col)) {
|
||||
metricIds.add(pair.get(col).get(0).getId());
|
||||
}
|
||||
}
|
||||
return metricIds;
|
||||
}
|
||||
|
||||
public String getDateWhereClause(QueryStructReq queryStructCmd) {
|
||||
DateConf dateInfo = queryStructCmd.getDateInfo();
|
||||
if (Objects.isNull(dateInfo) || Objects.isNull(dateInfo.getDateMode())) {
|
||||
return "";
|
||||
}
|
||||
if (dateInfo.getDateMode().equals(DateMode.RECENT)) {
|
||||
if (dateInfo.getUnit() <= 0) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
|
||||
List<Long> metricIds = getMetricIds(queryStructCmd);
|
||||
|
||||
ItemDateResp dateDate = catalog.getItemDate(
|
||||
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()),
|
||||
new ItemDateFilter(metricIds, TypeEnums.METRIC.getName()));
|
||||
if (Objects.isNull(dateDate)
|
||||
|| Strings.isEmpty(dateDate.getStartDate())
|
||||
&& Strings.isEmpty(dateDate.getEndDate())) {
|
||||
if (dateInfo.getDateMode().equals(DateMode.LIST)) {
|
||||
return dateModeUtils.listDateStr(dateDate, dateInfo);
|
||||
}
|
||||
if (dateInfo.getDateMode().equals(DateMode.BETWEEN)) {
|
||||
return dateModeUtils.betweenDateStr(dateDate, dateInfo);
|
||||
}
|
||||
if (dateModeUtils.hasAvailableDataMode(dateInfo)) {
|
||||
return dateModeUtils.hasDataModeStr(dateDate, dateInfo);
|
||||
}
|
||||
|
||||
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
|
||||
}
|
||||
log.info("dateDate:{}", dateDate);
|
||||
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
|
||||
}
|
||||
|
||||
public String generateWhere(QueryStructReq queryStructCmd) {
|
||||
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructCmd.getOriginalFilter());
|
||||
String whereFromDate = getDateWhereClause(queryStructCmd);
|
||||
return mergeDateWhereClause(queryStructCmd, whereClauseFromFilter, whereFromDate);
|
||||
}
|
||||
|
||||
public String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter,
|
||||
String whereFromDate) {
|
||||
if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
|
||||
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
|
||||
} else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
|
||||
return whereClauseFromFilter;
|
||||
} else if (Strings.isNotEmpty(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
|
||||
return whereFromDate;
|
||||
} else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
|
||||
log.info("the current date information is empty, enter the date initialization logic");
|
||||
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
|
||||
}
|
||||
return whereClauseFromFilter;
|
||||
}
|
||||
|
||||
public Set<String> getResNameEn(QueryStructReq queryStructCmd) {
|
||||
Set<String> resNameEnSet = new HashSet<>();
|
||||
queryStructCmd.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn()));
|
||||
resNameEnSet.addAll(queryStructCmd.getGroups());
|
||||
queryStructCmd.getOrders().stream().forEach(order -> resNameEnSet.add(order.getColumn()));
|
||||
sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
|
||||
return resNameEnSet;
|
||||
}
|
||||
|
||||
public Set<String> getResName(QueryS2SQLReq queryS2SQLReq) {
|
||||
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql())
|
||||
.stream().collect(Collectors.toSet());
|
||||
return resNameSet;
|
||||
}
|
||||
|
||||
public Set<String> getResNameEnExceptInternalCol(QueryStructReq queryStructCmd) {
|
||||
Set<String> resNameEnSet = getResNameEn(queryStructCmd);
|
||||
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public Set<String> getResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq, User user) {
|
||||
Set<String> resNameSet = getResName(queryS2SQLReq);
|
||||
Set<String> resNameEnSet = new HashSet<>();
|
||||
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
|
||||
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds());
|
||||
filter.setModelIds(modelIds);
|
||||
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
|
||||
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
|
||||
List<MetricSchemaResp> metrics = modelSchemaRespList.get(0).getMetrics();
|
||||
List<DimSchemaResp> dimensions = modelSchemaRespList.get(0).getDimensions();
|
||||
metrics.stream().forEach(o -> {
|
||||
if (resNameSet.contains(o.getName())) {
|
||||
resNameEnSet.add(o.getBizName());
|
||||
}
|
||||
});
|
||||
dimensions.stream().forEach(o -> {
|
||||
if (resNameSet.contains(o.getName())) {
|
||||
resNameEnSet.add(o.getBizName());
|
||||
}
|
||||
});
|
||||
}
|
||||
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public Set<String> getFilterResNameEn(QueryStructReq queryStructCmd) {
|
||||
Set<String> resNameEnSet = new HashSet<>();
|
||||
sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
|
||||
return resNameEnSet;
|
||||
}
|
||||
|
||||
public Set<String> getFilterResNameEnExceptInternalCol(QueryStructReq queryStructCmd) {
|
||||
Set<String> resNameEnSet = getFilterResNameEn(queryStructCmd);
|
||||
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public Set<String> getFilterResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq) {
|
||||
String sql = queryS2SQLReq.getSql();
|
||||
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
|
||||
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public String generateInternalMetricName(Long modelId, List<String> groups) {
|
||||
String internalMetricNamePrefix = "";
|
||||
MetaFilter modelFilter = new MetaFilter(Collections.singletonList(modelId));
|
||||
List<DimensionResp> dimensions = catalog.getDimensions(modelFilter);
|
||||
if (!CollectionUtils.isEmpty(dimensions)) {
|
||||
internalMetricNamePrefix = dimensions.get(0).getModelBizName();
|
||||
}
|
||||
return internalMetricNamePrefix + UNDERLINE + internalMetricNameSuffix;
|
||||
}
|
||||
|
||||
public boolean isSupportWith(EngineTypeEnum engineTypeEnum, String version) {
|
||||
if (engineTypeEnum.equals(EngineTypeEnum.MYSQL) && Objects.nonNull(version) && version.startsWith(
|
||||
mysqlLowVersion)) {
|
||||
return false;
|
||||
}
|
||||
if (engineTypeEnum.equals(EngineTypeEnum.CLICKHOUSE) && Objects.nonNull(version)
|
||||
&& StringUtil.compareVersion(version,
|
||||
ckLowVersion) < 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public String generateZipperWhere(QueryStructReq queryStructCmd) {
|
||||
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructCmd.getOriginalFilter());
|
||||
String whereFromDate = getZipperDateWhereClause(queryStructCmd);
|
||||
return mergeDateWhereClause(queryStructCmd, whereClauseFromFilter, whereFromDate);
|
||||
}
|
||||
|
||||
public String generateZipperWhere(QueryStructReq queryStructCmd, ParseSqlReq parseSqlReq) {
|
||||
if (Objects.nonNull(parseSqlReq.getSql()) && !CollectionUtils.isEmpty(parseSqlReq.getTables())
|
||||
&& Objects.nonNull(queryStructCmd.getDateInfo())) {
|
||||
String sql = SqlParserRemoveHelper.removeWhere(parseSqlReq.getSql(),
|
||||
dateModeUtils.getDateCol());
|
||||
parseSqlReq.getTables().stream().forEach(t -> {
|
||||
if (Objects.nonNull(t)) {
|
||||
List<String> dimensions = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(t.getDimensions())) {
|
||||
dimensions.addAll(t.getDimensions().stream()
|
||||
.filter(d -> !dateModeUtils.getDateCol().contains(d.toLowerCase())).collect(
|
||||
Collectors.toList()));
|
||||
}
|
||||
dimensions.add(dateModeUtils.getDateColBegin(queryStructCmd.getDateInfo()));
|
||||
dimensions.add(dateModeUtils.getDateColEnd(queryStructCmd.getDateInfo()));
|
||||
t.setDimensions(dimensions);
|
||||
}
|
||||
});
|
||||
return SqlParserAddHelper.addWhere(sql,
|
||||
SqlParserSelectHelper.getTimeFilter(getTimeRanges(queryStructCmd),
|
||||
dateModeUtils.getDateColBegin(queryStructCmd.getDateInfo()),
|
||||
dateModeUtils.getDateColEnd(queryStructCmd.getDateInfo())));
|
||||
}
|
||||
return parseSqlReq.getSql();
|
||||
}
|
||||
|
||||
public String getZipperDateWhereClause(QueryStructReq queryStructCmd) {
|
||||
List<ImmutablePair<String, String>> timeRanges = getTimeRanges(queryStructCmd);
|
||||
List<String> wheres = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(timeRanges)) {
|
||||
for (ImmutablePair<String, String> range : timeRanges) {
|
||||
String strWhere = dateModeUtils.getDateWhereStr(queryStructCmd.getDateInfo(), range);
|
||||
if (!strWhere.isEmpty()) {
|
||||
wheres.add(strWhere);
|
||||
}
|
||||
}
|
||||
if (!wheres.isEmpty()) {
|
||||
return wheres.stream().collect(Collectors.joining(" or ", "(", ")"));
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public Triple<String, String, String> getBeginEndTime(QueryStructReq queryStructCmd) {
|
||||
if (Objects.isNull(queryStructCmd.getDateInfo())) {
|
||||
return Triple.of("", "", "");
|
||||
}
|
||||
DateConf dateConf = queryStructCmd.getDateInfo();
|
||||
String dateInfo = dateModeUtils.getSysDateCol(dateConf);
|
||||
if (dateInfo.isEmpty()) {
|
||||
return Triple.of("", "", "");
|
||||
}
|
||||
switch (dateConf.getDateMode()) {
|
||||
case AVAILABLE:
|
||||
case BETWEEN:
|
||||
return Triple.of(dateInfo, dateConf.getStartDate(), dateConf.getEndDate());
|
||||
case LIST:
|
||||
return Triple.of(dateInfo, Collections.min(dateConf.getDateList()),
|
||||
Collections.max(dateConf.getDateList()));
|
||||
case RECENT:
|
||||
ItemDateResp dateDate = getItemDateResp(queryStructCmd);
|
||||
LocalDate dateMax = LocalDate.now().minusDays(1);
|
||||
LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1);
|
||||
if (Objects.isNull(dateDate)) {
|
||||
return Triple.of(dateInfo, dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)),
|
||||
dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT)));
|
||||
}
|
||||
switch (dateConf.getPeriod()) {
|
||||
case DAY:
|
||||
ImmutablePair<String, String> dayInfo = dateModeUtils.recentDay(dateDate, dateConf);
|
||||
return Triple.of(dateInfo, dayInfo.left, dayInfo.right);
|
||||
case WEEK:
|
||||
ImmutablePair<String, String> weekInfo = dateModeUtils.recentWeek(dateDate, dateConf);
|
||||
return Triple.of(dateInfo, weekInfo.left, weekInfo.right);
|
||||
case MONTH:
|
||||
List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dateDate, dateConf);
|
||||
Optional<String> minBegins = rets.stream().map(i -> i.left).sorted().findFirst();
|
||||
Optional<String> maxBegins = rets.stream().map(i -> i.right).sorted(Comparator.reverseOrder())
|
||||
.findFirst();
|
||||
if (minBegins.isPresent() && maxBegins.isPresent()) {
|
||||
return Triple.of(dateInfo, minBegins.get(), maxBegins.get());
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
|
||||
}
|
||||
return Triple.of("", "", "");
|
||||
}
|
||||
|
||||
public List<ImmutablePair<String, String>> getTimeRanges(QueryStructReq queryStructCmd) {
|
||||
List<ImmutablePair<String, String>> ret = new ArrayList<>();
|
||||
if (Objects.isNull(queryStructCmd) || Objects.isNull(queryStructCmd.getDateInfo())) {
|
||||
return ret;
|
||||
}
|
||||
DateConf dateConf = queryStructCmd.getDateInfo();
|
||||
switch (dateConf.getDateMode()) {
|
||||
case AVAILABLE:
|
||||
case BETWEEN:
|
||||
ret.add(ImmutablePair.of(dateConf.getStartDate(), dateConf.getEndDate()));
|
||||
break;
|
||||
case LIST:
|
||||
for (String date : dateConf.getDateList()) {
|
||||
ret.add(ImmutablePair.of(date, date));
|
||||
}
|
||||
break;
|
||||
case RECENT:
|
||||
ItemDateResp dateDate = getItemDateResp(queryStructCmd);
|
||||
LocalDate dateMax = LocalDate.now().minusDays(1);
|
||||
LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1);
|
||||
if (Objects.isNull(dateDate)) {
|
||||
ret.add(ImmutablePair.of(dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)),
|
||||
dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT))));
|
||||
break;
|
||||
}
|
||||
switch (dateConf.getPeriod()) {
|
||||
case DAY:
|
||||
ret.add(dateModeUtils.recentDay(dateDate, dateConf));
|
||||
break;
|
||||
case WEEK:
|
||||
ret.add(dateModeUtils.recentWeek(dateDate, dateConf));
|
||||
break;
|
||||
case MONTH:
|
||||
List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dateDate, dateConf);
|
||||
ret.addAll(rets);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) {
|
||||
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
|
||||
List<Long> metricIds = getMetricIds(queryStructCmd);
|
||||
ItemDateResp dateDate = catalog.getItemDate(
|
||||
new ItemDateFilter(dimensionIds, TypeEnums.DIMENSION.getName()),
|
||||
new ItemDateFilter(metricIds, TypeEnums.METRIC.getName()));
|
||||
return dateDate;
|
||||
}
|
||||
|
||||
public DateConf getDateConfBySql(String sql) {
|
||||
List<FieldExpression> fieldExpressions = SqlParserSelectHelper.getFilterExpression(sql);
|
||||
if (!CollectionUtils.isEmpty(fieldExpressions)) {
|
||||
Set<String> dateList = new HashSet<>();
|
||||
String startDate = "";
|
||||
String endDate = "";
|
||||
String period = "";
|
||||
for (FieldExpression f : fieldExpressions) {
|
||||
if (Objects.isNull(f.getFieldName()) || !internalCols.contains(f.getFieldName().toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
if (Objects.isNull(f.getFieldValue()) || !dateModeUtils.isDateStr(f.getFieldValue().toString())) {
|
||||
continue;
|
||||
}
|
||||
period = dateModeUtils.getPeriodByCol(f.getFieldName().toLowerCase());
|
||||
if ("".equals(period)) {
|
||||
continue;
|
||||
}
|
||||
if ("=".equals(f.getOperator())) {
|
||||
dateList.add(f.getFieldValue().toString());
|
||||
} else if ("<".equals(f.getOperator()) || "<=".equals(f.getOperator())) {
|
||||
if (startDate.isEmpty() || startDate.compareTo(f.getFieldValue().toString()) > 0) {
|
||||
startDate = f.getFieldValue().toString();
|
||||
}
|
||||
} else if (">".equals(f.getOperator()) || ">=".equals(f.getOperator())) {
|
||||
if (endDate.isEmpty() || endDate.compareTo(f.getFieldValue().toString()) < 0) {
|
||||
endDate = f.getFieldValue().toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!"".equals(period)) {
|
||||
DateConf dateConf = new DateConf();
|
||||
dateConf.setPeriod(period);
|
||||
if (!CollectionUtils.isEmpty(dateList)) {
|
||||
dateConf.setDateList(new ArrayList<>(dateList));
|
||||
dateConf.setDateMode(DateMode.LIST);
|
||||
return dateConf;
|
||||
}
|
||||
if (!"".equals(startDate) && !"".equals(endDate)) {
|
||||
dateConf.setStartDate(startDate);
|
||||
dateConf.setEndDate(endDate);
|
||||
dateConf.setDateMode(DateMode.BETWEEN);
|
||||
return dateConf;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<String> getDateCol() {
|
||||
return dateModeUtils.getDateCol();
|
||||
}
|
||||
|
||||
public String getVariablePrefix() {
|
||||
return variablePrefix;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,255 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.utils;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.cache.CacheUtils;
|
||||
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
|
||||
import com.tencent.supersonic.headless.api.enums.SemanticType;
|
||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class QueryUtils {
|
||||
|
||||
private static final String pattern = "\\(`(.*?)`\\)";
|
||||
|
||||
private static final String no_quotation_pattern = "\\((.*?)\\)";
|
||||
|
||||
private final Set<Pattern> patterns = new HashSet<>();
|
||||
@Value("${query.cache.enable:true}")
|
||||
private Boolean cacheEnable;
|
||||
|
||||
@Value("${query.optimizer.enable:true}")
|
||||
private Boolean optimizeEnable;
|
||||
|
||||
private final CacheUtils cacheUtils;
|
||||
private final StatUtils statUtils;
|
||||
|
||||
private final Catalog catalog;
|
||||
|
||||
public QueryUtils(
|
||||
CacheUtils cacheUtils, StatUtils statUtils, Catalog catalog) {
|
||||
|
||||
this.cacheUtils = cacheUtils;
|
||||
this.statUtils = statUtils;
|
||||
this.catalog = catalog;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void fillPattern() {
|
||||
Set<String> aggFunctions = new HashSet<>(Arrays.asList("MAX", "MIN", "SUM", "AVG"));
|
||||
String patternStr = "\\s*(%s\\((.*)\\)) AS";
|
||||
for (String agg : aggFunctions) {
|
||||
patterns.add(Pattern.compile(String.format(patternStr, agg)));
|
||||
}
|
||||
}
|
||||
|
||||
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, List<Long> modelIds) {
|
||||
MetaFilter metaFilter = new MetaFilter(modelIds);
|
||||
List<MetricResp> metricDescList = catalog.getMetrics(metaFilter);
|
||||
List<DimensionResp> dimensionDescList = catalog.getDimensions(metaFilter);
|
||||
Map<String, MetricResp> metricRespMap =
|
||||
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, String> namePair = new HashMap<>();
|
||||
Map<String, String> nameTypePair = new HashMap<>();
|
||||
addSysTimeDimension(namePair, nameTypePair);
|
||||
metricDescList.forEach(metricDesc -> {
|
||||
namePair.put(metricDesc.getBizName(), metricDesc.getName());
|
||||
nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name());
|
||||
});
|
||||
dimensionDescList.forEach(dimensionDesc -> {
|
||||
namePair.put(dimensionDesc.getBizName(), dimensionDesc.getName());
|
||||
nameTypePair.put(dimensionDesc.getBizName(), dimensionDesc.getSemanticType());
|
||||
});
|
||||
List<QueryColumn> columns = queryResultWithColumns.getColumns();
|
||||
columns.forEach(column -> {
|
||||
String nameEn = column.getNameEn().toLowerCase();
|
||||
if (nameEn.contains(JOIN_UNDERLINE)) {
|
||||
nameEn = nameEn.split(JOIN_UNDERLINE)[1];
|
||||
}
|
||||
if (namePair.containsKey(nameEn)) {
|
||||
column.setName(namePair.get(nameEn));
|
||||
} else {
|
||||
String nameEnByRegex = getNameEnByRegex(nameEn, pattern);
|
||||
if (StringUtils.isEmpty(nameEnByRegex)) {
|
||||
nameEnByRegex = getNameEnByRegex(nameEn, no_quotation_pattern);
|
||||
}
|
||||
if (StringUtils.isNotEmpty(nameEnByRegex) && StringUtils.isNotEmpty(namePair.get(nameEnByRegex))) {
|
||||
String filedName = namePair.get(nameEnByRegex);
|
||||
column.setName(nameEn.replaceAll(nameEnByRegex, filedName));
|
||||
}
|
||||
}
|
||||
if (nameTypePair.containsKey(nameEn)) {
|
||||
column.setShowType(nameTypePair.get(nameEn));
|
||||
}
|
||||
if (!nameTypePair.containsKey(nameEn) && isNumberType(column.getType())) {
|
||||
column.setShowType(SemanticType.NUMBER.name());
|
||||
}
|
||||
if (metricRespMap.containsKey(nameEn)) {
|
||||
column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType());
|
||||
column.setDataFormat(metricRespMap.get(nameEn).getDataFormat());
|
||||
}
|
||||
if (StringUtils.isEmpty(column.getShowType())) {
|
||||
column.setShowType(SemanticType.CATEGORY.name());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns,
|
||||
QueryMultiStructReq queryMultiStructCmd) {
|
||||
List<Aggregator> aggregators = queryMultiStructCmd.getQueryStructReqs().stream()
|
||||
.flatMap(queryStructCmd -> queryStructCmd.getAggregators().stream())
|
||||
.collect(Collectors.toList());
|
||||
log.info("multi agg merge:{}", aggregators);
|
||||
Map<String, String> metricNameFromAgg = getMetricNameFromAgg(aggregators);
|
||||
log.info("metricNameFromAgg:{}", metricNameFromAgg);
|
||||
Map<String, String> namePair = new HashMap<>();
|
||||
Map<String, String> nameTypePair = new HashMap<>();
|
||||
addSysTimeDimension(namePair, nameTypePair);
|
||||
namePair.putAll(metricNameFromAgg);
|
||||
List<QueryColumn> columns = queryResultWithColumns.getColumns();
|
||||
columns.forEach(column -> {
|
||||
String nameEn = column.getNameEn().toLowerCase();
|
||||
if (nameEn.contains(JOIN_UNDERLINE)) {
|
||||
nameEn = nameEn.split(JOIN_UNDERLINE)[1];
|
||||
}
|
||||
if (namePair.containsKey(nameEn)) {
|
||||
column.setName(namePair.get(nameEn));
|
||||
} else {
|
||||
if (nameEn.startsWith("name")) {
|
||||
column.setName("名称");
|
||||
} else if (nameEn.startsWith("value")) {
|
||||
column.setName("指标值");
|
||||
}
|
||||
}
|
||||
if (nameTypePair.containsKey(nameEn)) {
|
||||
column.setShowType(nameTypePair.get(nameEn));
|
||||
} else {
|
||||
if (nameEn.startsWith("name")) {
|
||||
column.setShowType("CATEGORY");
|
||||
} else if (nameEn.startsWith("value")) {
|
||||
column.setShowType("NUMBER");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private String getNameEnByRegex(String nameEn, String pattern) {
|
||||
Pattern p = Pattern.compile(pattern);
|
||||
Matcher m = p.matcher(nameEn);
|
||||
|
||||
if (m.find()) {
|
||||
String result = m.group(1);
|
||||
return result;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean isNumberType(String type) {
|
||||
if (StringUtils.isBlank(type)) {
|
||||
return false;
|
||||
}
|
||||
if (type.equalsIgnoreCase("int") || type.equalsIgnoreCase("bigint")
|
||||
|| type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")) {
|
||||
return true;
|
||||
}
|
||||
if (type.toLowerCase().startsWith("uint") || type.toLowerCase().startsWith("int")) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private Map<String, String> getMetricNameFromAgg(List<Aggregator> aggregators) {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
if (CollectionUtils.isEmpty(aggregators)) {
|
||||
return map;
|
||||
}
|
||||
for (int i = 0; i < aggregators.size(); i++) {
|
||||
Aggregator aggregator = aggregators.get(i);
|
||||
if (StringUtils.isBlank(aggregator.getNameCh())) {
|
||||
continue;
|
||||
}
|
||||
map.put("value" + (i + 1), aggregator.getNameCh());
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private static void addSysTimeDimension(Map<String, String> namePair, Map<String, String> nameTypePair) {
|
||||
for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) {
|
||||
namePair.put(timeDimensionEnum.getName(), "date");
|
||||
nameTypePair.put(timeDimensionEnum.getName(), "DATE");
|
||||
}
|
||||
}
|
||||
|
||||
public void checkSqlParse(QueryStatement sqlParser) {
|
||||
if (com.google.common.base.Strings.isNullOrEmpty(sqlParser.getSql())
|
||||
|| com.google.common.base.Strings.isNullOrEmpty(sqlParser.getSourceId())) {
|
||||
throw new RuntimeException("parse Exception: " + sqlParser.getErrMsg());
|
||||
}
|
||||
}
|
||||
|
||||
public QueryStatement sqlParserUnion(QueryMultiStructReq queryMultiStructCmd, List<QueryStatement> sqlParsers) {
|
||||
QueryStatement sqlParser = new QueryStatement();
|
||||
StringBuilder unionSqlBuilder = new StringBuilder();
|
||||
for (int i = 0; i < sqlParsers.size(); i++) {
|
||||
String selectStr = SqlGenerateUtils.getUnionSelect(queryMultiStructCmd.getQueryStructReqs().get(i));
|
||||
unionSqlBuilder.append(String.format("select %s from ( %s ) sub_sql_%s",
|
||||
selectStr,
|
||||
sqlParsers.get(i).getSql(), i));
|
||||
unionSqlBuilder.append(UNIONALL);
|
||||
}
|
||||
String unionSql = unionSqlBuilder.substring(0, unionSqlBuilder.length() - Constants.UNIONALL.length());
|
||||
sqlParser.setSql(unionSql);
|
||||
sqlParser.setSourceId(sqlParsers.get(0).getSourceId());
|
||||
log.info("union sql parser:{}", sqlParser);
|
||||
return sqlParser;
|
||||
}
|
||||
|
||||
public void cacheResultLogic(String key, QueryResultWithSchemaResp queryResultWithColumns) {
|
||||
if (cacheEnable && Objects.nonNull(queryResultWithColumns) && !CollectionUtils.isEmpty(
|
||||
queryResultWithColumns.getResultList())) {
|
||||
QueryResultWithSchemaResp finalQueryResultWithColumns = queryResultWithColumns;
|
||||
CompletableFuture.supplyAsync(() -> cacheUtils.put(key, finalQueryResultWithColumns))
|
||||
.exceptionally(exception -> {
|
||||
log.warn("exception:", exception);
|
||||
return null;
|
||||
});
|
||||
statUtils.updateResultCacheKey(key);
|
||||
log.info("add record to cache, key:{}", key);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public Boolean enableOptimize() {
|
||||
return optimizeEnable;
|
||||
}
|
||||
}
|
||||
@@ -1,21 +1,61 @@
|
||||
package com.tencent.supersonic.headless.core.utils;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.ItemDateResp;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.DateModeUtils;
|
||||
import com.tencent.supersonic.common.util.SqlFilterUtils;
|
||||
import com.tencent.supersonic.common.util.StringUtil;
|
||||
import com.tencent.supersonic.headless.api.enums.EngineType;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.ImmutablePair;
|
||||
import org.apache.commons.lang3.tuple.Triple;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.DAY;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class SqlGenerateUtils {
|
||||
|
||||
private final SqlFilterUtils sqlFilterUtils;
|
||||
|
||||
private final DateModeUtils dateModeUtils;
|
||||
|
||||
@Value("${metricParser.agg.mysql.lowVersion:5.7}")
|
||||
private String mysqlLowVersion;
|
||||
@Value("${metricParser.agg.ck.lowVersion:20.4}")
|
||||
private String ckLowVersion;
|
||||
@Value("${internal.metric.cnt.suffix:internal_cnt}")
|
||||
private String internalMetricNameSuffix;
|
||||
|
||||
public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils,
|
||||
DateModeUtils dateModeUtils) {
|
||||
this.sqlFilterUtils = sqlFilterUtils;
|
||||
this.dateModeUtils = dateModeUtils;
|
||||
}
|
||||
|
||||
public static String getUnionSelect(QueryStructReq queryStructCmd) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int locate = 0;
|
||||
@@ -82,4 +122,112 @@ public class SqlGenerateUtils {
|
||||
.collect(Collectors.joining(","));
|
||||
}
|
||||
|
||||
public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) {
|
||||
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter());
|
||||
String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp);
|
||||
return mergeDateWhereClause(queryStructReq, whereClauseFromFilter, whereFromDate);
|
||||
}
|
||||
|
||||
private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter,
|
||||
String whereFromDate) {
|
||||
if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
|
||||
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
|
||||
} else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
|
||||
return whereClauseFromFilter;
|
||||
} else if (Strings.isNotEmpty(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
|
||||
return whereFromDate;
|
||||
} else if (Objects.isNull(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
|
||||
log.info("the current date information is empty, enter the date initialization logic");
|
||||
return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
|
||||
}
|
||||
return whereClauseFromFilter;
|
||||
}
|
||||
|
||||
private String getDateWhereClause(DateConf dateInfo, ItemDateResp dateDate) {
|
||||
if (Objects.isNull(dateDate)
|
||||
|| Strings.isEmpty(dateDate.getStartDate())
|
||||
&& Strings.isEmpty(dateDate.getEndDate())) {
|
||||
if (dateInfo.getDateMode().equals(DateConf.DateMode.LIST)) {
|
||||
return dateModeUtils.listDateStr(dateDate, dateInfo);
|
||||
}
|
||||
if (dateInfo.getDateMode().equals(DateConf.DateMode.BETWEEN)) {
|
||||
return dateModeUtils.betweenDateStr(dateDate, dateInfo);
|
||||
}
|
||||
if (dateModeUtils.hasAvailableDataMode(dateInfo)) {
|
||||
return dateModeUtils.hasDataModeStr(dateDate, dateInfo);
|
||||
}
|
||||
|
||||
return dateModeUtils.defaultRecentDateInfo(dateInfo);
|
||||
}
|
||||
log.info("dateDate:{}", dateDate);
|
||||
return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
|
||||
}
|
||||
|
||||
public Triple<String, String, String> getBeginEndTime(QueryStructReq queryStructCmd, ItemDateResp dataDate) {
|
||||
if (Objects.isNull(queryStructCmd.getDateInfo())) {
|
||||
return Triple.of("", "", "");
|
||||
}
|
||||
DateConf dateConf = queryStructCmd.getDateInfo();
|
||||
String dateInfo = dateModeUtils.getSysDateCol(dateConf);
|
||||
if (dateInfo.isEmpty()) {
|
||||
return Triple.of("", "", "");
|
||||
}
|
||||
switch (dateConf.getDateMode()) {
|
||||
case AVAILABLE:
|
||||
case BETWEEN:
|
||||
return Triple.of(dateInfo, dateConf.getStartDate(), dateConf.getEndDate());
|
||||
case LIST:
|
||||
return Triple.of(dateInfo, Collections.min(dateConf.getDateList()),
|
||||
Collections.max(dateConf.getDateList()));
|
||||
case RECENT:
|
||||
LocalDate dateMax = LocalDate.now().minusDays(1);
|
||||
LocalDate dateMin = dateMax.minusDays(dateConf.getUnit() - 1);
|
||||
if (Objects.isNull(dataDate)) {
|
||||
return Triple.of(dateInfo, dateMin.format(DateTimeFormatter.ofPattern(DAY_FORMAT)),
|
||||
dateMax.format(DateTimeFormatter.ofPattern(DAY_FORMAT)));
|
||||
}
|
||||
switch (dateConf.getPeriod()) {
|
||||
case DAY:
|
||||
ImmutablePair<String, String> dayInfo = dateModeUtils.recentDay(dataDate, dateConf);
|
||||
return Triple.of(dateInfo, dayInfo.left, dayInfo.right);
|
||||
case WEEK:
|
||||
ImmutablePair<String, String> weekInfo = dateModeUtils.recentWeek(dataDate, dateConf);
|
||||
return Triple.of(dateInfo, weekInfo.left, weekInfo.right);
|
||||
case MONTH:
|
||||
List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dataDate, dateConf);
|
||||
Optional<String> minBegins = rets.stream().map(i -> i.left).sorted().findFirst();
|
||||
Optional<String> maxBegins = rets.stream().map(i -> i.right).sorted(Comparator.reverseOrder())
|
||||
.findFirst();
|
||||
if (minBegins.isPresent() && maxBegins.isPresent()) {
|
||||
return Triple.of(dateInfo, minBegins.get(), maxBegins.get());
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
|
||||
}
|
||||
return Triple.of("", "", "");
|
||||
}
|
||||
|
||||
public boolean isSupportWith(EngineType engineTypeEnum, String version) {
|
||||
if (engineTypeEnum.equals(EngineType.MYSQL) && Objects.nonNull(version) && version.startsWith(
|
||||
mysqlLowVersion)) {
|
||||
return false;
|
||||
}
|
||||
if (engineTypeEnum.equals(EngineType.CLICKHOUSE) && Objects.nonNull(version)
|
||||
&& StringUtil.compareVersion(version,
|
||||
ckLowVersion) < 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public String generateInternalMetricName(String modelBizName) {
|
||||
return modelBizName + UNDERLINE + internalMetricNameSuffix;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
package com.tencent.supersonic.headless.core.utils;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.headless.api.enums.DataType;
|
||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
|
||||
import java.rmi.ServerException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import javax.sql.DataSource;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class SqlUtils {
|
||||
|
||||
@Getter
|
||||
private DatabaseResp databaseResp;
|
||||
|
||||
@Autowired
|
||||
private JdbcDataSource jdbcDataSource;
|
||||
|
||||
@Value("${source.result-limit:1000000}")
|
||||
private int resultLimit;
|
||||
|
||||
@Value("${source.enable-query-log:false}")
|
||||
private boolean isQueryLogEnable;
|
||||
|
||||
@Getter
|
||||
private DataType dataTypeEnum;
|
||||
|
||||
@Getter
|
||||
private JdbcDataSourceUtils jdbcDataSourceUtils;
|
||||
|
||||
public SqlUtils() {
|
||||
|
||||
}
|
||||
|
||||
public SqlUtils(DatabaseResp databaseResp) {
|
||||
this.databaseResp = databaseResp;
|
||||
this.dataTypeEnum = DataType.urlOf(databaseResp.getUrl());
|
||||
}
|
||||
|
||||
public SqlUtils init(DatabaseResp databaseResp) {
|
||||
//todo Password decryption
|
||||
return SqlUtilsBuilder
|
||||
.getBuilder()
|
||||
.withName(databaseResp.getId() + AT_SYMBOL + databaseResp.getName())
|
||||
.withType(databaseResp.getType())
|
||||
.withJdbcUrl(databaseResp.getUrl())
|
||||
.withUsername(databaseResp.getUsername())
|
||||
.withPassword(databaseResp.getPassword())
|
||||
.withJdbcDataSource(this.jdbcDataSource)
|
||||
.withResultLimit(this.resultLimit)
|
||||
.withIsQueryLogEnable(this.isQueryLogEnable)
|
||||
.build();
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> execute(String sql) throws ServerException {
|
||||
try {
|
||||
List<Map<String, Object>> list = jdbcTemplate().queryForList(sql);
|
||||
log.info("list:{}", list);
|
||||
return list;
|
||||
} catch (Exception e) {
|
||||
log.error(e.toString(), e);
|
||||
throw new ServerException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void execute(String sql, QueryResultWithSchemaResp queryResultWithColumns) {
|
||||
getResult(sql, queryResultWithColumns, jdbcTemplate());
|
||||
}
|
||||
|
||||
public JdbcTemplate jdbcTemplate() throws RuntimeException {
|
||||
Connection connection = null;
|
||||
try {
|
||||
connection = jdbcDataSourceUtils.getConnection(databaseResp);
|
||||
} catch (Exception e) {
|
||||
log.warn("e:", e);
|
||||
} finally {
|
||||
JdbcDataSourceUtils.releaseConnection(connection);
|
||||
}
|
||||
DataSource dataSource = jdbcDataSourceUtils.getDataSource(databaseResp);
|
||||
JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
|
||||
jdbcTemplate.setDatabaseProductName(databaseResp.getName());
|
||||
jdbcTemplate.setFetchSize(500);
|
||||
log.info("jdbcTemplate:{}, dataSource:{}", jdbcTemplate, dataSource);
|
||||
return jdbcTemplate;
|
||||
}
|
||||
|
||||
public void queryInternal(String sql, QueryResultWithSchemaResp queryResultWithColumns) {
|
||||
getResult(sql, queryResultWithColumns, jdbcTemplate());
|
||||
}
|
||||
|
||||
private QueryResultWithSchemaResp getResult(String sql, QueryResultWithSchemaResp queryResultWithColumns,
|
||||
JdbcTemplate jdbcTemplate) {
|
||||
jdbcTemplate.query(sql, rs -> {
|
||||
if (null == rs) {
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
ResultSetMetaData metaData = rs.getMetaData();
|
||||
List<QueryColumn> queryColumns = new ArrayList<>();
|
||||
for (int i = 1; i <= metaData.getColumnCount(); i++) {
|
||||
String key = metaData.getColumnLabel(i);
|
||||
queryColumns.add(new QueryColumn(key, metaData.getColumnTypeName(i)));
|
||||
}
|
||||
queryResultWithColumns.setColumns(queryColumns);
|
||||
|
||||
List<Map<String, Object>> resultList = getAllData(rs, queryColumns);
|
||||
queryResultWithColumns.setResultList(resultList);
|
||||
return queryResultWithColumns;
|
||||
});
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> getAllData(ResultSet rs, List<QueryColumn> queryColumns) {
|
||||
List<Map<String, Object>> data = new ArrayList<>();
|
||||
try {
|
||||
while (rs.next()) {
|
||||
data.add(getLineData(rs, queryColumns));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("error in getAllData, e:", e);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
private Map<String, Object> getLineData(ResultSet rs, List<QueryColumn> queryColumns) throws SQLException {
|
||||
Map<String, Object> map = new LinkedHashMap<>();
|
||||
for (QueryColumn queryColumn : queryColumns) {
|
||||
String colName = queryColumn.getNameEn();
|
||||
Object value = rs.getObject(colName);
|
||||
map.put(colName, getValue(value));
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private Object getValue(Object value) {
|
||||
if (value instanceof LocalDate) {
|
||||
LocalDate localDate = (LocalDate) value;
|
||||
return localDate.format(DateTimeFormatter.ofPattern(DateUtils.DATE_FORMAT));
|
||||
} else if (value instanceof LocalDateTime) {
|
||||
LocalDateTime localDateTime = (LocalDateTime) value;
|
||||
return localDateTime.format(DateTimeFormatter.ofPattern(DateUtils.TIME_FORMAT));
|
||||
} else if (value instanceof Date) {
|
||||
Date date = (Date) value;
|
||||
return DateUtils.format(date);
|
||||
} else if (value instanceof byte[]) {
|
||||
return new String((byte[]) value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
public static final class SqlUtilsBuilder {
|
||||
|
||||
private JdbcDataSource jdbcDataSource;
|
||||
private int resultLimit;
|
||||
private boolean isQueryLogEnable;
|
||||
private String name;
|
||||
private String type;
|
||||
private String jdbcUrl;
|
||||
private String username;
|
||||
private String password;
|
||||
|
||||
private SqlUtilsBuilder() {
|
||||
|
||||
}
|
||||
|
||||
public static SqlUtilsBuilder getBuilder() {
|
||||
return new SqlUtilsBuilder();
|
||||
}
|
||||
|
||||
SqlUtilsBuilder withJdbcDataSource(JdbcDataSource jdbcDataSource) {
|
||||
this.jdbcDataSource = jdbcDataSource;
|
||||
return this;
|
||||
}
|
||||
|
||||
SqlUtilsBuilder withResultLimit(int resultLimit) {
|
||||
this.resultLimit = resultLimit;
|
||||
return this;
|
||||
}
|
||||
|
||||
SqlUtilsBuilder withIsQueryLogEnable(boolean isQueryLogEnable) {
|
||||
this.isQueryLogEnable = isQueryLogEnable;
|
||||
return this;
|
||||
}
|
||||
|
||||
SqlUtilsBuilder withName(String name) {
|
||||
this.name = name;
|
||||
return this;
|
||||
}
|
||||
|
||||
SqlUtilsBuilder withType(String type) {
|
||||
this.type = type;
|
||||
return this;
|
||||
}
|
||||
|
||||
SqlUtilsBuilder withJdbcUrl(String jdbcUrl) {
|
||||
this.jdbcUrl = jdbcUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
SqlUtilsBuilder withUsername(String username) {
|
||||
this.username = username;
|
||||
return this;
|
||||
}
|
||||
|
||||
SqlUtilsBuilder withPassword(String password) {
|
||||
this.password = password;
|
||||
return this;
|
||||
}
|
||||
|
||||
public SqlUtils build() {
|
||||
DatabaseResp databaseResp = DatabaseResp.builder()
|
||||
.name(this.name)
|
||||
.type(this.type)
|
||||
.url(this.jdbcUrl)
|
||||
.username(this.username)
|
||||
.password(this.password)
|
||||
.build();
|
||||
|
||||
SqlUtils sqlUtils = new SqlUtils(databaseResp);
|
||||
sqlUtils.jdbcDataSource = this.jdbcDataSource;
|
||||
sqlUtils.resultLimit = this.resultLimit;
|
||||
sqlUtils.isQueryLogEnable = this.isQueryLogEnable;
|
||||
sqlUtils.jdbcDataSourceUtils = new JdbcDataSourceUtils(this.jdbcDataSource);
|
||||
|
||||
return sqlUtils;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
package com.tencent.supersonic.headless.core.utils;
|
||||
|
||||
import com.alibaba.ttl.TransmittableThreadLocal;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.common.util.SqlFilterUtils;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||
import com.tencent.supersonic.headless.api.request.ItemUseReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.ItemUseResp;
|
||||
import com.tencent.supersonic.headless.api.enums.QueryOptMode;
|
||||
import com.tencent.supersonic.headless.api.enums.QueryTypeBack;
|
||||
import com.tencent.supersonic.headless.api.enums.QueryType;
|
||||
import com.tencent.supersonic.headless.api.pojo.QueryStat;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.persistence.repository.StatRepository;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.logging.log4j.util.Strings;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class StatUtils {
|
||||
|
||||
private static final TransmittableThreadLocal<QueryStat> STATS = new TransmittableThreadLocal<>();
|
||||
private final StatRepository statRepository;
|
||||
private final SqlFilterUtils sqlFilterUtils;
|
||||
|
||||
private final ModelService modelService;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
public StatUtils(StatRepository statRepository,
|
||||
SqlFilterUtils sqlFilterUtils,
|
||||
ModelService modelService) {
|
||||
|
||||
this.statRepository = statRepository;
|
||||
this.sqlFilterUtils = sqlFilterUtils;
|
||||
this.modelService = modelService;
|
||||
}
|
||||
|
||||
public static QueryStat get() {
|
||||
return STATS.get();
|
||||
}
|
||||
|
||||
public static void set(QueryStat queryStatInfo) {
|
||||
STATS.set(queryStatInfo);
|
||||
}
|
||||
|
||||
public static void remove() {
|
||||
STATS.remove();
|
||||
}
|
||||
|
||||
public void statInfo2DbAsync(TaskStatusEnum state) {
|
||||
QueryStat queryStatInfo = get();
|
||||
queryStatInfo.setElapsedMs(System.currentTimeMillis() - queryStatInfo.getStartTime());
|
||||
queryStatInfo.setQueryState(state.getStatus());
|
||||
log.info("queryStatInfo: {}", queryStatInfo);
|
||||
CompletableFuture.runAsync(() -> {
|
||||
statRepository.createRecord(queryStatInfo);
|
||||
}).exceptionally(exception -> {
|
||||
log.warn("queryStatInfo, exception:", exception);
|
||||
return null;
|
||||
});
|
||||
|
||||
remove();
|
||||
}
|
||||
|
||||
public Boolean updateResultCacheKey(String key) {
|
||||
STATS.get().setResultCacheKey(key);
|
||||
return true;
|
||||
}
|
||||
|
||||
public void initStatInfo(QueryS2SQLReq queryS2SQLReq, User facadeUser) {
|
||||
QueryStat queryStatInfo = new QueryStat();
|
||||
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
|
||||
queryStatInfo.setModelId(queryS2SQLReq.getModelIds().get(0));
|
||||
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelIds().get(0));
|
||||
|
||||
List<String> dimensions = new ArrayList<>();
|
||||
List<String> metrics = new ArrayList<>();
|
||||
if (Objects.nonNull(modelSchemaResp)) {
|
||||
dimensions = getFieldNames(allFields, modelSchemaResp.getDimensions());
|
||||
metrics = getFieldNames(allFields, modelSchemaResp.getMetrics());
|
||||
}
|
||||
|
||||
String userName = getUserName(facadeUser);
|
||||
try {
|
||||
queryStatInfo.setTraceId("")
|
||||
.setModelId(queryS2SQLReq.getModelIds().get(0))
|
||||
.setUser(userName)
|
||||
.setQueryType(QueryType.SQL.getValue())
|
||||
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
|
||||
.setQuerySqlCmd(queryS2SQLReq.toString())
|
||||
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2SQLReq.toString()))
|
||||
.setStartTime(System.currentTimeMillis())
|
||||
.setUseResultCache(true)
|
||||
.setUseSqlCache(true)
|
||||
.setMetrics(objectMapper.writeValueAsString(metrics))
|
||||
.setDimensions(objectMapper.writeValueAsString(dimensions));
|
||||
} catch (JsonProcessingException e) {
|
||||
log.error("initStatInfo:{}", e);
|
||||
}
|
||||
StatUtils.set(queryStatInfo);
|
||||
}
|
||||
|
||||
public void initStatInfo(QueryStructReq queryStructCmd, User facadeUser) {
|
||||
QueryStat queryStatInfo = new QueryStat();
|
||||
String traceId = "";
|
||||
List<String> dimensions = queryStructCmd.getGroups();
|
||||
|
||||
List<String> metrics = new ArrayList<>();
|
||||
queryStructCmd.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn()));
|
||||
String user = getUserName(facadeUser);
|
||||
|
||||
try {
|
||||
queryStatInfo.setTraceId(traceId)
|
||||
.setModelId(1L)
|
||||
.setUser(user)
|
||||
.setQueryType(QueryType.STRUCT.getValue())
|
||||
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
|
||||
.setQueryStructCmd(queryStructCmd.toString())
|
||||
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructCmd.toString()))
|
||||
.setStartTime(System.currentTimeMillis())
|
||||
.setNativeQuery(queryStructCmd.getQueryType().isNativeAggQuery())
|
||||
.setGroupByCols(objectMapper.writeValueAsString(queryStructCmd.getGroups()))
|
||||
.setAggCols(objectMapper.writeValueAsString(queryStructCmd.getAggregators()))
|
||||
.setOrderByCols(objectMapper.writeValueAsString(queryStructCmd.getOrders()))
|
||||
.setFilterCols(objectMapper.writeValueAsString(
|
||||
sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter())))
|
||||
.setUseResultCache(true)
|
||||
.setUseSqlCache(true)
|
||||
.setMetrics(objectMapper.writeValueAsString(metrics))
|
||||
.setDimensions(objectMapper.writeValueAsString(dimensions))
|
||||
.setQueryOptMode(QueryOptMode.NONE.name());
|
||||
} catch (JsonProcessingException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
StatUtils.set(queryStatInfo);
|
||||
|
||||
}
|
||||
|
||||
private List<String> getFieldNames(List<String> allFields, List<? extends SchemaItem> schemaItems) {
|
||||
Set<String> fieldNames = schemaItems
|
||||
.stream()
|
||||
.map(dimSchemaResp -> dimSchemaResp.getBizName())
|
||||
.collect(Collectors.toSet());
|
||||
if (!CollectionUtils.isEmpty(fieldNames)) {
|
||||
return allFields.stream().filter(fieldName -> fieldNames.contains(fieldName))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
private String getUserName(User facadeUser) {
|
||||
return (Objects.nonNull(facadeUser) && Strings.isNotEmpty(facadeUser.getName())) ? facadeUser.getName()
|
||||
: "Admin";
|
||||
}
|
||||
|
||||
public Boolean updateQueryOptMode(String mode) {
|
||||
STATS.get().setQueryOptMode(mode);
|
||||
return true;
|
||||
}
|
||||
|
||||
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend) {
|
||||
return statRepository.getStatInfo(itemUseCommend);
|
||||
}
|
||||
|
||||
public List<QueryStat> getQueryStatInfoWithoutCache(ItemUseReq itemUseCommend) {
|
||||
return statRepository.getQueryStatInfoWithoutCache(itemUseCommend);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package com.tencent.supersonic.headless.core.utils;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.headless.api.enums.DimensionType;
|
||||
import com.tencent.supersonic.headless.api.pojo.Dim;
|
||||
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
|
||||
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
public class SysTimeDimensionBuilder {
|
||||
|
||||
public static void addSysTimeDimension(List<Dim> dims, DbAdaptor engineAdaptor) {
|
||||
log.info("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor);
|
||||
Dim timeDim = getTimeDim(dims);
|
||||
if (timeDim == null) {
|
||||
timeDim = Dim.getDefault();
|
||||
//todo not find the time dimension
|
||||
return;
|
||||
}
|
||||
dims.add(generateSysDayDimension(timeDim, engineAdaptor));
|
||||
dims.add(generateSysWeekDimension(timeDim, engineAdaptor));
|
||||
dims.add(generateSysMonthDimension(timeDim, engineAdaptor));
|
||||
log.debug("addSysTimeDimension after:{}, engineAdaptor:{}", dims, engineAdaptor);
|
||||
}
|
||||
|
||||
private static Dim generateSysDayDimension(Dim timeDim, DbAdaptor engineAdaptor) {
|
||||
Dim dim = new Dim();
|
||||
dim.setBizName(TimeDimensionEnum.DAY.getName());
|
||||
dim.setType(DimensionType.time.name());
|
||||
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(), engineAdaptor));
|
||||
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
|
||||
typeParams.setTimeGranularity(TimeDimensionEnum.DAY.name().toLowerCase());
|
||||
typeParams.setIsPrimary("true");
|
||||
dim.setTypeParams(typeParams);
|
||||
return dim;
|
||||
}
|
||||
|
||||
private static Dim generateSysWeekDimension(Dim timeDim, DbAdaptor engineAdaptor) {
|
||||
Dim dim = new Dim();
|
||||
dim.setBizName(TimeDimensionEnum.WEEK.getName());
|
||||
dim.setType(DimensionType.time.name());
|
||||
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(), engineAdaptor));
|
||||
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
|
||||
typeParams.setTimeGranularity(TimeDimensionEnum.WEEK.name().toLowerCase());
|
||||
typeParams.setIsPrimary("false");
|
||||
dim.setTypeParams(typeParams);
|
||||
return dim;
|
||||
}
|
||||
|
||||
private static Dim generateSysMonthDimension(Dim timeDim, DbAdaptor engineAdaptor) {
|
||||
Dim dim = new Dim();
|
||||
dim.setBizName(TimeDimensionEnum.MONTH.getName());
|
||||
dim.setType(DimensionType.time.name());
|
||||
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(), engineAdaptor));
|
||||
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
|
||||
typeParams.setTimeGranularity(TimeDimensionEnum.MONTH.name().toLowerCase());
|
||||
typeParams.setIsPrimary("false");
|
||||
dim.setTypeParams(typeParams);
|
||||
return dim;
|
||||
}
|
||||
|
||||
private static String generateTimeExpr(Dim timeDim, String dateType, DbAdaptor engineAdaptor) {
|
||||
String bizName = timeDim.getBizName();
|
||||
String dateFormat = timeDim.getDateFormat();
|
||||
return engineAdaptor.getDateFormat(dateType, dateFormat, bizName);
|
||||
}
|
||||
|
||||
private static Dim getTimeDim(List<Dim> timeDims) {
|
||||
for (Dim dim : timeDims) {
|
||||
if (dim.getType().equalsIgnoreCase(DimensionType.time.name())) {
|
||||
return dim;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
|
||||
<mapper namespace="com.tencent.supersonic.headless.core.persistence.mapper.StatMapper">
|
||||
|
||||
<resultMap id="QueryStatDO"
|
||||
type="com.tencent.supersonic.headless.api.pojo.QueryStat">
|
||||
<id column="id" property="id"/>
|
||||
<result column="trace_id" property="traceId"/>
|
||||
<result column="model_id" property="modelId"/>
|
||||
<result column="user" property="user"/>
|
||||
<result column="created_at" property="createdAt"/>
|
||||
<result column="query_type" property="queryType"/>
|
||||
<result column="query_type_back" property="queryTypeBack"/>
|
||||
<result column="query_sql_cmd" property="querySqlCmd"/>
|
||||
<result column="sql_cmd_md5" property="querySqlCmdMd5"/>
|
||||
<result column="query_struct_cmd" property="queryStructCmd"/>
|
||||
<result column="struct_cmd_md5" property="queryStructCmdMd5"/>
|
||||
<result column="sql" property="sql"/>
|
||||
<result column="sql_md5" property="sqlMd5"/>
|
||||
<result column="query_engine" property="queryEngine"/>
|
||||
<result column="elapsed_ms" property="elapsedMs"/>
|
||||
<result column="query_state" property="queryState"/>
|
||||
<result column="native_query" property="nativeQuery"/>
|
||||
<result column="start_date" property="startDate"/>
|
||||
<result column="end_date" property="endDate"/>
|
||||
<result column="dimensions" property="dimensions"/>
|
||||
<result column="metrics" property="metrics"/>
|
||||
<result column="select_cols" property="selectCols"/>
|
||||
<result column="agg_cols" property="aggCols"/>
|
||||
<result column="filter_cols" property="filterCols"/>
|
||||
<result column="group_by_cols" property="groupByCols"/>
|
||||
<result column="order_by_cols" property="orderByCols"/>
|
||||
<result column="use_result_cache" property="useResultCache"/>
|
||||
<result column="use_sql_cache" property="useSqlCache"/>
|
||||
<result column="sql_cache_key" property="sqlCacheKey"/>
|
||||
<result column="result_cache_key" property="resultCacheKey"/>
|
||||
<result column="query_opt_mode" property="queryOptMode"/>
|
||||
</resultMap>
|
||||
|
||||
<insert id="createRecord">
|
||||
insert into s2_query_stat_info
|
||||
(
|
||||
trace_id, model_id, `user`, query_type, query_type_back, query_sql_cmd, sql_cmd_md5, query_struct_cmd, struct_cmd_md5, `sql`, sql_md5, query_engine,
|
||||
elapsed_ms, query_state, native_query, start_date, end_date, dimensions, metrics, select_cols, agg_cols, filter_cols, group_by_cols,
|
||||
order_by_cols, use_result_cache, use_sql_cache, sql_cache_key, result_cache_key, query_opt_mode
|
||||
)
|
||||
values
|
||||
(
|
||||
#{traceId}, #{modelId}, #{user}, #{queryType}, #{queryTypeBack}, #{querySqlCmd}, #{querySqlCmdMd5}, #{queryStructCmd}, #{queryStructCmdMd5}, #{sql}, #{sqlMd5}, #{queryEngine},
|
||||
#{elapsedMs}, #{queryState}, #{nativeQuery}, #{startDate}, #{endDate}, #{dimensions}, #{metrics}, #{selectCols}, #{aggCols}, #{filterCols}, #{groupByCols},
|
||||
#{orderByCols}, #{useResultCache}, #{useSqlCache}, #{sqlCacheKey}, #{resultCacheKey}, #{queryOptMode}
|
||||
)
|
||||
</insert>
|
||||
|
||||
<select id="getStatInfo"
|
||||
resultType="com.tencent.supersonic.headless.api.pojo.QueryStat">
|
||||
select *
|
||||
from s2_query_stat_info
|
||||
<where>
|
||||
<if test="startTime != null">
|
||||
and start_time >= #{startTime}
|
||||
</if>
|
||||
<if test="modelId != null">
|
||||
and model_id = #{modelId}
|
||||
</if>
|
||||
<if test="modelIds != null and modelIds.size() > 0">
|
||||
and model_id in
|
||||
<foreach item="id" collection="modelIds" open="(" separator="," close=")">
|
||||
#{id}
|
||||
</foreach>
|
||||
</if>
|
||||
<if test="metric != null">
|
||||
and metrics like concat('%',#{metric},'%')
|
||||
</if>
|
||||
</where>
|
||||
|
||||
</select>
|
||||
|
||||
</mapper>
|
||||
@@ -1,255 +0,0 @@
|
||||
package com.tencent.supersonic.headless.query.domain.calcite;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.ColumnOrder;
|
||||
import com.tencent.supersonic.headless.api.response.SqlParserResp;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
|
||||
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
|
||||
import com.tencent.supersonic.headless.api.enums.AggOption;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.HeadlessSchemaManager;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.planner.AggPlanner;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
class HeadlessParserServiceTest {
|
||||
|
||||
private static Map<String, HeadlessSchema> headlessSchemaMap = new HashMap<>();
|
||||
|
||||
public static SqlParserResp parser(HeadlessSchema headlessSchema, MetricQueryReq metricCommand, boolean isAgg) {
|
||||
SqlParserResp sqlParser = new SqlParserResp();
|
||||
if (metricCommand.getRootPath().isEmpty()) {
|
||||
sqlParser.setErrMsg("rootPath empty");
|
||||
return sqlParser;
|
||||
}
|
||||
try {
|
||||
if (headlessSchema == null) {
|
||||
sqlParser.setErrMsg("headlessSchema not found");
|
||||
return sqlParser;
|
||||
}
|
||||
AggPlanner aggBuilder = new AggPlanner(headlessSchema);
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setMetricReq(metricCommand);
|
||||
aggBuilder.explain(queryStatement, AggOption.getAggregation(!isAgg));
|
||||
sqlParser.setSql(aggBuilder.getSql());
|
||||
sqlParser.setSourceId(aggBuilder.getSourceId());
|
||||
} catch (Exception e) {
|
||||
sqlParser.setErrMsg(e.getMessage());
|
||||
log.error("parser error MetricCommand[{}] error [{}]", metricCommand, e);
|
||||
}
|
||||
return sqlParser;
|
||||
}
|
||||
|
||||
public void test() throws Exception {
|
||||
|
||||
DataModelYamlTpl datasource = new DataModelYamlTpl();
|
||||
datasource.setName("s2_pv_uv_statis");
|
||||
datasource.setSourceId(1L);
|
||||
datasource.setSqlQuery("SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis");
|
||||
|
||||
MeasureYamlTpl measure = new MeasureYamlTpl();
|
||||
measure.setAgg("sum");
|
||||
measure.setName("s2_pv_uv_statis_pv");
|
||||
measure.setExpr("pv");
|
||||
List<MeasureYamlTpl> measures = new ArrayList<>();
|
||||
measures.add(measure);
|
||||
|
||||
MeasureYamlTpl measure2 = new MeasureYamlTpl();
|
||||
measure2.setAgg("count");
|
||||
measure2.setName("s2_pv_uv_statis_internal_cnt");
|
||||
measure2.setExpr("1");
|
||||
measure2.setCreateMetric("true");
|
||||
measures.add(measure2);
|
||||
|
||||
MeasureYamlTpl measure3 = new MeasureYamlTpl();
|
||||
measure3.setAgg("count");
|
||||
measure3.setName("s2_pv_uv_statis_uv");
|
||||
measure3.setExpr("uv");
|
||||
measure3.setCreateMetric("true");
|
||||
measures.add(measure3);
|
||||
|
||||
datasource.setMeasures(measures);
|
||||
|
||||
DimensionYamlTpl dimension = new DimensionYamlTpl();
|
||||
dimension.setName("imp_date");
|
||||
dimension.setExpr("imp_date");
|
||||
dimension.setType("time");
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl();
|
||||
dimensionTimeTypeParams.setIsPrimary("true");
|
||||
dimensionTimeTypeParams.setTimeGranularity("day");
|
||||
dimension.setTypeParams(dimensionTimeTypeParams);
|
||||
List<DimensionYamlTpl> dimensions = new ArrayList<>();
|
||||
dimensions.add(dimension);
|
||||
|
||||
DimensionYamlTpl dimension2 = new DimensionYamlTpl();
|
||||
dimension2.setName("sys_imp_date");
|
||||
dimension2.setExpr("imp_date");
|
||||
dimension2.setType("time");
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParams2 = new DimensionTimeTypeParamsTpl();
|
||||
dimensionTimeTypeParams2.setIsPrimary("true");
|
||||
dimensionTimeTypeParams2.setTimeGranularity("day");
|
||||
dimension2.setTypeParams(dimensionTimeTypeParams2);
|
||||
dimensions.add(dimension2);
|
||||
|
||||
DimensionYamlTpl dimension3 = new DimensionYamlTpl();
|
||||
dimension3.setName("sys_imp_week");
|
||||
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
|
||||
dimension3.setType("time");
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl();
|
||||
dimensionTimeTypeParams3.setIsPrimary("true");
|
||||
dimensionTimeTypeParams3.setTimeGranularity("day");
|
||||
dimension3.setTypeParams(dimensionTimeTypeParams3);
|
||||
dimensions.add(dimension3);
|
||||
|
||||
datasource.setDimensions(dimensions);
|
||||
|
||||
List<IdentifyYamlTpl> identifies = new ArrayList<>();
|
||||
IdentifyYamlTpl identify = new IdentifyYamlTpl();
|
||||
identify.setName("user_name");
|
||||
identify.setType("primary");
|
||||
identifies.add(identify);
|
||||
datasource.setIdentifiers(identifies);
|
||||
|
||||
HeadlessSchema headlessSchema = HeadlessSchema.newBuilder("s2").build();
|
||||
|
||||
HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getDatasource(datasource));
|
||||
|
||||
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
|
||||
dimension1.setExpr("page");
|
||||
dimension1.setName("page");
|
||||
dimension1.setType("categorical");
|
||||
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
|
||||
dimensionYamlTpls.add(dimension1);
|
||||
|
||||
HeadlessSchemaManager.update(headlessSchema, "s2_pv_uv_statis",
|
||||
HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
|
||||
|
||||
MetricYamlTpl metric1 = new MetricYamlTpl();
|
||||
metric1.setName("pv");
|
||||
metric1.setType("expr");
|
||||
MetricTypeParamsYamlTpl metricTypeParams = new MetricTypeParamsYamlTpl();
|
||||
List<MeasureYamlTpl> measures1 = new ArrayList<>();
|
||||
MeasureYamlTpl measure1 = new MeasureYamlTpl();
|
||||
measure1.setName("s2_pv_uv_statis_pv");
|
||||
measures1.add(measure1);
|
||||
metricTypeParams.setMeasures(measures1);
|
||||
metricTypeParams.setExpr("s2_pv_uv_statis_pv");
|
||||
metric1.setTypeParams(metricTypeParams);
|
||||
List<MetricYamlTpl> metric = new ArrayList<>();
|
||||
metric.add(metric1);
|
||||
|
||||
MetricYamlTpl metric2 = new MetricYamlTpl();
|
||||
metric2.setName("uv");
|
||||
metric2.setType("expr");
|
||||
MetricTypeParamsYamlTpl metricTypeParams1 = new MetricTypeParamsYamlTpl();
|
||||
List<MeasureYamlTpl> measures2 = new ArrayList<>();
|
||||
MeasureYamlTpl measure4 = new MeasureYamlTpl();
|
||||
measure4.setName("s2_pv_uv_statis_uv");
|
||||
measures2.add(measure4);
|
||||
metricTypeParams1.setMeasures(measures2);
|
||||
metricTypeParams1.setExpr("s2_pv_uv_statis_uv");
|
||||
metric2.setTypeParams(metricTypeParams1);
|
||||
metric.add(metric2);
|
||||
|
||||
HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric));
|
||||
|
||||
MetricQueryReq metricCommand = new MetricQueryReq();
|
||||
metricCommand.setRootPath("s2");
|
||||
metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date")));
|
||||
metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv")));
|
||||
metricCommand.setWhere("user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
|
||||
metricCommand.setLimit(1000L);
|
||||
List<ColumnOrder> orders = new ArrayList<>();
|
||||
orders.add(ColumnOrder.buildDesc("sys_imp_date"));
|
||||
metricCommand.setOrder(orders);
|
||||
System.out.println(parser(headlessSchema, metricCommand, true));
|
||||
|
||||
addDepartment(headlessSchema);
|
||||
|
||||
MetricQueryReq metricCommand2 = new MetricQueryReq();
|
||||
metricCommand2.setRootPath("s2");
|
||||
metricCommand2.setDimensions(new ArrayList<>(
|
||||
Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page")));
|
||||
metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv")));
|
||||
metricCommand2.setWhere(
|
||||
"user_name = 'ab' and (sys_imp_date >= '2023-02-28' and sys_imp_date <= '2023-05-28') ");
|
||||
metricCommand2.setLimit(1000L);
|
||||
List<ColumnOrder> orders2 = new ArrayList<>();
|
||||
orders2.add(ColumnOrder.buildDesc("sys_imp_date"));
|
||||
metricCommand2.setOrder(orders2);
|
||||
System.out.println(parser(headlessSchema, metricCommand2, true));
|
||||
|
||||
|
||||
}
|
||||
|
||||
private static void addDepartment(HeadlessSchema headlessSchema) {
|
||||
DataModelYamlTpl datasource = new DataModelYamlTpl();
|
||||
datasource.setName("user_department");
|
||||
datasource.setSourceId(1L);
|
||||
datasource.setSqlQuery("SELECT imp_date,user_name,department FROM s2_user_department");
|
||||
|
||||
MeasureYamlTpl measure = new MeasureYamlTpl();
|
||||
measure.setAgg("count");
|
||||
measure.setName("user_department_internal_cnt");
|
||||
measure.setCreateMetric("true");
|
||||
measure.setExpr("1");
|
||||
List<MeasureYamlTpl> measures = new ArrayList<>();
|
||||
measures.add(measure);
|
||||
|
||||
datasource.setMeasures(measures);
|
||||
|
||||
DimensionYamlTpl dimension = new DimensionYamlTpl();
|
||||
dimension.setName("sys_imp_date");
|
||||
dimension.setExpr("imp_date");
|
||||
dimension.setType("time");
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl();
|
||||
dimensionTimeTypeParams.setIsPrimary("true");
|
||||
dimensionTimeTypeParams.setTimeGranularity("day");
|
||||
dimension.setTypeParams(dimensionTimeTypeParams);
|
||||
List<DimensionYamlTpl> dimensions = new ArrayList<>();
|
||||
dimensions.add(dimension);
|
||||
|
||||
DimensionYamlTpl dimension3 = new DimensionYamlTpl();
|
||||
dimension3.setName("sys_imp_week");
|
||||
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
|
||||
dimension3.setType("time");
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl();
|
||||
dimensionTimeTypeParams3.setIsPrimary("true");
|
||||
dimensionTimeTypeParams3.setTimeGranularity("week");
|
||||
dimension3.setTypeParams(dimensionTimeTypeParams3);
|
||||
dimensions.add(dimension3);
|
||||
|
||||
datasource.setDimensions(dimensions);
|
||||
|
||||
List<IdentifyYamlTpl> identifies = new ArrayList<>();
|
||||
IdentifyYamlTpl identify = new IdentifyYamlTpl();
|
||||
identify.setName("user_name");
|
||||
identify.setType("primary");
|
||||
identifies.add(identify);
|
||||
datasource.setIdentifiers(identifies);
|
||||
|
||||
headlessSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource));
|
||||
|
||||
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
|
||||
dimension1.setExpr("department");
|
||||
dimension1.setName("department");
|
||||
dimension1.setType("categorical");
|
||||
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
|
||||
dimensionYamlTpls.add(dimension1);
|
||||
|
||||
headlessSchema.getDimension()
|
||||
.put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package com.tencent.supersonic.headless.query.domain.calcite;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.core.persistence.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
|
||||
import com.tencent.supersonic.headless.core.parser.QueryParser;
|
||||
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.headless.core.utils.QueryUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
@Slf4j
|
||||
public class MaterializationQueryTest {
|
||||
|
||||
private final QueryParser queryParser;
|
||||
private final QueryUtils queryUtils;
|
||||
|
||||
public MaterializationQueryTest(QueryParser queryParser,
|
||||
QueryUtils queryUtils) {
|
||||
this.queryParser = queryParser;
|
||||
this.queryUtils = queryUtils;
|
||||
}
|
||||
|
||||
public void test() {
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setModelId(1L);
|
||||
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setFunc(AggOperatorEnum.UNKNOWN);
|
||||
aggregator.setColumn("pv");
|
||||
queryStructReq.setAggregators(Arrays.asList(aggregator));
|
||||
|
||||
queryStructReq.setGroups(Arrays.asList("department"));
|
||||
DateConf dateConf = new DateConf();
|
||||
dateConf.setDateMode(DateMode.LIST);
|
||||
dateConf.setDateList(Arrays.asList("2023-08-01"));
|
||||
queryStructReq.setDateInfo(dateConf);
|
||||
|
||||
try {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq(queryStructReq);
|
||||
queryStatement.setIsS2SQL(false);
|
||||
queryStatement = queryParser.logicSql(queryStatement);
|
||||
queryUtils.checkSqlParse(queryStatement);
|
||||
queryStatement.setModelIds(queryStructReq.getModelIds());
|
||||
log.info("queryStatement:{}", queryStatement);
|
||||
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
|
||||
queryOptimizer.rewrite(queryStructReq, queryStatement);
|
||||
}
|
||||
//queryParser.test(queryStructReq,metricReq);
|
||||
log.info("queryStatement:{}", queryStatement);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
package com.tencent.supersonic.headless.query.service;
|
||||
|
||||
import com.alibaba.excel.util.FileUtils;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
|
||||
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
|
||||
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
|
||||
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.service.QueryService;
|
||||
import com.tencent.supersonic.headless.core.service.impl.DownloadServiceImpl;
|
||||
import com.tencent.supersonic.headless.server.service.ModelService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
||||
class DownloadServiceImplTest {
|
||||
@Test
|
||||
void testBatchDownload() throws Exception {
|
||||
ModelService modelService = Mockito.mock(ModelService.class);
|
||||
QueryService queryService = Mockito.mock(QueryService.class);
|
||||
when(modelService.fetchModelSchema(any())).thenReturn(Lists.newArrayList(mockModelSchemaResp()));
|
||||
when(queryService.queryByStruct(any(), any())).thenReturn(mockQueryResult());
|
||||
DownloadServiceImpl downloadService = new DownloadServiceImpl(modelService, queryService);
|
||||
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
|
||||
File file = FileUtils.createTmpFile(fileName);
|
||||
downloadService.batchDownload(buildBatchDownloadReq(), User.getFakeUser(), file);
|
||||
}
|
||||
|
||||
private ModelSchemaResp mockModelSchemaResp() {
|
||||
ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
|
||||
modelSchemaResp.setId(1L);
|
||||
List<MetricSchemaResp> metricResps = Lists.newArrayList();
|
||||
metricResps.add(mockMetricPv());
|
||||
metricResps.add(mockMetricUv());
|
||||
modelSchemaResp.setMetrics(metricResps);
|
||||
List<DimSchemaResp> dimSchemaResps = Lists.newArrayList();
|
||||
dimSchemaResps.add(mockDimension(1L, "user_name", "用户名"));
|
||||
dimSchemaResps.add(mockDimension(2L, "department", "部门"));
|
||||
dimSchemaResps.add(mockDimension(3L, "page", "页面"));
|
||||
modelSchemaResp.setDimensions(dimSchemaResps);
|
||||
return modelSchemaResp;
|
||||
}
|
||||
|
||||
private MetricSchemaResp mockMetric(Long id, String bizName, String name, List<Long> drillDownloadDimensions) {
|
||||
MetricSchemaResp metricResp = new MetricSchemaResp();
|
||||
metricResp.setId(id);
|
||||
metricResp.setBizName(bizName);
|
||||
metricResp.setName(name);
|
||||
RelateDimension relateDimension = new RelateDimension();
|
||||
relateDimension.setDrillDownDimensions(drillDownloadDimensions.stream()
|
||||
.map(DrillDownDimension::new).collect(Collectors.toList()));
|
||||
metricResp.setRelateDimension(relateDimension);
|
||||
return metricResp;
|
||||
}
|
||||
|
||||
private DimSchemaResp mockDimension(Long id, String bizName, String name) {
|
||||
DimSchemaResp dimSchemaResp = new DimSchemaResp();
|
||||
dimSchemaResp.setId(id);
|
||||
dimSchemaResp.setBizName(bizName);
|
||||
dimSchemaResp.setName(name);
|
||||
return dimSchemaResp;
|
||||
}
|
||||
|
||||
private MetricSchemaResp mockMetricPv() {
|
||||
return mockMetric(1L, "pv", "访问次数", Lists.newArrayList(1L, 2L));
|
||||
}
|
||||
|
||||
private MetricSchemaResp mockMetricUv() {
|
||||
return mockMetric(2L, "uv", "访问用户数", Lists.newArrayList(2L));
|
||||
}
|
||||
|
||||
private BatchDownloadReq buildBatchDownloadReq() {
|
||||
BatchDownloadReq batchDownloadReq = new BatchDownloadReq();
|
||||
batchDownloadReq.setMetricIds(Lists.newArrayList(1L));
|
||||
batchDownloadReq.setDateInfo(mockDataConf());
|
||||
return batchDownloadReq;
|
||||
}
|
||||
|
||||
private DateConf mockDataConf() {
|
||||
DateConf dateConf = new DateConf();
|
||||
dateConf.setStartDate("2023-10-11");
|
||||
dateConf.setEndDate("2023-10-15");
|
||||
dateConf.setDateMode(DateConf.DateMode.BETWEEN);
|
||||
return dateConf;
|
||||
}
|
||||
|
||||
private QueryResultWithSchemaResp mockQueryResult() {
|
||||
QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp();
|
||||
List<Map<String, Object>> resultList = Lists.newArrayList();
|
||||
resultList.add(createMap("2023-10-11", "tom", "hr", "1"));
|
||||
resultList.add(createMap("2023-10-12", "alice", "sales", "2"));
|
||||
resultList.add(createMap("2023-10-13", "jack", "sales", "3"));
|
||||
resultList.add(createMap("2023-10-14", "luck", "market", "4"));
|
||||
resultList.add(createMap("2023-10-15", "tom", "hr", "5"));
|
||||
queryResultWithSchemaResp.setResultList(resultList);
|
||||
return queryResultWithSchemaResp;
|
||||
}
|
||||
|
||||
private static Map<String, Object> createMap(String sysImpDate, String d1, String d2, String m1) {
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
map.put("sys_imp_date", sysImpDate);
|
||||
map.put("user_name", d1);
|
||||
map.put("department", d2);
|
||||
map.put("pv", m1);
|
||||
return map;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package com.tencent.supersonic.headless.query.utils;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.headless.core.utils.DataTransformUtils;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
class DataTransformUtilsTest {
|
||||
|
||||
@Test
|
||||
public void testTransform() {
|
||||
List<Map<String, Object>> inputData = new ArrayList<>();
|
||||
inputData.add(createMap("2023/10/11", "a", "b", "1"));
|
||||
inputData.add(createMap("2023/10/12", "a", "c", "2"));
|
||||
inputData.add(createMap("2023/10/13", "a", "b", "3"));
|
||||
inputData.add(createMap("2023/10/14", "a", "c", "4"));
|
||||
inputData.add(createMap("2023/10/15", "b", "b", "5"));
|
||||
List<String> groups = Lists.newArrayList("d1", "d2");
|
||||
String metric = "m1";
|
||||
List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData,
|
||||
metric, groups, new DateConf());
|
||||
Assertions.assertEquals(3, resultData.size());
|
||||
}
|
||||
|
||||
private static Map<String, Object> createMap(String sysImpDate, String d1, String d2, String m1) {
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
map.put("sys_imp_date", sysImpDate);
|
||||
map.put("d1", d1);
|
||||
map.put("d2", d2);
|
||||
map.put("m1", m1);
|
||||
return map;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user