topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(build) Add spotless during the build process. (#1639)

Browse Source
This commit is contained in:
lexluo09
2024-09-07 00:36:17 +08:00
committed by GitHub
parent ee15a88b06
commit 5f59e89eea
986 changed files with 15609 additions and 12706 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/ApiHeaderCheck.java
View File

@@ -7,6 +7,4 @@ import java.lang.annotation.Target;
@Target({ElementType.PARAMETER, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface ApiHeaderCheck {
}
public @interface ApiHeaderCheck {}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/annotation/S2DataPermission.java
View File

@@ -1,14 +1,12 @@
package com.tencent.supersonic.headless.server.annotation;
import java.lang.annotation.Target;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.Documented;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface S2DataPermission {
}
public @interface S2DataPermission {}

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/ApiHeaderCheckAspect.java
View File

@@ -1,5 +1,7 @@
package com.tencent.supersonic.headless.server.aspect;
import javax.servlet.http.HttpServletRequest;
import com.tencent.supersonic.common.pojo.Pair;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.SignatureUtils;
@@ -16,8 +18,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import javax.servlet.http.HttpServletRequest;
@Component
@Aspect
@Order(1)
@@ -30,12 +30,10 @@ public class ApiHeaderCheckAspect {
private static final String SIGNATURE = "signature";
@Autowired
private AppService appService;
@Autowired private AppService appService;
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck)")
private void apiPermissionCheck() {
}
private void apiPermissionCheck() {}
@Around("apiPermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
@@ -65,8 +63,12 @@ public class ApiHeaderCheckAspect {
if (!AppStatus.ONLINE.equals(appDetailResp.getAppStatus())) {
throw new InvalidArgumentException("该应用暂时为非在线状态");
}
Pair<Boolean, String> checkResult = SignatureUtils.isValidSignature(appId, appDetailResp.getAppSecret(),
Long.parseLong(timestampStr), signature);
Pair<Boolean, String> checkResult =
SignatureUtils.isValidSignature(
appId,
appDetailResp.getAppSecret(),
Long.parseLong(timestampStr),
signature);
if (!checkResult.first) {
throw new InvalidArgumentException(checkResult.second);
}

100
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -10,13 +10,13 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -43,10 +43,11 @@ public class DimValueAspect {
@Value("${s2.dimension.value.map.enable:true}")
private Boolean dimensionValueMapEnable;
@Autowired
private DimensionService dimensionService;
@Around("execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))")
@Autowired private DimensionService dimensionService;
@Around(
"execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
@@ -70,8 +71,10 @@ public class DimValueAspect {
QueryStructReq queryStructReq = (QueryStructReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryStructReq.getModelIds()));
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair = getAliasAndBizNameToTechName(dimensions);
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair = getTechNameToBizName(dimensions);
Map<String, Map<String, String>> dimAndAliasAndTechNamePair =
getAliasAndBizNameToTechName(dimensions);
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair =
getTechNameToBizName(dimensions);
rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair);
@@ -91,7 +94,8 @@ public class DimValueAspect {
log.debug("correctorSql before replacing:{}", sql);
List<FieldExpression> fieldExpressionList = SqlSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
Set<String> fieldNames = dimensions.stream().map(SchemaItem::getName).collect(Collectors.toSet());
Set<String> fieldNames =
dimensions.stream().map(SchemaItem::getName).collect(Collectors.toSet());
Map<String, Map<String, String>> filedNameToValueMap = new HashMap<>();
for (FieldExpression expression : fieldExpressionList) {
if (!fieldNames.contains(expression.getFieldName())) {
@@ -104,13 +108,22 @@ public class DimValueAspect {
}
// consider '=' filter
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())) {
dimension.getDimValueMaps().stream().forEach(dimValue -> {
if (!CollectionUtils.isEmpty(dimValue.getAlias())
&& dimValue.getAlias().contains(expression.getFieldValue().toString())) {
getFiledNameToValueMap(filedNameToValueMap, expression.getFieldValue().toString(),
dimValue.getTechName(), expression.getFieldName());
}
});
dimension.getDimValueMaps().stream()
.forEach(
dimValue -> {
if (!CollectionUtils.isEmpty(dimValue.getAlias())
&& dimValue.getAlias()
.contains(
expression
.getFieldValue()
.toString())) {
getFiledNameToValueMap(
filedNameToValueMap,
expression.getFieldValue().toString(),
dimValue.getTechName(),
expression.getFieldName());
}
});
}
// consider 'in' filter,each element needs to judge.
replaceInCondition(expression, dimension, filedNameToValueMap);
@@ -128,7 +141,9 @@ public class DimValueAspect {
return queryResultWithColumns;
}
public void replaceInCondition(FieldExpression expression, DimensionResp dimension,
public void replaceInCondition(
FieldExpression expression,
DimensionResp dimension,
Map<String, Map<String, String>> filedNameToValueMap) {
if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) {
String fieldValue = JsonUtil.toString(expression.getFieldValue());
@@ -150,20 +165,27 @@ public class DimValueAspect {
}
}
if (!revisedValues.equals(values)) {
getFiledNameToValueMap(filedNameToValueMap, JsonUtil.toString(values),
JsonUtil.toString(revisedValues), expression.getFieldName());
getFiledNameToValueMap(
filedNameToValueMap,
JsonUtil.toString(values),
JsonUtil.toString(revisedValues),
expression.getFieldName());
}
}
}
public void getFiledNameToValueMap(Map<String, Map<String, String>> filedNameToValueMap,
String oldValue, String newValue, String fieldName) {
public void getFiledNameToValueMap(
Map<String, Map<String, String>> filedNameToValueMap,
String oldValue,
String newValue,
String fieldName) {
Map<String, String> map = new HashMap<>();
map.put(oldValue, newValue);
filedNameToValueMap.put(fieldName, map);
}
private void rewriteDimValue(SemanticQueryResp semanticQueryResp,
private void rewriteDimValue(
SemanticQueryResp semanticQueryResp,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (!selectDimValueMap(semanticQueryResp.getColumns(), dimAndTechNameAndBizNamePair)) {
return;
@@ -171,10 +193,12 @@ public class DimValueAspect {
log.debug("start rewriteDimValue for resultList");
for (Map<String, Object> line : semanticQueryResp.getResultList()) {
for (String bizName : line.keySet()) {
if (dimAndTechNameAndBizNamePair.containsKey(bizName) && Objects.nonNull(line.get(bizName))) {
if (dimAndTechNameAndBizNamePair.containsKey(bizName)
&& Objects.nonNull(line.get(bizName))) {
String techName = line.get(bizName).toString();
Map<String, String> techAndBizPair = dimAndTechNameAndBizNamePair.get(bizName);
if (!CollectionUtils.isEmpty(techAndBizPair) && techAndBizPair.containsKey(techName)) {
if (!CollectionUtils.isEmpty(techAndBizPair)
&& techAndBizPair.containsKey(techName)) {
String bizValueName = techAndBizPair.get(techName);
if (StringUtils.isNotEmpty(bizValueName)) {
line.put(bizName, bizValueName);
@@ -185,8 +209,9 @@ public class DimValueAspect {
}
}
private boolean selectDimValueMap(List<QueryColumn> columns, Map<String,
Map<String, String>> dimAndTechNameAndBizNamePair) {
private boolean selectDimValueMap(
List<QueryColumn> columns,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)
|| CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)) {
return false;
@@ -200,7 +225,8 @@ public class DimValueAspect {
return false;
}
private void rewriteFilter(List<Filter> dimensionFilters, Map<String, Map<String, String>> aliasAndTechNamePair) {
private void rewriteFilter(
List<Filter> dimensionFilters, Map<String, Map<String, String>> aliasAndTechNamePair) {
for (Filter filter : dimensionFilters) {
if (Objects.isNull(filter)) {
continue;
@@ -238,7 +264,8 @@ public class DimValueAspect {
}
}
private Map<String, Map<String, String>> getAliasAndBizNameToTechName(List<DimensionResp> dimensions) {
private Map<String, Map<String, String>> getAliasAndBizNameToTechName(
List<DimensionResp> dimensions) {
if (CollectionUtils.isEmpty(dimensions)) {
return new HashMap<>();
}
@@ -256,14 +283,18 @@ public class DimValueAspect {
continue;
}
if (StringUtils.isNotEmpty(dimValueMap.getBizName())) {
aliasAndBizNameToTechName.put(dimValueMap.getBizName(), dimValueMap.getTechName());
aliasAndBizNameToTechName.put(
dimValueMap.getBizName(), dimValueMap.getTechName());
}
if (!CollectionUtils.isEmpty(dimValueMap.getAlias())) {
dimValueMap.getAlias().stream().forEach(alias -> {
if (StringUtils.isNotEmpty(alias)) {
aliasAndBizNameToTechName.put(alias, dimValueMap.getTechName());
}
});
dimValueMap.getAlias().stream()
.forEach(
alias -> {
if (StringUtils.isNotEmpty(alias)) {
aliasAndBizNameToTechName.put(
alias, dimValueMap.getTechName());
}
});
}
}
@@ -308,7 +339,8 @@ public class DimValueAspect {
}
private boolean needSkipDimension(DimensionResp dimension) {
return Objects.isNull(dimension) || StringUtils.isEmpty(dimension.getBizName()) || CollectionUtils.isEmpty(
dimension.getDimValueMaps());
return Objects.isNull(dimension)
|| StringUtils.isEmpty(dimension.getBizName())
|| CollectionUtils.isEmpty(dimension.getDimValueMaps());
}
}

183
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.aspect;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
@@ -15,6 +14,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
@@ -22,10 +22,9 @@ import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -38,6 +37,7 @@ import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -51,22 +51,17 @@ import java.util.stream.Collectors;
@Slf4j
public class S2DataPermissionAspect {
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private ModelService modelService;
@Autowired
private SchemaService schemaService;
@Autowired
private AuthService authService;
@Autowired private QueryStructUtils queryStructUtils;
@Autowired private ModelService modelService;
@Autowired private SchemaService schemaService;
@Autowired private AuthService authService;
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)")
private void s2PermissionCheck() {
}
private void s2PermissionCheck() {}
@Around("s2PermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
//1. check args
// 1. check args
Object[] objects = joinPoint.getArgs();
boolean needQueryData = true;
SemanticQueryReq queryReq = null;
@@ -88,24 +83,24 @@ public class S2DataPermissionAspect {
SemanticSchemaResp semanticSchemaResp = getSemanticSchemaResp(queryReq);
List<Long> modelIds = getModelIds(semanticSchemaResp);
//2. determine whether admin of the model
// 2. determine whether admin of the model
if (checkModelAdmin(user, modelIds)) {
return joinPoint.proceed();
}
//3. determine whether the model is visible to cur user
// 3. determine whether the model is visible to cur user
checkModelVisible(user, modelIds);
//4. get permissions auth to cur user
// 4. get permissions auth to cur user
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIds);
//5. check col permission
// 5. check col permission
if (needQueryData) {
checkColPermission(queryReq, authorizedResource, modelIds, semanticSchemaResp);
}
//6. check row permission
// 6. check row permission
checkRowPermission(queryReq, authorizedResource);
//7. add hint to user
// 7. add hint to user
Object result = joinPoint.proceed();
if (result instanceof SemanticQueryResp) {
addHint(modelIds, (SemanticQueryResp) result, authorizedResource);
@@ -113,29 +108,39 @@ public class S2DataPermissionAspect {
return result;
}
private void checkColPermission(SemanticQueryReq semanticQueryReq, AuthorizedResourceResp authorizedResource,
List<Long> modelIds, SemanticSchemaResp semanticSchemaResp) {
private void checkColPermission(
SemanticQueryReq semanticQueryReq,
AuthorizedResourceResp authorizedResource,
List<Long> modelIds,
SemanticSchemaResp semanticSchemaResp) {
// get high sensitive fields in query
Set<String> bizNamesInQueryReq = getBizNameInQueryReq(semanticQueryReq, semanticSchemaResp);
Set<String> sensitiveBizNamesByModel = getHighSensitiveBizNamesByModelId(semanticSchemaResp);
Set<String> sensitiveBizNameInQuery = bizNamesInQueryReq.parallelStream()
.filter(sensitiveBizNamesByModel::contains).collect(Collectors.toSet());
Set<String> sensitiveBizNamesByModel =
getHighSensitiveBizNamesByModelId(semanticSchemaResp);
Set<String> sensitiveBizNameInQuery =
bizNamesInQueryReq
.parallelStream()
.filter(sensitiveBizNamesByModel::contains)
.collect(Collectors.toSet());
//get high sensitive field cur user has been authed
Set<String> sensitiveBizNameUserAuthed = authorizedResource.getAuthResList()
.stream().map(AuthRes::getName).collect(Collectors.toSet());
// get high sensitive field cur user has been authed
Set<String> sensitiveBizNameUserAuthed =
authorizedResource.getAuthResList().stream()
.map(AuthRes::getName)
.collect(Collectors.toSet());
sensitiveBizNameInQuery.removeAll(sensitiveBizNameUserAuthed);
if (!CollectionUtils.isEmpty(sensitiveBizNameInQuery)) {
Set<String> sensitiveResNames = semanticSchemaResp.getNameFromBizNames(sensitiveBizNameInQuery);
Set<String> sensitiveResNames =
semanticSchemaResp.getNameFromBizNames(sensitiveBizNameInQuery);
List<String> modelAdmin = modelService.getModelAdmin(modelIds.get(0));
String message = String.format("存在以下敏感资源:%s您暂无权限，请联系管理员%s申请",
sensitiveResNames, modelAdmin);
String message =
String.format("存在以下敏感资源:%s您暂无权限，请联系管理员%s申请", sensitiveResNames, modelAdmin);
throw new InvalidPermissionException(message);
}
}
private void checkRowPermission(SemanticQueryReq queryReq,
AuthorizedResourceResp authorizedResource) {
private void checkRowPermission(
SemanticQueryReq queryReq, AuthorizedResourceResp authorizedResource) {
if (queryReq instanceof QuerySqlReq) {
doRowPermission((QuerySqlReq) queryReq, authorizedResource);
}
@@ -144,7 +149,8 @@ public class S2DataPermissionAspect {
}
}
private Set<String> getBizNameInQueryReq(SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) {
private Set<String> getBizNameInQueryReq(
SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) {
if (queryReq instanceof QuerySqlReq) {
return queryStructUtils.getBizNameFromSql((QuerySqlReq) queryReq, semanticSchemaResp);
}
@@ -163,10 +169,12 @@ public class S2DataPermissionAspect {
private List<Long> getModelIds(SemanticSchemaResp semanticSchemaResp) {
return semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
.map(ModelResp::getId)
.collect(Collectors.toList());
}
private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(
QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -180,11 +188,14 @@ public class S2DataPermissionAspect {
return;
}
dimensionFilters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
dimensionFilters.stream()
.forEach(
filter -> {
if (StringUtils.isNotEmpty(filter)
&& StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
@@ -196,10 +207,10 @@ public class S2DataPermissionAspect {
} catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
}
}
private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(
QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -213,37 +224,45 @@ public class S2DataPermissionAspect {
return;
}
dimensionFilters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
dimensionFilters.stream()
.forEach(
filter -> {
if (StringUtils.isNotEmpty(filter)
&& StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
if (StringUtils.isNotEmpty(joiner.toString())) {
log.info("before doRowPermission, queryStructReq:{}", queryStructReq);
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString());
List<Filter> filters = Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>()
: queryStructReq.getOriginalFilter();
List<Filter> filters =
Objects.isNull(queryStructReq.getOriginalFilter())
? new ArrayList<>()
: queryStructReq.getOriginalFilter();
filters.add(filter);
queryStructReq.setDimensionFilters(filters);
log.info("after doRowPermission, queryStructReq:{}", queryStructReq);
}
}
public boolean checkModelAdmin(User user, List<Long> modelIds) {
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
List<ModelResp> modelListAdmin =
modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
} else {
Set<Long> modelAdmins = modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet());
Set<Long> modelAdmins =
modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet());
return !CollectionUtils.isEmpty(modelAdmins) && modelAdmins.containsAll(modelIds);
}
}
public void checkModelVisible(User user, List<Long> modelIds) {
List<Long> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE)
.stream().map(ModelResp::getId).collect(Collectors.toList());
List<Long> modelListVisible =
modelService.getModelListWithAuth(user, null, AuthType.VISIBLE).stream()
.map(ModelResp::getId)
.collect(Collectors.toList());
List<Long> modelIdCopied = new ArrayList<>(modelIds);
modelIdCopied.removeAll(modelListVisible);
if (!CollectionUtils.isEmpty(modelIdCopied)) {
@@ -254,7 +273,9 @@ public class S2DataPermissionAspect {
if (modelResp == null) {
throw new InvalidArgumentException("查询的模型不存在");
}
String message = String.format("您没有模型[%s]权限，请联系管理员%s开通", modelResp.getName(), modelResp.getAdmins());
String message =
String.format(
"您没有模型[%s]权限，请联系管理员%s开通", modelResp.getName(), modelResp.getAdmins());
throw new InvalidPermissionException(message);
}
}
@@ -262,13 +283,21 @@ public class S2DataPermissionAspect {
public Set<String> getHighSensitiveBizNamesByModelId(SemanticSchemaResp semanticSchemaResp) {
Set<String> highSensitiveCols = new HashSet<>();
if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) {
semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp ->
SensitiveLevelEnum.HIGH.getCode().equals(dimSchemaResp.getSensitiveLevel()))
semanticSchemaResp.getDimensions().stream()
.filter(
dimSchemaResp ->
SensitiveLevelEnum.HIGH
.getCode()
.equals(dimSchemaResp.getSensitiveLevel()))
.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
if (!CollectionUtils.isEmpty(semanticSchemaResp.getMetrics())) {
semanticSchemaResp.getMetrics().stream().filter(metricSchemaResp ->
SensitiveLevelEnum.HIGH.getCode().equals(metricSchemaResp.getSensitiveLevel()))
semanticSchemaResp.getMetrics().stream()
.filter(
metricSchemaResp ->
SensitiveLevelEnum.HIGH
.getCode()
.equals(metricSchemaResp.getSensitiveLevel()))
.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
}
return highSensitiveCols;
@@ -278,8 +307,11 @@ public class S2DataPermissionAspect {
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setModelIds(modelIds);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}",
user.getName(), modelIds, authorizedResource);
log.info(
"user:{}, domainId:{}, after queryAuthorizedResources:{}",
user.getName(),
modelIds,
authorizedResource);
return authorizedResource;
}
@@ -288,8 +320,10 @@ public class S2DataPermissionAspect {
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
public void addHint(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource) {
public void addHint(
List<Long> modelIds,
SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(filters)) {
return;
@@ -300,17 +334,22 @@ public class S2DataPermissionAspect {
ModelResp modelResp = modelService.getModel(modelIds.get(0));
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
filters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter.getDescription())) {
descList.add(filter.getDescription());
}
exprList.add(filter.getExpressions().toString());
});
filters.stream()
.forEach(
filter -> {
if (StringUtils.isNotEmpty(filter.getDescription())) {
descList.add(filter.getDescription());
}
exprList.add(filter.getExpressions().toString());
});
String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s";
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
String message =
String.format(
promptInfo,
CollectionUtils.isEmpty(descList) ? exprList : descList,
admins);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
}
}
}

44
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/ChatQueryApiController.java
View File

@@ -1,5 +1,8 @@
package com.tencent.supersonic.headless.server.facade.rest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -14,48 +17,51 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class ChatQueryApiController {
@Autowired
private ChatLayerService chatLayerService;
@Autowired private ChatLayerService chatLayerService;
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired private SemanticLayerService semanticLayerService;
@PostMapping("/chat/search")
public Object search(@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
public Object search(
@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
queryNLReq.setUser(UserHolder.findUser(request, response));
return chatLayerService.retrieve(queryNLReq);
}
@PostMapping("/chat/map")
public Object map(@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response) {
public Object map(
@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response) {
queryNLReq.setUser(UserHolder.findUser(request, response));
return chatLayerService.performMapping(queryNLReq);
}
@PostMapping("/chat/parse")
public Object parse(@RequestBody QueryNLReq queryNLReq,
public Object parse(
@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response)
throws Exception {
queryNLReq.setUser(UserHolder.findUser(request, response));
return chatLayerService.performParsing(queryNLReq);
}
@PostMapping("/chat")
public Object queryByNL(@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
public Object queryByNL(
@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
ParseResp parseResp = chatLayerService.performParsing(queryNLReq);
if (parseResp.getState().equals(ParseResp.ParseState.COMPLETED)) {
@@ -66,7 +72,7 @@ public class ChatQueryApiController {
return semanticLayerService.queryByReq(sqlReq, user);
}
throw new RuntimeException("Failed to parse natural language query: " + queryNLReq.getQueryText());
throw new RuntimeException(
"Failed to parse natural language query: " + queryNLReq.getQueryText());
}
}

21
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/DataSetQueryApiController.java
View File

@@ -1,11 +1,14 @@
package com.tencent.supersonic.headless.server.facade.rest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryDataSetReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
@@ -13,26 +16,22 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class DataSetQueryApiController {
@Autowired
private DataSetService dataSetService;
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired private DataSetService dataSetService;
@Autowired private SemanticLayerService semanticLayerService;
@PostMapping("/dataSet")
public Object queryByDataSet(@RequestBody QueryDataSetReq queryDataSetReq,
public Object queryByDataSet(
@RequestBody QueryDataSetReq queryDataSetReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
SemanticQueryReq queryReq = dataSetService.convert(queryDataSetReq);
return semanticLayerService.queryByReq(queryReq, user);
}
}

17
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetaDiscoveryApiController.java
View File

@@ -1,5 +1,8 @@
package com.tencent.supersonic.headless.server.facade.rest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
@@ -11,23 +14,21 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestController
@RequestMapping("/api/semantic/meta")
@Slf4j
public class MetaDiscoveryApiController {
@Autowired
private ChatLayerService chatLayerService;
@Autowired private ChatLayerService chatLayerService;
@PostMapping("map")
public Object map(@RequestBody QueryMapReq queryMapReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
public Object map(
@RequestBody QueryMapReq queryMapReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
queryMapReq.setUser(user);
return chatLayerService.map(queryMapReq);
}
}

40
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/MetricQueryApiController.java
View File

@@ -1,14 +1,17 @@
package com.tencent.supersonic.headless.server.facade.rest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.pojo.request.DownloadMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMetricReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.service.DownloadService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
@@ -16,46 +19,45 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestController
@RequestMapping("/api/semantic/query")
@Slf4j
public class MetricQueryApiController {
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired private SemanticLayerService semanticLayerService;
@Autowired
private MetricService metricService;
@Autowired private MetricService metricService;
@Autowired
private DownloadService downloadService;
@Autowired private DownloadService downloadService;
@PostMapping("/metric")
public Object queryByMetric(@RequestBody QueryMetricReq queryMetricReq,
public Object queryByMetric(
@RequestBody QueryMetricReq queryMetricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
QueryStructReq queryStructReq = metricService.convert(queryMetricReq);
return semanticLayerService.queryByReq(queryStructReq.convert(true), user);
}
@PostMapping("/download/metric")
public void downloadMetric(@RequestBody DownloadMetricReq downloadMetricReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
public void downloadMetric(
@RequestBody DownloadMetricReq downloadMetricReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
downloadService.downloadByStruct(downloadMetricReq, user, response);
}
@PostMapping("/downloadBatch/metric")
public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
public void downloadBatch(
@RequestBody BatchDownloadReq batchDownloadReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
downloadService.batchDownload(batchDownloadReq, user, response);
}
}

104
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/SqlQueryApiController.java
View File

@@ -1,5 +1,8 @@
package com.tencent.supersonic.headless.server.facade.rest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.util.StringUtil;
@@ -17,8 +20,6 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@@ -29,16 +30,16 @@ import java.util.stream.Collectors;
@Slf4j
public class SqlQueryApiController {
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired private SemanticLayerService semanticLayerService;
@Autowired
private ChatLayerService chatLayerService;
@Autowired private ChatLayerService chatLayerService;
@PostMapping("/sql")
public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
public Object queryBySql(
@RequestBody QuerySqlReq querySqlReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
String sql = querySqlReq.getSql();
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
@@ -47,48 +48,68 @@ public class SqlQueryApiController {
}
@PostMapping("/sqls")
public Object queryBySqls(@RequestBody QuerySqlsReq querySqlsReq,
public Object queryBySqls(
@RequestBody QuerySqlsReq querySqlsReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
List<SemanticQueryReq> semanticQueryReqs = querySqlsReq.getSqls()
.stream().map(sql -> {
QuerySqlReq querySqlReq = new QuerySqlReq();
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
chatLayerService.correct(querySqlReq, user);
return querySqlReq;
}).collect(Collectors.toList());
List<SemanticQueryReq> semanticQueryReqs =
querySqlsReq.getSqls().stream()
.map(
sql -> {
QuerySqlReq querySqlReq = new QuerySqlReq();
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
chatLayerService.correct(querySqlReq, user);
return querySqlReq;
})
.collect(Collectors.toList());
List<CompletableFuture<SemanticQueryResp>> futures = semanticQueryReqs.stream()
.map(querySqlReq -> CompletableFuture.supplyAsync(() -> {
try {
return semanticLayerService.queryByReq(querySqlReq, user);
} catch (Exception e) {
log.error("querySqlReq:{},queryByReq error:", querySqlReq, e);
return new SemanticQueryResp();
}
})).collect(Collectors.toList());
List<CompletableFuture<SemanticQueryResp>> futures =
semanticQueryReqs.stream()
.map(
querySqlReq ->
CompletableFuture.supplyAsync(
() -> {
try {
return semanticLayerService.queryByReq(
querySqlReq, user);
} catch (Exception e) {
log.error(
"querySqlReq:{},queryByReq error:",
querySqlReq,
e);
return new SemanticQueryResp();
}
}))
.collect(Collectors.toList());
return futures.stream().map(CompletableFuture::join).collect(Collectors.toList());
}
@PostMapping("/sqlsWithException")
public Object queryBySqlsWithException(@RequestBody QuerySqlsReq querySqlsReq,
public Object queryBySqlsWithException(
@RequestBody QuerySqlsReq querySqlsReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
List<SemanticQueryReq> semanticQueryReqs = querySqlsReq.getSqls()
.stream().map(sql -> {
QuerySqlReq querySqlReq = new QuerySqlReq();
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
chatLayerService.correct(querySqlReq, user);
return querySqlReq;
}).collect(Collectors.toList());
List<SemanticQueryReq> semanticQueryReqs =
querySqlsReq.getSqls().stream()
.map(
sql -> {
QuerySqlReq querySqlReq = new QuerySqlReq();
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
chatLayerService.correct(querySqlReq, user);
return querySqlReq;
})
.collect(Collectors.toList());
List<SemanticQueryResp> semanticQueryRespList = new ArrayList<>();
try {
for (SemanticQueryReq semanticQueryReq : semanticQueryReqs) {
SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(semanticQueryReq, user);
SemanticQueryResp semanticQueryResp =
semanticLayerService.queryByReq(semanticQueryReq, user);
semanticQueryRespList.add(semanticQueryResp);
}
} catch (Exception e) {
@@ -98,13 +119,14 @@ public class SqlQueryApiController {
}
@PostMapping("/validate")
public Object validate(@RequestBody QuerySqlReq querySqlReq,
public Object validate(
@RequestBody QuerySqlReq querySqlReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
String sql = querySqlReq.getSql();
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
return chatLayerService.validate(querySqlReq, user);
}
}

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/rest/TagQueryApiController.java
View File

@@ -1,12 +1,12 @@
package com.tencent.supersonic.headless.server.facade.rest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
@@ -19,15 +19,15 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
public class TagQueryApiController {
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired private SemanticLayerService semanticLayerService;
@PostMapping("/tag")
public Object queryByTag(@RequestBody QueryStructReq queryStructReq,
public Object queryByTag(
@RequestBody QueryStructReq queryStructReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
HttpServletResponse response)
throws Exception {
User user = UserHolder.findUser(request, response);
return semanticLayerService.queryByReq(queryStructReq.convert(), user);
}
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/ChatLayerService.java
View File

@@ -12,9 +12,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import java.util.List;
/**
* This interface adds natural language support to the semantic layer.
*/
/** This interface adds natural language support to the semantic layer. */
public interface ChatLayerService {
MapResp performMapping(QueryNLReq queryNLReq);
@@ -29,4 +27,3 @@ public interface ChatLayerService {
SqlEvaluation validate(QuerySqlReq querySqlReq, User user);
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/SemanticLayerService.java
View File

@@ -15,9 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp;
import java.util.List;
/**
* This interface abstracts functionalities provided by a semantic layer.
*/
/** This interface abstracts functionalities provided by a semantic layer. */
public interface SemanticLayerService {
DataSetSchema getDataSetSchema(Long id);
@@ -35,5 +33,4 @@ public interface SemanticLayerService {
List<DimensionResp> getDimensions(MetaFilter metaFilter);
List<MetricResp> getMetrics(MetaFilter metaFilter);
}

178
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java
View File

@@ -5,6 +5,7 @@ import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
@@ -28,12 +29,11 @@ import com.tencent.supersonic.headless.chat.corrector.GrammarCorrector;
import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector;
import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.RetrieveService;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.ChatWorkflowEngine;
import com.tencent.supersonic.headless.server.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.BeanUtils;
@@ -53,22 +53,20 @@ import java.util.stream.Collectors;
@Service
@Slf4j
public class S2ChatLayerService implements ChatLayerService {
@Autowired
private SchemaService schemaService;
@Autowired
private DataSetService dataSetService;
@Autowired
private RetrieveService retrieveService;
@Autowired
private ChatWorkflowEngine chatWorkflowEngine;
@Autowired private SchemaService schemaService;
@Autowired private DataSetService dataSetService;
@Autowired private RetrieveService retrieveService;
@Autowired private ChatWorkflowEngine chatWorkflowEngine;
@Override
public MapResp performMapping(QueryNLReq queryNLReq) {
MapResp mapResp = new MapResp(queryNLReq.getQueryText());
ChatQueryContext queryCtx = buildChatQueryContext(queryNLReq);
ComponentFactory.getSchemaMappers().forEach(mapper -> {
mapper.map(queryCtx);
});
ComponentFactory.getSchemaMappers()
.forEach(
mapper -> {
mapper.map(queryCtx);
});
mapResp.setMapInfo(queryCtx.getMapInfo());
return mapResp;
}
@@ -77,7 +75,8 @@ public class S2ChatLayerService implements ChatLayerService {
public MapInfoResp map(QueryMapReq queryMapReq) {
QueryNLReq queryNLReq = new QueryNLReq();
BeanUtils.copyProperties(queryMapReq, queryNLReq);
List<DataSetResp> dataSets = dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
List<DataSetResp> dataSets =
dataSetService.getDataSets(queryMapReq.getDataSetNames(), queryMapReq.getUser());
Set<Long> dataSetIds = dataSets.stream().map(SchemaItem::getId).collect(Collectors.toSet());
queryNLReq.setDataSetIds(dataSetIds);
@@ -97,16 +96,17 @@ public class S2ChatLayerService implements ChatLayerService {
private ChatQueryContext buildChatQueryContext(QueryNLReq queryNLReq) {
SemanticSchema semanticSchema = schemaService.getSemanticSchema(queryNLReq.getDataSetIds());
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds();
ChatQueryContext queryCtx = ChatQueryContext.builder()
.queryFilters(queryNLReq.getQueryFilters())
.semanticSchema(semanticSchema)
.candidateQueries(new ArrayList<>())
.mapInfo(new SchemaMapInfo())
.modelIdToDataSetIds(modelIdToDataSetIds)
.text2SQLType(queryNLReq.getText2SQLType())
.mapModeEnum(queryNLReq.getMapModeEnum())
.dataSetIds(queryNLReq.getDataSetIds())
.build();
ChatQueryContext queryCtx =
ChatQueryContext.builder()
.queryFilters(queryNLReq.getQueryFilters())
.semanticSchema(semanticSchema)
.candidateQueries(new ArrayList<>())
.mapInfo(new SchemaMapInfo())
.modelIdToDataSetIds(modelIdToDataSetIds)
.text2SQLType(queryNLReq.getText2SQLType())
.mapModeEnum(queryNLReq.getMapModeEnum())
.dataSetIds(queryNLReq.getDataSetIds())
.build();
BeanUtils.copyProperties(queryNLReq, queryCtx);
return queryCtx;
}
@@ -129,8 +129,8 @@ public class S2ChatLayerService implements ChatLayerService {
private SemanticParseInfo correctSqlReq(QuerySqlReq querySqlReq, User user) {
ChatQueryContext queryCtx = new ChatQueryContext();
SemanticSchema semanticSchema = schemaService.getSemanticSchema(
Sets.newHashSet(querySqlReq.getDataSetId()));
SemanticSchema semanticSchema =
schemaService.getSemanticSchema(Sets.newHashSet(querySqlReq.getDataSetId()));
queryCtx.setSemanticSchema(semanticSchema);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
SqlInfo sqlInfo = new SqlInfo();
@@ -146,11 +146,14 @@ public class S2ChatLayerService implements ChatLayerService {
SchemaElement dataSet = semanticSchema.getDataSet(dataSetId);
semanticParseInfo.setDataSet(dataSet);
ComponentFactory.getSemanticCorrectors().forEach(corrector -> {
if (!(corrector instanceof GrammarCorrector || (corrector instanceof SchemaCorrector))) {
corrector.correct(queryCtx, semanticParseInfo);
}
});
ComponentFactory.getSemanticCorrectors()
.forEach(
corrector -> {
if (!(corrector instanceof GrammarCorrector
|| (corrector instanceof SchemaCorrector))) {
corrector.correct(queryCtx, semanticParseInfo);
}
});
log.info("chatQueryServiceImpl correct:{}", sqlInfo.getCorrectedS2SQL());
return semanticParseInfo;
}
@@ -164,16 +167,15 @@ public class S2ChatLayerService implements ChatLayerService {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(new ArrayList<>(dataSetIds));
List<DataSetResp> dataSetList = dataSetService.getDataSetList(metaFilter);
Map<Long, DataSetResp> dataSetMap = dataSetList.stream()
.collect(Collectors.toMap(DataSetResp::getId, d -> d));
Map<Long, DataSetResp> dataSetMap =
dataSetList.stream().collect(Collectors.toMap(DataSetResp::getId, d -> d));
mapInfoResp.setDataSetMapInfo(getDataSetInfo(mapResp.getMapInfo(), dataSetMap, topN));
mapInfoResp.setTerms(getTerms(mapResp.getMapInfo(), dataSetMap));
return mapInfoResp;
}
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap,
Integer topN) {
private Map<String, DataSetMapInfo> getDataSetInfo(
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap, Integer topN) {
Map<String, DataSetMapInfo> map = new HashMap<>();
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
@@ -195,14 +197,18 @@ public class S2ChatLayerService implements ChatLayerService {
return map;
}
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
private Map<Long, List<SchemaElementMatch>> getMapFields(
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values = entry.getValue().stream()
.filter(schemaElementMatch ->
!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (Map.Entry<Long, List<SchemaElementMatch>> entry :
mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values =
entry.getValue().stream()
.filter(
schemaElementMatch ->
!SchemaElementType.TERM.equals(
schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
result.put(entry.getKey(), values);
}
@@ -210,15 +216,15 @@ public class S2ChatLayerService implements ChatLayerService {
return result;
}
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN,
SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
private Map<Long, List<SchemaElementMatch>> getTopFields(
Integer topN, SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
if (0 == topN) {
return result;
}
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry :
mapInfo.getDataSetElementMatches().entrySet()) {
Long dataSetId = entry.getKey();
List<SchemaElementMatch> values = entry.getValue();
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
@@ -226,18 +232,24 @@ public class S2ChatLayerService implements ChatLayerService {
continue;
}
String dataSetName = dataSetResp.getName();
//topN dimensions
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
// topN dimensions
Set<SchemaElementMatch> dimensions =
semanticSchema.getDimensions(dataSetId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1)
.map(mergeFunction())
.collect(Collectors.toSet());
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
dimensions.add(timeDimensionMatch);
//topN metrics
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId)
.stream().sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN).map(mergeFunction()).collect(Collectors.toSet());
// topN metrics
Set<SchemaElementMatch> metrics =
semanticSchema.getMetrics(dataSetId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN)
.map(mergeFunction())
.collect(Collectors.toSet());
dimensions.addAll(metrics);
result.put(dataSetId, new ArrayList<>(dimensions));
@@ -245,44 +257,64 @@ public class S2ChatLayerService implements ChatLayerService {
return result;
}
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetNameMap) {
private Map<String, List<SchemaElementMatch>> getTerms(
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetNameMap) {
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
Map<Long, List<SchemaElementMatch>> dataSetElementMatches = mapInfo.getDataSetElementMatches();
Map<Long, List<SchemaElementMatch>> dataSetElementMatches =
mapInfo.getDataSetElementMatches();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
DataSetResp dataSetResp = dataSetNameMap.get(entry.getKey());
if (dataSetResp == null) {
continue;
}
List<SchemaElementMatch> terms = entry.getValue().stream().filter(schemaElementMatch
-> SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
List<SchemaElementMatch> terms =
entry.getValue().stream()
.filter(
schemaElementMatch ->
SchemaElementType.TERM.equals(
schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
termMap.put(dataSetResp.getName(), terms);
}
return termMap;
}
/***
* get time dimension SchemaElementMatch
/**
* * get time dimension SchemaElementMatch
*
* @param dataSetId
* @param dataSetName
* @return
*/
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
SchemaElement element = SchemaElement.builder().dataSetId(dataSetId).dataSetName(dataSetName)
.type(SchemaElementType.DIMENSION).bizName(TimeDimensionEnum.DAY.getName()).build();
SchemaElement element =
SchemaElement.builder()
.dataSetId(dataSetId)
.dataSetName(dataSetName)
.type(SchemaElementType.DIMENSION)
.bizName(TimeDimensionEnum.DAY.getName())
.build();
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
.detectWord(TimeDimensionEnum.DAY.getChName()).word(TimeDimensionEnum.DAY.getChName())
.similarity(1L).frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
SchemaElementMatch timeDimensionMatch =
SchemaElementMatch.builder()
.element(element)
.detectWord(TimeDimensionEnum.DAY.getChName())
.word(TimeDimensionEnum.DAY.getChName())
.similarity(1L)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY)
.build();
return timeDimensionMatch;
}
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()).similarity(1)
.detectWord(schemaElement.getName()).build();
return schemaElement ->
SchemaElementMatch.builder()
.element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY)
.word(schemaElement.getName())
.similarity(1)
.detectWord(schemaElement.getName())
.build();
}
}

190
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -52,14 +52,14 @@ import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.MetricDrillDownChecker;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -77,7 +77,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Slf4j
public class S2SemanticLayerService implements SemanticLayerService {
@@ -144,10 +143,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
TaskStatusEnum state = TaskStatusEnum.SUCCESS;
log.info("[queryReq:{}]", queryReq);
try {
//1.initStatInfo
// 1.initStatInfo
statUtils.initStatInfo(queryReq, user);
//2.query from cache
// 2.query from cache
String cacheKey = queryCache.getCacheKey(queryReq);
Object query = queryCache.query(queryReq, cacheKey);
@@ -159,7 +158,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
}
StatUtils.get().setUseResultCache(false);
//3 query
// 3 query
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
SemanticQueryResp queryResp = null;
@@ -172,11 +171,12 @@ public class S2SemanticLayerService implements SemanticLayerService {
for (QueryExecutor queryExecutor : queryExecutors) {
if (queryExecutor.accept(queryStatement)) {
queryResp = queryExecutor.execute(queryStatement);
queryUtils.populateQueryColumns(queryResp, queryStatement.getSemanticSchemaResp());
queryUtils.populateQueryColumns(
queryResp, queryStatement.getSemanticSchemaResp());
}
}
//4 reset cache and set stateInfo
// 4 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(cacheKey, queryResp);
if (setCacheSuccess) {
// if result is not null, update cache data
@@ -218,7 +218,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
return semanticQueryResp;
}
private List<String> getDimensionValuesFromDict(DimensionValueReq dimensionValueReq, Set<Long> dataSetIds) {
private List<String> getDimensionValuesFromDict(
DimensionValueReq dimensionValueReq, Set<Long> dataSetIds) {
if (StringUtils.isBlank(dimensionValueReq.getValue())) {
return SearchService.getDimensionValue(dimensionValueReq);
}
@@ -226,20 +227,28 @@ public class S2SemanticLayerService implements SemanticLayerService {
Map<Long, List<Long>> modelIdToDataSetIds = new HashMap<>();
modelIdToDataSetIds.put(dimensionValueReq.getModelId(), new ArrayList<>(dataSetIds));
List<HanlpMapResult> hanlpMapResultList = knowledgeBaseService.prefixSearch(
dimensionValueReq.getValue(), 2000, modelIdToDataSetIds, dataSetIds);
List<HanlpMapResult> hanlpMapResultList =
knowledgeBaseService.prefixSearch(
dimensionValueReq.getValue(), 2000, modelIdToDataSetIds, dataSetIds);
HanlpHelper.transLetterOriginal(hanlpMapResultList);
return hanlpMapResultList.stream()
.filter(o -> o.getNatures().stream()
.map(NatureHelper::getElementID)
.anyMatch(elementID -> dimensionValueReq.getElementID().equals(elementID)))
.filter(
o ->
o.getNatures().stream()
.map(NatureHelper::getElementID)
.anyMatch(
elementID ->
dimensionValueReq
.getElementID()
.equals(elementID)))
.map(MapResult::getName)
.collect(Collectors.toList());
}
private SemanticQueryResp getDimensionValuesFromDb(DimensionValueReq dimensionValueReq, User user) {
private SemanticQueryResp getDimensionValuesFromDb(
DimensionValueReq dimensionValueReq, User user) {
QuerySqlReq querySqlReq = buildQuerySqlReq(dimensionValueReq);
return queryByReq(querySqlReq, user);
}
@@ -256,14 +265,15 @@ public class S2SemanticLayerService implements SemanticLayerService {
return columns;
}
private List<Map<String, Object>> createResultList(DimensionValueReq dimensionValueReq,
List<String> dimensionValues) {
private List<Map<String, Object>> createResultList(
DimensionValueReq dimensionValueReq, List<String> dimensionValues) {
return dimensionValues.stream()
.map(value -> {
Map<String, Object> map = new HashMap<>();
map.put(dimensionValueReq.getBizName(), value);
return map;
})
.map(
value -> {
Map<String, Object> map = new HashMap<>();
map.put(dimensionValueReq.getBizName(), value);
return map;
})
.collect(Collectors.toList());
}
@@ -278,10 +288,12 @@ public class S2SemanticLayerService implements SemanticLayerService {
return dimensionResp;
}
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
public EntityInfo getEntityInfo(
SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
if (parseInfo.getDimensionFilters().size() <= 0 || entityInfo.getDataSetInfo() == null) {
if (parseInfo.getDimensionFilters().size() <= 0
|| entityInfo.getDataSetInfo() == null) {
entityInfo.setMetrics(null);
entityInfo.setDimensions(null);
return entityInfo;
@@ -290,8 +302,9 @@ public class S2SemanticLayerService implements SemanticLayerService {
if (StringUtils.isNotBlank(primaryKey)) {
String entityId = "";
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
.equals(primaryKey)) {
if (chatFilter != null
&& chatFilter.getBizName() != null
&& chatFilter.getBizName().equals(primaryKey)) {
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
entityId = chatFilter.getValue().toString();
}
@@ -344,8 +357,9 @@ public class S2SemanticLayerService implements SemanticLayerService {
return metrics;
}
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception {
//If dataSetId or DataSetName is empty, parse dataSetId from the SQL
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user)
throws Exception {
// If dataSetId or DataSetName is empty, parse dataSetId from the SQL
if (querySqlReq.needGetDataSetId()) {
Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user);
querySqlReq.setDataSetId(dataSetId);
@@ -360,7 +374,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryStatement;
}
private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) throws Exception {
private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user)
throws Exception {
QueryStatement queryStatement = null;
if (semanticQueryReq instanceof QuerySqlReq) {
queryStatement = buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user);
@@ -369,9 +384,11 @@ public class S2SemanticLayerService implements SemanticLayerService {
queryStatement = buildStructQueryStatement((QueryStructReq) semanticQueryReq);
}
if (semanticQueryReq instanceof QueryMultiStructReq) {
queryStatement = buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user);
queryStatement =
buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user);
}
if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo())
if (Objects.nonNull(queryStatement)
&& Objects.nonNull(semanticQueryReq.getSqlInfo())
&& StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) {
queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
@@ -395,8 +412,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryStatement;
}
private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq, User user)
throws Exception {
private QueryStatement buildMultiStructQueryStatement(
QueryMultiStructReq queryMultiStructReq, User user) throws Exception {
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq, user);
@@ -420,20 +437,34 @@ public class S2SemanticLayerService implements SemanticLayerService {
private QuerySqlReq buildQuerySqlReq(DimensionValueReq queryDimValueReq) {
QuerySqlReq querySqlReq = new QuerySqlReq();
List<ModelResp> modelResps = schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getBizName(),
queryDimValueReq.getModelId());
List<ModelResp> modelResps =
schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp =
schemaService.getDimension(
queryDimValueReq.getBizName(), queryDimValueReq.getModelId());
ModelResp modelResp = modelResps.get(0);
String sql = String.format("select distinct %s from %s where 1=1",
dimensionResp.getName(), modelResp.getName());
String sql =
String.format(
"select distinct %s from %s where 1=1",
dimensionResp.getName(), modelResp.getName());
List<Dim> timeDims = modelResp.getTimeDimension();
if (CollectionUtils.isNotEmpty(timeDims)) {
sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getEndDate());
sql =
String.format(
"%s and %s >= '%s' and %s <= '%s'",
sql,
TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getStartDate(),
TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getEndDate());
}
if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%" + queryDimValueReq.getValue() + "%'";
sql +=
" AND "
+ queryDimValueReq.getBizName()
+ " LIKE '%"
+ queryDimValueReq.getValue()
+ "%'";
}
querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
querySqlReq.setSql(sql);
@@ -441,8 +472,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
}
private void queryPreCheck(QueryStatement queryStatement) {
//Check whether the dimensions of the metric drill-down are correct temporarily,
//add the abstraction of a validator later.
// Check whether the dimensions of the metric drill-down are correct temporarily,
// add the abstraction of a validator later.
metricDrillDownChecker.checkQuery(queryStatement);
}
@@ -466,28 +497,48 @@ public class S2SemanticLayerService implements SemanticLayerService {
if (tagTypeDefaultConfig == null || tagTypeDefaultConfig.getDefaultDisplayInfo() == null) {
return entityInfo;
}
List<DataInfo> dimensions = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> {
SchemaElement element = dataSetSchema.getElement(SchemaElementType.DIMENSION, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
List<DataInfo> metrics = tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(id -> {
SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(), element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
List<DataInfo> dimensions =
tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(
id -> {
SchemaElement element =
dataSetSchema.getElement(
SchemaElementType.DIMENSION, id);
if (element == null) {
return null;
}
return new DataInfo(
element.getId().intValue(),
element.getName(),
element.getBizName(),
null);
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
List<DataInfo> metrics =
tagTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(
id -> {
SchemaElement element =
dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (element == null) {
return null;
}
return new DataInfo(
element.getId().intValue(),
element.getName(),
element.getBizName(),
null);
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
entityInfo.setDimensions(dimensions);
entityInfo.setMetrics(metrics);
return entityInfo;
}
private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
private void fillEntityInfoValue(
EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
SemanticQueryResp queryResultWithColumns =
getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user);
if (queryResultWithColumns != null) {
@@ -498,17 +549,19 @@ public class S2SemanticLayerService implements SemanticLayerService {
if (entry.getValue() == null || entryKey == null) {
continue;
}
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
entityInfo.getDimensions().stream()
.filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
entityInfo.getMetrics().stream()
.filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
}
}
}
}
private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
DataSetSchema dataSetSchema, User user) {
private SemanticQueryResp getQueryResultWithSchemaResp(
EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
semanticParseInfo.setQueryType(QueryType.DETAIL);
@@ -532,7 +585,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
semanticParseInfo.setDateInfo(dateInfo);
}
//add filter
// add filter
QueryFilter chatFilter = getQueryFilter(entityInfo);
Set<QueryFilter> chatFilters = Sets.newHashSet();
chatFilters.add(chatFilter);
@@ -568,5 +621,4 @@ public class S2SemanticLayerService implements SemanticLayerService {
private String getEntityPrimaryName(EntityInfo entityInfo) {
return entityInfo.getDataSetInfo().getPrimaryKey();
}
}

7
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/MetaEmbeddingListener.java
View File

@@ -21,11 +21,9 @@ import java.util.List;
@Slf4j
public class MetaEmbeddingListener implements ApplicationListener<DataEvent> {
@Autowired
private EmbeddingConfig embeddingConfig;
@Autowired private EmbeddingConfig embeddingConfig;
@Autowired
private EmbeddingService embeddingService;
@Autowired private EmbeddingService embeddingService;
@Value("${s2.embedding.operation.sleep.time:3000}")
private Integer embeddingOperationSleepTime;
@@ -59,5 +57,4 @@ public class MetaEmbeddingListener implements ApplicationListener<DataEvent> {
log.error("", e);
}
}
}

42
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java
View File

@@ -22,24 +22,28 @@ public class SchemaDictUpdateListener implements ApplicationListener<DataEvent>
if (CollectionUtils.isEmpty(dataEvent.getDataItems())) {
return;
}
dataEvent.getDataItems().forEach(dataItem -> {
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());
String sign = DictWordType.NATURE_SPILT;
String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);
if (EventType.ADD.equals(dataEvent.getEventType())) {
HanlpHelper.addToCustomDictionary(dictWord);
} else if (EventType.DELETE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
} else if (EventType.UPDATE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
dictWord.setWord(dataItem.getNewName());
HanlpHelper.addToCustomDictionary(dictWord);
}
});
dataEvent
.getDataItems()
.forEach(
dataItem -> {
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());
String sign = DictWordType.NATURE_SPILT;
String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
String nature =
sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);
if (EventType.ADD.equals(dataEvent.getEventType())) {
HanlpHelper.addToCustomDictionary(dictWord);
} else if (EventType.DELETE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
} else if (EventType.UPDATE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
dictWord.setWord(dataItem.getNewName());
HanlpHelper.addToCustomDictionary(dictWord);
}
});
}
}

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/DimensionYamlManager.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.manager;
import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
@@ -13,9 +12,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* manager to handle the dimension
*/
/** manager to handle the dimension */
@Slf4j
@Service
public class DimensionYamlManager {
@@ -25,8 +22,14 @@ public class DimensionYamlManager {
return new ArrayList<>();
}
return dimensions.stream()
.filter(dimension -> !dimension.getType().name().equalsIgnoreCase(IdentifyType.primary.name()))
.map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList());
.filter(
dimension ->
!dimension
.getType()
.name()
.equalsIgnoreCase(IdentifyType.primary.name()))
.map(DimensionYamlManager::convert2DimensionYamlTpl)
.collect(Collectors.toList());
}
public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) {
@@ -36,5 +39,4 @@ public class DimensionYamlManager {
dimensionYamlTpl.setOwners(dimension.getCreatedBy());
return dimensionYamlTpl;
}
}

13
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java
View File

@@ -1,13 +1,13 @@
package com.tencent.supersonic.headless.server.manager;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.FieldParam;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
@@ -22,9 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* manager to handle the metric
*/
/** manager to handle the metric */
@Slf4j
@Service
public class MetricYamlManager {
@@ -46,7 +44,8 @@ public class MetricYamlManager {
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
if (MetricDefineType.MEASURE.equals(metric.getMetricDefineType())) {
MetricDefineByMeasureParams metricDefineParams = metric.getMetricDefineByMeasureParams();
MetricDefineByMeasureParams metricDefineParams =
metric.getMetricDefineByMeasureParams();
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
List<MeasureParam> measures = metricDefineParams.getMeasures();
metricTypeParamsYamlTpl.setMeasures(
@@ -58,7 +57,8 @@ public class MetricYamlManager {
metricTypeParamsYamlTpl.setFields(
fields.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
} else if (MetricDefineType.METRIC.equals(metric.getMetricDefineType())) {
MetricDefineByMetricParams metricDefineByMetricParams = metric.getMetricDefineByMetricParams();
MetricDefineByMetricParams metricDefineByMetricParams =
metric.getMetricDefineByMetricParams();
metricTypeParamsYamlTpl.setExpr(metricDefineByMetricParams.getExpr());
List<MetricParam> metrics = metricDefineByMetricParams.getMetrics();
metricTypeParamsYamlTpl.setMetrics(
@@ -88,5 +88,4 @@ public class MetricYamlManager {
metricParamYamlTpl.setId(metricParam.getId());
return metricParamYamlTpl;
}
}

29
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
View File

@@ -23,14 +23,13 @@ import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* manager to handle the model
*/
/** manager to handle the model */
@Service
@Slf4j
public class ModelYamlManager {
public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) {
public static synchronized DataModelYamlTpl convert2YamlObj(
ModelResp modelResp, DatabaseResp databaseResp) {
ModelDetail modelDetail = modelResp.getModelDetail();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
@@ -38,12 +37,18 @@ public class ModelYamlManager {
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
dataModelYamlTpl.setType(databaseResp.getType());
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);
dataModelYamlTpl.setIdentifiers(modelDetail.getIdentifiers().stream().map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setDimensions(modelDetail.getDimensions().stream().map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setMeasures(modelDetail.getMeasures().stream().map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setIdentifiers(
modelDetail.getIdentifiers().stream()
.map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setDimensions(
modelDetail.getDimensions().stream()
.map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setMeasures(
modelDetail.getMeasures().stream()
.map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setName(modelResp.getBizName());
dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
if (modelDetail.getQueryType().equalsIgnoreCase(ModelDefineType.SQL_QUERY.getName())) {
@@ -64,7 +69,8 @@ public class ModelYamlManager {
dimensionYamlTpl.setExpr(dim.getBizName());
}
if (dim.getTypeParams() != null) {
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl = new DimensionTimeTypeParamsTpl();
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl =
new DimensionTimeTypeParamsTpl();
dimensionTimeTypeParamsTpl.setIsPrimary(dim.getTypeParams().getIsPrimary());
dimensionTimeTypeParamsTpl.setTimeGranularity(dim.getTypeParams().getTimeGranularity());
dimensionYamlTpl.setTypeParams(dimensionTimeTypeParamsTpl);
@@ -97,5 +103,4 @@ public class ModelYamlManager {
measure.setIsCreateMetric(1);
datasourceDetail.getMeasures().add(measure);
}
}

181
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -29,8 +29,8 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.stereotype.Service;
@@ -47,7 +47,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
@Service
public class SemanticSchemaManager {
@@ -65,16 +64,25 @@ public class SemanticSchemaManager {
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
Map<Long, String> modelIdName = new HashMap<>();
schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls,
dataModelYamlTpls, metricYamlTpls, modelIdName);
schemaService.getSchemaYamlTpl(
semanticSchemaResp,
dimensionYamlTpls,
dataModelYamlTpls,
metricYamlTpls,
modelIdName);
DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp();
semanticModel.setDatabase(DatabaseConverter.convert(databaseResp));
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) {
semanticModel.setJoinRelations(getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
semanticModel.setJoinRelations(
getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource)
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
Map<String, DataSource> dataSourceMap =
dataModelYamlTpls.stream()
.map(SemanticSchemaManager::getDatasource)
.collect(
Collectors.toMap(
DataSource::getName, item -> item, (k1, k2) -> k1));
semanticModel.setDatasourceMap(dataSourceMap);
}
if (!dimensionYamlTpls.isEmpty()) {
@@ -90,12 +98,13 @@ public class SemanticSchemaManager {
return semanticModel;
}
public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception {
public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp)
throws Exception {
if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) {
throw new Exception("semanticSchemaResp tag is empty");
}
SemanticModel semanticModel = getSemanticModel(semanticSchemaResp);
//Map<String, List<Dimension>> dimensions = new HashMap<>();
// Map<String, List<Dimension>> dimensions = new HashMap<>();
Map<Long, List<TagResp>> tagMap = new HashMap<>();
for (TagResp tagResp : semanticSchemaResp.getTags()) {
if (!tagMap.containsKey(tagResp.getModelId())) {
@@ -103,8 +112,10 @@ public class SemanticSchemaManager {
}
tagMap.get(tagResp.getModelId()).add(tagResp);
}
if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) {
for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap().entrySet()) {
if (Objects.nonNull(semanticModel.getDatasourceMap())
&& !semanticModel.getDatasourceMap().isEmpty()) {
for (Map.Entry<String, DataSource> entry :
semanticModel.getDatasourceMap().entrySet()) {
List<Dimension> modelDimensions = new ArrayList<>();
if (!semanticModel.getDimensionMap().containsKey(entry.getKey())) {
semanticModel.getDimensionMap().put(entry.getKey(), modelDimensions);
@@ -122,16 +133,18 @@ public class SemanticSchemaManager {
return semanticModel;
}
private void addTagModel(TagResp tagResp, List<Dimension> modelDimensions, List<Metric> modelMetrics)
private void addTagModel(
TagResp tagResp, List<Dimension> modelDimensions, List<Metric> modelMetrics)
throws Exception {
TagDefineType tagDefineType = TagDefineType.valueOf(tagResp.getTagDefineType());
switch (tagDefineType) {
case FIELD:
case DIMENSION:
if (TagDefineType.DIMENSION.equals(tagResp.getTagDefineType())) {
Optional<Dimension> modelDimension = modelDimensions.stream()
// .filter(d -> d.getBizName().equals(tagResp.getExpr()))
.findFirst();
Optional<Dimension> modelDimension =
modelDimensions.stream()
// .filter(d -> d.getBizName().equals(tagResp.getExpr()))
.findFirst();
if (modelDimension.isPresent()) {
modelDimension.get().setName(tagResp.getBizName());
return;
@@ -152,13 +165,15 @@ public class SemanticSchemaManager {
modelDimensions.add(dimension);
return;
case METRIC:
Optional<Metric> modelMetric = modelMetrics.stream()
// .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr()))
.findFirst();
Optional<Metric> modelMetric =
modelMetrics.stream()
// .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr()))
.findFirst();
if (modelMetric.isPresent()) {
modelMetric.get().setName(tagResp.getBizName());
} else {
throw new Exception(String.format("tag [{}] cant find the metric", tagResp.getBizName()));
throw new Exception(
String.format("tag [{}] cant find the metric", tagResp.getBizName()));
}
return;
default:
@@ -174,21 +189,37 @@ public class SemanticSchemaManager {
}
public static DataSource getDatasource(final DataModelYamlTpl d) {
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId())
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery())
.identifiers(getIdentify(d.getIdentifiers())).measures(getMeasureParams(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions())).build();
DataSource datasource =
DataSource.builder()
.id(d.getId())
.sourceId(d.getSourceId())
.type(d.getType())
.sqlQuery(d.getSqlQuery())
.name(d.getName())
.tableQuery(d.getTableQuery())
.identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasureParams(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions()))
.build();
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
}
if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
Set<String> measures = datasource.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet());
Set<String> measures =
datasource.getMeasures().stream()
.map(mm -> mm.getName())
.collect(Collectors.toSet());
for (Field f : d.getFields()) {
if (!measures.contains(f.getFieldName())) {
datasource.getMeasures()
.add(Measure.builder().expr(f.getFieldName()).name(f.getFieldName()).agg("").build());
datasource
.getMeasures()
.add(
Measure.builder()
.expr(f.getFieldName())
.name(f.getFieldName())
.agg("")
.build());
}
}
}
@@ -196,9 +227,12 @@ public class SemanticSchemaManager {
}
private static String getDataSourceAggTime(List<Dimension> dimensions) {
Optional<Dimension> timeDimension = dimensions.stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType())).findFirst();
if (timeDimension.isPresent() && Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
Optional<Dimension> timeDimension =
dimensions.stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
if (timeDimension.isPresent()
&& Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
}
return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE;
@@ -217,7 +251,8 @@ public class SemanticSchemaManager {
return metrics;
}
private static MetricTypeParams getMetricTypeParams(MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
private static MetricTypeParams getMetricTypeParams(
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
MetricTypeParams metricTypeParams = new MetricTypeParams();
metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
metricTypeParams.setFieldMetric(false);
@@ -294,7 +329,8 @@ public class SemanticSchemaManager {
if (Objects.nonNull(dimensionYamlTpl.getExt())) {
dimension.setExt(dimensionYamlTpl.getExt());
}
dimension.setDimensionTimeTypeParams(getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams()));
dimension.setDimensionTimeTypeParams(
getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams()));
dimensions.add(dimension);
}
return dimensions;
@@ -304,7 +340,8 @@ public class SemanticSchemaManager {
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl) {
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
if (dimensionTimeTypeParamsTpl != null) {
dimensionTimeTypeParams.setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity());
dimensionTimeTypeParams.setTimeGranularity(
dimensionTimeTypeParamsTpl.getTimeGranularity());
dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary());
}
return dimensionTimeTypeParams;
@@ -321,24 +358,38 @@ public class SemanticSchemaManager {
return identifies;
}
private static List<JoinRelation> getJoinRelation(List<ModelRela> modelRelas, Map<Long, String> modelIdName) {
private static List<JoinRelation> getJoinRelation(
List<ModelRela> modelRelas, Map<Long, String> modelIdName) {
List<JoinRelation> joinRelations = new ArrayList<>();
modelRelas.stream().forEach(r -> {
if (modelIdName.containsKey(r.getFromModelId()) && modelIdName.containsKey(r.getToModelId())) {
JoinRelation joinRelation = JoinRelation.builder().left(modelIdName.get(r.getFromModelId()))
.right(modelIdName.get(r.getToModelId())).joinType(r.getJoinType()).build();
List<Triple<String, String, String>> conditions = new ArrayList<>();
r.getJoinConditions().stream().forEach(rr -> {
if (FilterOperatorEnum.isValueCompare(rr.getOperator())) {
conditions.add(
Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField()));
}
});
joinRelation.setId(r.getId());
joinRelation.setJoinCondition(conditions);
joinRelations.add(joinRelation);
}
});
modelRelas.stream()
.forEach(
r -> {
if (modelIdName.containsKey(r.getFromModelId())
&& modelIdName.containsKey(r.getToModelId())) {
JoinRelation joinRelation =
JoinRelation.builder()
.left(modelIdName.get(r.getFromModelId()))
.right(modelIdName.get(r.getToModelId()))
.joinType(r.getJoinType())
.build();
List<Triple<String, String, String>> conditions = new ArrayList<>();
r.getJoinConditions().stream()
.forEach(
rr -> {
if (FilterOperatorEnum.isValueCompare(
rr.getOperator())) {
conditions.add(
Triple.of(
rr.getLeftField(),
rr.getOperator().getValue(),
rr.getRightField()));
}
});
joinRelation.setId(r.getId());
joinRelation.setJoinCondition(conditions);
joinRelations.add(joinRelation);
}
});
return joinRelations;
}
@@ -348,11 +399,14 @@ public class SemanticSchemaManager {
}
}
public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) throws Exception {
public static void update(SemanticSchema schema, DataSource datasourceYamlTpl)
throws Exception {
if (schema != null) {
String dataSourceName = datasourceYamlTpl.getName();
Optional<Entry<String, DataSource>> datasourceYamlTplMap = schema.getDatasource().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
Optional<Entry<String, DataSource>> datasourceYamlTplMap =
schema.getDatasource().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName))
.findFirst();
if (datasourceYamlTplMap.isPresent()) {
datasourceYamlTplMap.get().setValue(datasourceYamlTpl);
} else {
@@ -361,11 +415,14 @@ public class SemanticSchemaManager {
}
}
public static void update(SemanticSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
public static void update(
SemanticSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
throws Exception {
if (schema != null) {
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema.getDimension().entrySet()
.stream().filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap =
schema.getDimension().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName))
.findFirst();
if (datasourceYamlTplMap.isPresent()) {
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
} else {
@@ -376,11 +433,13 @@ public class SemanticSchemaManager {
}
}
private static void updateDimension(List<Dimension> dimensionYamlTpls, List<Dimension> dimensions) {
private static void updateDimension(
List<Dimension> dimensionYamlTpls, List<Dimension> dimensions) {
if (CollectionUtils.isEmpty(dimensionYamlTpls)) {
return;
}
Set<String> toAdd = dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
Set<String> toAdd =
dimensionYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
Iterator<Dimension> iterator = dimensions.iterator();
while (iterator.hasNext()) {
Dimension cur = iterator.next();
@@ -395,7 +454,8 @@ public class SemanticSchemaManager {
if (CollectionUtils.isEmpty(metricYamlTpls)) {
return;
}
Set<String> toAdd = metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
Set<String> toAdd =
metricYamlTpls.stream().map(m -> m.getName()).collect(Collectors.toSet());
Iterator<Metric> iterator = metrics.iterator();
while (iterator.hasNext()) {
Metric cur = iterator.next();
@@ -405,5 +465,4 @@ public class SemanticSchemaManager {
}
metrics.addAll(metricYamlTpls);
}
}
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/AppDO.java
View File

@@ -37,5 +37,4 @@ public class AppDO {
private Date createdAt;
private Date updatedAt;
}

26
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CanvasDO.java
View File

@@ -10,38 +10,28 @@ import java.util.Date;
@Data
@TableName("s2_canvas")
public class CanvasDO {
/**
*/
/** */
@TableId(type = IdType.AUTO)
private Long id;
/**
*/
/** */
private Long domainId;
/**
* datasource、dimension、metric
*/
/** datasource、dimension、metric */
private String type;
/**
*/
/** */
private Date createdAt;
/**
*/
/** */
private String createdBy;
/**
*/
/** */
private Date updatedAt;
/**
*/
/** */
private String updatedBy;
/**
* config detail
*/
/** config detail */
private String config;
}

13
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ClassDO.java
View File

@@ -22,21 +22,16 @@ public class ClassDO {
private String description;
private Long parentId;
/**
* 分类状态
*/
/** 分类状态 */
private Integer status;
/**
* METRIC、DIMENSION、TAG
*/
/** METRIC、DIMENSION、TAG */
private String type;
private String itemIds;
private Date createdAt;
private String createdBy;
private Date updatedAt;
private String updatedBy;
}
}

26
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/CollectDO.java
View File

@@ -12,9 +12,7 @@ import java.io.Serializable;
import java.time.LocalDateTime;
/**
* <p>
* 收藏项表
* </p>
*
* @author yannsu
* @since 2023-11-09 03:49:33
@@ -26,39 +24,27 @@ public class CollectDO implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 收藏项ID
*/
/** 收藏项ID */
@TableId(value = "id", type = IdType.AUTO)
private Long id;
/**
* 收藏的类型 metric,dimension,tag
*/
/** 收藏的类型 metric,dimension,tag */
@TableField("type")
private String type;
/**
* 收藏人
*/
/** 收藏人 */
@TableField("username")
private String username;
/**
* 收藏ID
*/
/** 收藏ID */
@TableField("collect_id")
private Long collectId;
/**
* 创建时间
*/
/** 创建时间 */
@TableField(value = "create_time", fill = FieldFill.INSERT)
private LocalDateTime createTime;
/**
* 更新时间
*/
/** 更新时间 */
@TableField(value = "update_time", fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DataSetDO.java
View File

@@ -3,9 +3,10 @@ package com.tencent.supersonic.headless.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.util.Date;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_data_set")
public class DataSetDO {

106
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDO.java
View File

@@ -10,85 +10,56 @@ import java.util.Date;
@Data
@TableName("s2_database")
public class DatabaseDO {
/**
*
*/
/** */
@TableId(type = IdType.AUTO)
private Long id;
/**
* 名称
*/
/** 名称 */
private String name;
/**
* 描述
*/
/** 描述 */
private String description;
/**
*
*/
/** */
private String version;
/**
* 类型 mysql,clickhouse,tdw
*/
/** 类型 mysql,clickhouse,tdw */
private String type;
/**
* 创建时间
*/
/** 创建时间 */
private Date createdAt;
/**
* 创建人
*/
/** 创建人 */
private String createdBy;
/**
* 更新时间
*/
/** 更新时间 */
private Date updatedAt;
/**
* 更新人
*/
/** 更新人 */
private String updatedBy;
/**
*
*/
/** */
private String admin;
/**
*
*/
/** */
private String viewer;
/**
* 配置信息
*/
/** 配置信息 */
private String config;
/**
*
* @return id
*/
/** @return id */
public Long getId() {
return id;
}
/**
*
* @param id
*/
/** @param id */
public void setId(Long id) {
this.id = id;
}
/**
* 名称
*
* @return name 名称
*/
public String getName() {
@@ -97,6 +68,7 @@ public class DatabaseDO {
/**
* 名称
*
* @param name 名称
*/
public void setName(String name) {
@@ -105,6 +77,7 @@ public class DatabaseDO {
/**
* 描述
*
* @return description 描述
*/
public String getDescription() {
@@ -113,30 +86,26 @@ public class DatabaseDO {
/**
* 描述
*
* @param description 描述
*/
public void setDescription(String description) {
this.description = description == null ? null : description.trim();
}
/**
*
* @return version
*/
/** @return version */
public String getVersion() {
return version;
}
/**
*
* @param version
*/
/** @param version */
public void setVersion(String version) {
this.version = version == null ? null : version.trim();
}
/**
* 类型 mysql,clickhouse,tdw
*
* @return type 类型 mysql,clickhouse,tdw
*/
public String getType() {
@@ -145,6 +114,7 @@ public class DatabaseDO {
/**
* 类型 mysql,clickhouse,tdw
*
* @param type 类型 mysql,clickhouse,tdw
*/
public void setType(String type) {
@@ -153,6 +123,7 @@ public class DatabaseDO {
/**
* 创建时间
*
* @return created_at 创建时间
*/
public Date getCreatedAt() {
@@ -161,6 +132,7 @@ public class DatabaseDO {
/**
* 创建时间
*
* @param createdAt 创建时间
*/
public void setCreatedAt(Date createdAt) {
@@ -169,6 +141,7 @@ public class DatabaseDO {
/**
* 创建人
*
* @return created_by 创建人
*/
public String getCreatedBy() {
@@ -177,6 +150,7 @@ public class DatabaseDO {
/**
* 创建人
*
* @param createdBy 创建人
*/
public void setCreatedBy(String createdBy) {
@@ -185,6 +159,7 @@ public class DatabaseDO {
/**
* 更新时间
*
* @return updated_at 更新时间
*/
public Date getUpdatedAt() {
@@ -193,6 +168,7 @@ public class DatabaseDO {
/**
* 更新时间
*
* @param updatedAt 更新时间
*/
public void setUpdatedAt(Date updatedAt) {
@@ -201,6 +177,7 @@ public class DatabaseDO {
/**
* 更新人
*
* @return updated_by 更新人
*/
public String getUpdatedBy() {
@@ -209,46 +186,36 @@ public class DatabaseDO {
/**
* 更新人
*
* @param updatedBy 更新人
*/
public void setUpdatedBy(String updatedBy) {
this.updatedBy = updatedBy == null ? null : updatedBy.trim();
}
/**
*
* @return admin
*/
/** @return admin */
public String getAdmin() {
return admin;
}
/**
*
* @param admin
*/
/** @param admin */
public void setAdmin(String admin) {
this.admin = admin == null ? null : admin.trim();
}
/**
*
* @return viewer
*/
/** @return viewer */
public String getViewer() {
return viewer;
}
/**
*
* @param viewer
*/
/** @param viewer */
public void setViewer(String viewer) {
this.viewer = viewer == null ? null : viewer.trim();
}
/**
* 配置信息
*
* @return config 配置信息
*/
public String getConfig() {
@@ -257,9 +224,10 @@ public class DatabaseDO {
/**
* 配置信息
*
* @param config 配置信息
*/
public void setConfig(String config) {
this.config = config == null ? null : config.trim();
}
}
}

119
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DatabaseDOExample.java
View File

@@ -5,101 +5,64 @@ import java.util.Date;
import java.util.List;
public class DatabaseDOExample {
/**
* s2_database
*/
/** s2_database */
protected String orderByClause;
/**
* s2_database
*/
/** s2_database */
protected boolean distinct;
/**
* s2_database
*/
/** s2_database */
protected List<Criteria> oredCriteria;
/**
* s2_database
*/
/** s2_database */
protected Integer limitStart;
/**
* s2_database
*/
/** s2_database */
protected Integer limitEnd;
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public DatabaseDOExample() {
oredCriteria = new ArrayList<Criteria>();
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public void setOrderByClause(String orderByClause) {
this.orderByClause = orderByClause;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public String getOrderByClause() {
return orderByClause;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public boolean isDistinct() {
return distinct;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public List<Criteria> getOredCriteria() {
return oredCriteria;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public void or(Criteria criteria) {
oredCriteria.add(criteria);
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public Criteria or() {
Criteria criteria = createCriteriaInternal();
oredCriteria.add(criteria);
return criteria;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public Criteria createCriteria() {
Criteria criteria = createCriteriaInternal();
if (oredCriteria.size() == 0) {
@@ -108,60 +71,40 @@ public class DatabaseDOExample {
return criteria;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
protected Criteria createCriteriaInternal() {
Criteria criteria = new Criteria();
return criteria;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public void clear() {
oredCriteria.clear();
orderByClause = null;
distinct = false;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public void setLimitStart(Integer limitStart) {
this.limitStart=limitStart;
this.limitStart = limitStart;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public Integer getLimitStart() {
return limitStart;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public void setLimitEnd(Integer limitEnd) {
this.limitEnd=limitEnd;
this.limitEnd = limitEnd;
}
/**
*
* @mbg.generated
*/
/** @mbg.generated */
public Integer getLimitEnd() {
return limitEnd;
}
/**
* s2_database null
*/
/** s2_database null */
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
@@ -196,7 +139,8 @@ public class DatabaseDOExample {
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(String condition, Object value1, Object value2, String property) {
protected void addCriterion(
String condition, Object value1, Object value2, String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
@@ -944,9 +888,7 @@ public class DatabaseDOExample {
}
}
/**
* s2_database
*/
/** s2_database */
public static class Criteria extends GeneratedCriteria {
protected Criteria() {
@@ -954,9 +896,7 @@ public class DatabaseDOExample {
}
}
/**
* s2_database null
*/
/** s2_database null */
public static class Criterion {
private String condition;
@@ -1029,7 +969,8 @@ public class DatabaseDOExample {
this(condition, value, null);
}
protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
protected Criterion(
String condition, Object value, Object secondValue, String typeHandler) {
super();
this.condition = condition;
this.value = value;
@@ -1042,4 +983,4 @@ public class DatabaseDOExample {
this(condition, value, secondValue, null);
}
}
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DateInfoDO.java
View File

@@ -15,6 +15,4 @@ public class DateInfoDO {
private String createdBy;
private String updatedBy;
private String datePeriod;
}
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictConfDO.java
View File

@@ -12,6 +12,7 @@ import java.util.Date;
public class DictConfDO {
@TableId(type = IdType.AUTO)
private Long id;
private String description;
private String type;
private Long itemId;
@@ -19,4 +20,4 @@ public class DictConfDO {
private String status;
private Date createdAt;
private String createdBy;
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DictTaskDO.java
View File

@@ -12,6 +12,7 @@ import java.util.Date;
public class DictTaskDO {
@TableId(type = IdType.AUTO)
private Long id;
private String name;
private String description;
private String type;
@@ -21,5 +22,4 @@ public class DictTaskDO {
private Date createdAt;
private String createdBy;
private Long elapsedMs;
}
}

57
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDO.java
View File

@@ -10,75 +10,46 @@ import java.util.Date;
@Data
@TableName("s2_domain")
public class DomainDO {
/**
* 自增ID
*/
/** 自增ID */
@TableId(type = IdType.AUTO)
private Long id;
/**
* 主题域名称
*/
/** 主题域名称 */
private String name;
/**
* 内部名称
*/
/** 内部名称 */
private String bizName;
/**
* 父主题域ID
*/
/** 父主题域ID */
private Long parentId;
/**
* 主题域状态
*/
/** 主题域状态 */
private Integer status;
/**
* 创建时间
*/
/** 创建时间 */
private Date createdAt;
/**
* 创建人
*/
/** 创建人 */
private String createdBy;
/**
* 更新时间
*/
/** 更新时间 */
private Date updatedAt;
/**
* 更新人
*/
/** 更新人 */
private String updatedBy;
/**
* 主题域管理员
*/
/** 主题域管理员 */
private String admin;
/**
* 主题域管理员组织
*/
/** 主题域管理员组织 */
private String adminOrg;
/**
* 主题域是否公开
*/
/** 主题域是否公开 */
private Integer isOpen;
/**
* 主题域可用用户
*/
/** 主题域可用用户 */
private String viewer;
/**
* 主题域可用组织
*/
/** 主题域可用组织 */
private String viewOrg;
}

98
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/DomainDOExample.java
View File

@@ -5,92 +5,64 @@ import java.util.Date;
import java.util.List;
public class DomainDOExample {
/**
* s2_domain
*/
/** s2_domain */
protected String orderByClause;
/**
* s2_domain
*/
/** s2_domain */
protected boolean distinct;
/**
* s2_domain
*/
/** s2_domain */
protected List<Criteria> oredCriteria;
/**
* s2_domain
*/
/** s2_domain */
protected Integer limitStart;
/**
* s2_domain
*/
/** s2_domain */
protected Integer limitEnd;
/**
* @mbg.generated
*/
/** @mbg.generated */
public DomainDOExample() {
oredCriteria = new ArrayList<Criteria>();
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public void setOrderByClause(String orderByClause) {
this.orderByClause = orderByClause;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public String getOrderByClause() {
return orderByClause;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public boolean isDistinct() {
return distinct;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public List<Criteria> getOredCriteria() {
return oredCriteria;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public void or(Criteria criteria) {
oredCriteria.add(criteria);
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public Criteria or() {
Criteria criteria = createCriteriaInternal();
oredCriteria.add(criteria);
return criteria;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public Criteria createCriteria() {
Criteria criteria = createCriteriaInternal();
if (oredCriteria.size() == 0) {
@@ -99,54 +71,40 @@ public class DomainDOExample {
return criteria;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
protected Criteria createCriteriaInternal() {
Criteria criteria = new Criteria();
return criteria;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public void clear() {
oredCriteria.clear();
orderByClause = null;
distinct = false;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public void setLimitStart(Integer limitStart) {
this.limitStart = limitStart;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public Integer getLimitStart() {
return limitStart;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public void setLimitEnd(Integer limitEnd) {
this.limitEnd = limitEnd;
}
/**
* @mbg.generated
*/
/** @mbg.generated */
public Integer getLimitEnd() {
return limitEnd;
}
/**
* s2_domain null
*/
/** s2_domain null */
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
@@ -181,7 +139,8 @@ public class DomainDOExample {
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(String condition, Object value1, Object value2, String property) {
protected void addCriterion(
String condition, Object value1, Object value2, String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
@@ -1109,9 +1068,7 @@ public class DomainDOExample {
}
}
/**
* s2_domain
*/
/** s2_domain */
public static class Criteria extends GeneratedCriteria {
protected Criteria() {
@@ -1119,9 +1076,7 @@ public class DomainDOExample {
}
}
/**
* s2_domain null
*/
/** s2_domain null */
public static class Criterion {
private String condition;
@@ -1162,7 +1117,8 @@ public class DomainDOExample {
this(condition, value, null);
}
protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
protected Criterion(
String condition, Object value, Object secondValue, String typeHandler) {
super();
this.condition = condition;
this.value = value;

68
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricDO.java
View File

@@ -15,89 +15,55 @@ public class MetricDO {
@TableId(type = IdType.AUTO)
private Long id;
/**
* 主体域ID
*/
/** 主体域ID */
private Long modelId;
/**
* 指标名称
*/
/** 指标名称 */
private String name;
/**
* 字段名称
*/
/** 字段名称 */
private String bizName;
/**
* 描述
*/
/** 描述 */
private String description;
/**
* 指标状态,0正常,1下架,2删除
*/
/** 指标状态,0正常,1下架,2删除 */
private Integer status;
/**
* 敏感级别
*/
/** 敏感级别 */
private Integer sensitiveLevel;
/**
* 指标类型 proxy,expr
*/
/** 指标类型 proxy,expr */
private String type;
/**
* 创建时间
*/
/** 创建时间 */
private Date createdAt;
/**
* 创建人
*/
/** 创建人 */
private String createdBy;
/**
* 更新时间
*/
/** 更新时间 */
private Date updatedAt;
/**
* 更新人
*/
/** 更新人 */
private String updatedBy;
/**
* 数值类型
*/
/** 数值类型 */
private String dataFormatType;
/**
* 数值类型参数
*/
/** 数值类型参数 */
private String dataFormat;
/**
*
*/
/** */
private String alias;
/**
*
*/
/** */
private String classifications;
/**
*
*/
/** */
private String relateDimensions;
/**
* 类型参数
*/
/** 类型参数 */
private String typeParams;
private String ext;

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/MetricQueryDefaultConfigDO.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
@@ -12,7 +11,6 @@ import java.util.Date;
@TableName("s2_metric_query_default_config")
public class MetricQueryDefaultConfigDO {
@TableId(value = "id", type = IdType.AUTO)
private Long id;
@@ -35,5 +33,4 @@ public class MetricQueryDefaultConfigDO {
private Date updatedAt;
private String updatedBy;
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelDO.java
View File

@@ -59,5 +59,4 @@ public class ModelDO {
private String sourceType;
private String ext;
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/ModelRelaDO.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
@@ -22,5 +21,4 @@ public class ModelRelaDO {
private String joinType;
private String joinCondition;
}

60
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/QueryRuleDO.java
View File

@@ -14,75 +14,45 @@ public class QueryRuleDO {
@TableId(type = IdType.AUTO)
private Long id;
/**
* dataSetID
*/
/** dataSetID */
private Long dataSetId;
/**
* 规则的优先级, 0-系统默认规则
*/
/** 规则的优先级, 0-系统默认规则 */
private Integer priority;
/**
* 规则类型
*/
/** 规则类型 */
private String ruleType;
/**
* 规则名称
*/
/** 规则名称 */
private String name;
/**
* 规则业务名称
*/
/** 规则业务名称 */
private String bizName;
/**
* 描述
*/
/** 描述 */
private String description;
/**
* 具体规则信息
*/
/** 具体规则信息 */
private String rule;
/**
* 规则输出信息
*/
/** 规则输出信息 */
private String action;
/**
* 状态,0-正常,1-下线,2-删除
*/
/** 状态,0-正常,1-下线,2-删除 */
private Integer status;
/**
* 创建时间
*/
/** 创建时间 */
private Date createdAt;
/**
* 创建人
*/
/** 创建人 */
private String createdBy;
/**
* 更新时间
*/
/** 更新时间 */
private Date updatedAt;
/**
* 更新人
*/
/** 更新人 */
private String updatedBy;
/**
* 扩展信息
*/
/** 扩展信息 */
private String ext;
}
}

28
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java
View File

@@ -3,9 +3,10 @@ package com.tencent.supersonic.headless.server.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.util.Date;
import lombok.Data;
import java.util.Date;
@Data
@TableName("s2_tag")
public class TagDO {
@@ -13,34 +14,21 @@ public class TagDO {
@TableId(type = IdType.AUTO)
private Long id;
/**
* itemID
*/
/** itemID */
private Long itemId;
/**
* 标签类型
*/
/** 标签类型 */
private String type;
/**
* 创建时间
*/
/** 创建时间 */
private Date createdAt;
/**
* 创建人
*/
/** 创建人 */
private String createdBy;
/**
* 更新时间
*/
/** 更新时间 */
private Date updatedAt;
/**
* 更新人
*/
/** 更新人 */
private String updatedBy;
}

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagObjectDO.java
View File

@@ -14,58 +14,36 @@ public class TagObjectDO {
@TableId(type = IdType.AUTO)
private Long id;
/**
* 关联到某个主题域下
*/
/** 关联到某个主题域下 */
private Long domainId;
/**
* 标签对象名称
*/
/** 标签对象名称 */
private String name;
/**
* 标签对象业务名称
*/
/** 标签对象业务名称 */
private String bizName;
/**
* 描述
*/
/** 描述 */
private String description;
/**
* 状态,0正常,1下架,2删除
*/
/** 状态,0正常,1下架,2删除 */
private Integer status;
/**
* 敏感级别
*/
/** 敏感级别 */
private Integer sensitiveLevel;
/**
* 创建时间
*/
/** 创建时间 */
private Date createdAt;
/**
* 创建人
*/
/** 创建人 */
private String createdBy;
/**
* 更新时间
*/
/** 更新时间 */
private Date updatedAt;
/**
* 更新人
*/
/** 更新人 */
private String updatedBy;
/**
* 扩展信息
*/
/** 扩展信息 */
private String ext;
}
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TermDO.java
View File

@@ -33,5 +33,4 @@ public class TermDO {
private Date updatedAt;
private String updatedBy;
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/AppMapper.java
View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.AppDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface AppMapper extends BaseMapper<AppDO> {
}
public interface AppMapper extends BaseMapper<AppDO> {}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CanvasDOMapper.java
View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface CanvasDOMapper extends BaseMapper<CanvasDO> {
}
public interface CanvasDOMapper extends BaseMapper<CanvasDO> {}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ClassMapper.java
View File

@@ -5,5 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ClassDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ClassMapper extends BaseMapper<ClassDO> {
}
public interface ClassMapper extends BaseMapper<ClassDO> {}

7
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/CollectMapper.java
View File

@@ -1,19 +1,14 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import org.apache.ibatis.annotations.Mapper;
/**
* <p>
* 收藏项表 Mapper 接口
* </p>
*
* @author yannsu
* @since 2023-11-09 03:49:33
*/
@Mapper
public interface CollectMapper extends BaseMapper<CollectDO> {
}
public interface CollectMapper extends BaseMapper<CollectDO> {}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DataSetDOMapper.java
View File

@@ -5,7 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DataSetDOMapper extends BaseMapper<DataSetDO> {
}
public interface DataSetDOMapper extends BaseMapper<DataSetDO> {}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DatabaseDOMapper.java
View File

@@ -1,11 +1,8 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DatabaseDOMapper extends BaseMapper<DatabaseDO> {
}
public interface DatabaseDOMapper extends BaseMapper<DatabaseDO> {}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DateInfoMapper.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
import org.apache.ibatis.annotations.Mapper;
@@ -13,4 +12,4 @@ public interface DateInfoMapper {
Boolean upsertDateInfo(DateInfoDO dateInfoDO);
List<DateInfoDO> getDateInfos(ItemDateFilter itemDateFilter);
}
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictConfMapper.java
View File

@@ -5,5 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DictConfMapper extends BaseMapper<DictConfDO> {
}
public interface DictConfMapper extends BaseMapper<DictConfDO> {}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DictTaskMapper.java
View File

@@ -1,11 +1,8 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DictTaskMapper extends BaseMapper<DictTaskDO> {
}
public interface DictTaskMapper extends BaseMapper<DictTaskDO> {}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java
View File

@@ -3,9 +3,10 @@ package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface DimensionDOCustomMapper {
@@ -18,5 +19,4 @@ public interface DimensionDOCustomMapper {
List<DimensionDO> query(DimensionFilter dimensionFilter);
List<DimensionDO> queryDimensions(DimensionsFilter dimensionsFilter);
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOMapper.java
View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DimensionDOMapper extends BaseMapper<DimensionDO> {
}
public interface DimensionDOMapper extends BaseMapper<DimensionDO> {}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DomainDOMapper.java
View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DomainDOMapper extends BaseMapper<DomainDO> {
}
public interface DomainDOMapper extends BaseMapper<DomainDO> {}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java
View File

@@ -4,6 +4,7 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOMapper.java
View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface MetricDOMapper extends BaseMapper<MetricDO> {
}
public interface MetricDOMapper extends BaseMapper<MetricDO> {}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricQueryDefaultConfigDOMapper.java
View File

@@ -1,14 +1,8 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface MetricQueryDefaultConfigDOMapper extends BaseMapper<MetricQueryDefaultConfigDO> {
}
public interface MetricQueryDefaultConfigDOMapper extends BaseMapper<MetricQueryDefaultConfigDO> {}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOCustomMapper.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import org.apache.ibatis.annotations.Mapper;
@@ -10,5 +9,4 @@ import java.util.List;
public interface ModelDOCustomMapper {
void batchUpdateStatus(List<ModelDO> modelDOS);
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelDOMapper.java
View File

@@ -5,7 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ModelDOMapper extends BaseMapper<ModelDO> {
}
public interface ModelDOMapper extends BaseMapper<ModelDO> {}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/ModelRelaDOMapper.java
View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ModelRelaDO
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ModelRelaDOMapper extends BaseMapper<ModelRelaDO> {
}
public interface ModelRelaDOMapper extends BaseMapper<ModelRelaDO> {}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/QueryRuleMapper.java
View File

@@ -5,5 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.QueryRuleDO
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface QueryRuleMapper extends BaseMapper<QueryRuleDO> {
}
public interface QueryRuleMapper extends BaseMapper<QueryRuleDO> {}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface StatMapper {

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java
View File

@@ -3,9 +3,10 @@ package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface TagCustomMapper {

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java
View File

@@ -5,6 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TagMapper extends BaseMapper<TagDO> {
}
public interface TagMapper extends BaseMapper<TagDO> {}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagObjectMapper.java
View File

@@ -5,5 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TagObjectDO
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TagObjectMapper extends BaseMapper<TagObjectDO> {
}
public interface TagObjectMapper extends BaseMapper<TagObjectDO> {}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TermMapper.java
View File

@@ -5,7 +5,4 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TermDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TermMapper extends BaseMapper<TermDO> {
}
public interface TermMapper extends BaseMapper<TermDO> {}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DateInfoRepository.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.request.DateInfoReq;
import com.tencent.supersonic.headless.server.persistence.dataobject.DateInfoDO;
@@ -12,5 +11,4 @@ public interface DateInfoRepository {
Integer upsertDateInfo(List<DateInfoReq> dateInfoReqs);
List<DateInfoDO> getDateInfos(ItemDateFilter itemDateFilter);
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DimensionRepository.java
View File

@@ -1,10 +1,9 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import java.util.List;
public interface DimensionRepository {

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/DomainRepository.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
import java.util.List;
@@ -16,5 +15,4 @@ public interface DomainRepository {
List<DomainDO> getDomainList();
DomainDO getDomainById(Long id);
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/MetricRepository.java
View File

@@ -1,11 +1,10 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import java.util.List;
public interface MetricRepository {

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/ModelRepository.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/QueryRuleRepository.java
View File

@@ -14,5 +14,4 @@ public interface QueryRuleRepository {
QueryRuleDO getQueryRuleById(Long id);
List<QueryRuleDO> getQueryRules(QueryRuleFilter filter);
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/StatRepository.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp;
import java.util.List;
public interface StatRepository {
@@ -12,4 +13,4 @@ public interface StatRepository {
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
List<QueryStat> getQueryStatInfoWithoutCache(ItemUseReq itemUseCommend);
}
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java
View File

@@ -1,12 +1,11 @@
package com.tencent.supersonic.headless.server.persistence.repository;
import com.tencent.supersonic.headless.api.pojo.request.TagDeleteReq;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import java.util.List;
import java.util.List;
public interface TagRepository {

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ClassRepositoryImpl.java
View File

@@ -75,4 +75,4 @@ public class ClassRepositoryImpl implements ClassRepository {
QueryWrapper<ClassDO> wrapper = new QueryWrapper();
return mapper.selectList(wrapper);
}
}
}

35
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DateInfoRepositoryImpl.java
View File

@@ -24,11 +24,9 @@ import java.util.concurrent.TimeUnit;
@Repository
public class DateInfoRepositoryImpl implements DateInfoRepository {
private ObjectMapper mapper = new ObjectMapper();
@Autowired
private DateInfoMapper dateInfoMapper;
@Autowired private DateInfoMapper dateInfoMapper;
@Override
public Integer upsertDateInfo(List<DateInfoReq> dateInfoCommends) {
@@ -38,25 +36,30 @@ public class DateInfoRepositoryImpl implements DateInfoRepository {
return 0;
}
dateInfoCommends.stream().forEach(commend -> {
DateInfoDO dateInfoDO = new DateInfoDO();
BeanUtils.copyProperties(commend, dateInfoDO);
try {
dateInfoDO.setUnavailableDateList(mapper.writeValueAsString(commend.getUnavailableDateList()));
dateInfoDO.setCreatedBy(Constants.ADMIN_LOWER);
dateInfoDO.setUpdatedBy(Constants.ADMIN_LOWER);
} catch (JsonProcessingException e) {
log.info("e,", e);
}
dateInfoDOList.add(dateInfoDO);
});
dateInfoCommends.stream()
.forEach(
commend -> {
DateInfoDO dateInfoDO = new DateInfoDO();
BeanUtils.copyProperties(commend, dateInfoDO);
try {
dateInfoDO.setUnavailableDateList(
mapper.writeValueAsString(
commend.getUnavailableDateList()));
dateInfoDO.setCreatedBy(Constants.ADMIN_LOWER);
dateInfoDO.setUpdatedBy(Constants.ADMIN_LOWER);
} catch (JsonProcessingException e) {
log.info("e,", e);
}
dateInfoDOList.add(dateInfoDO);
});
return batchUpsert(dateInfoDOList);
}
@Override
public List<DateInfoDO> getDateInfos(ItemDateFilter itemDateFilter) {
if (Objects.nonNull(itemDateFilter) && CollectionUtils.isEmpty(itemDateFilter.getItemIds())) {
if (Objects.nonNull(itemDateFilter)
&& CollectionUtils.isEmpty(itemDateFilter.getItemIds())) {
return new ArrayList<>();
}
return dateInfoMapper.getDateInfos(itemDateFilter);

27
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.persistence.repository.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
@@ -37,8 +36,11 @@ public class DictRepositoryImpl implements DictRepository {
private final DictUtils dictConverter;
private final DimensionService dimensionService;
public DictRepositoryImpl(DictTaskMapper dictTaskMapper, DictConfMapper dictConfMapper,
DictUtils dictConverter, DimensionService dimensionService) {
public DictRepositoryImpl(
DictTaskMapper dictTaskMapper,
DictConfMapper dictConfMapper,
DictUtils dictConverter,
DimensionService dimensionService) {
this.dictTaskMapper = dictTaskMapper;
this.dictConfMapper = dictConfMapper;
this.dictConverter = dictConverter;
@@ -88,9 +90,11 @@ public class DictRepositoryImpl implements DictRepository {
QueryWrapper<DictTaskDO> wrapper = new QueryWrapper<>();
wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId());
wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType());
List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper).stream()
.sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
.limit(dictTaskNum).collect(Collectors.toList());
List<DictTaskDO> dictTaskDOList =
dictTaskMapper.selectList(wrapper).stream()
.sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
.limit(dictTaskNum)
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(dictTaskDOList)) {
return taskResp;
}
@@ -109,10 +113,11 @@ public class DictRepositoryImpl implements DictRepository {
@Override
public Long editDictConf(DictConfDO dictConfDO) {
DictItemFilter filter = DictItemFilter.builder()
.type(TypeEnums.valueOf(dictConfDO.getType()))
.itemId(dictConfDO.getItemId())
.build();
DictItemFilter filter =
DictItemFilter.builder()
.type(TypeEnums.valueOf(dictConfDO.getType()))
.itemId(dictConfDO.getItemId())
.build();
List<DictConfDO> dictConfDOList = getDictConfDOList(filter);
if (CollectionUtils.isEmpty(dictConfDOList)) {
@@ -146,4 +151,4 @@ public class DictRepositoryImpl implements DictRepository {
List<DictConfDO> dictConfDOList = dictConfMapper.selectList(wrapper);
return dictConfDOList;
}
}
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java
View File

@@ -6,9 +6,10 @@ import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapp
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import java.util.List;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class DimensionRepositoryImpl implements DimensionRepository {
@@ -16,8 +17,8 @@ public class DimensionRepositoryImpl implements DimensionRepository {
private DimensionDOCustomMapper dimensionDOCustomMapper;
public DimensionRepositoryImpl(DimensionDOMapper dimensionDOMapper,
DimensionDOCustomMapper dimensionDOCustomMapper) {
public DimensionRepositoryImpl(
DimensionDOMapper dimensionDOMapper, DimensionDOCustomMapper dimensionDOCustomMapper) {
this.dimensionDOMapper = dimensionDOMapper;
this.dimensionDOCustomMapper = dimensionDOCustomMapper;
}
@@ -56,5 +57,4 @@ public class DimensionRepositoryImpl implements DimensionRepository {
public List<DimensionDO> getDimensions(DimensionsFilter dimensionsFilter) {
return dimensionDOCustomMapper.queryDimensions(dimensionsFilter);
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DomainRepositoryImpl.java
View File

@@ -9,7 +9,6 @@ import org.springframework.stereotype.Component;
import java.util.List;
@Component
@Slf4j
public class DomainRepositoryImpl implements DomainRepository {
@@ -44,5 +43,4 @@ public class DomainRepositoryImpl implements DomainRepository {
public DomainDO getDomainById(Long id) {
return domainDOMapper.selectById(id);
}
}

15
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java
View File

@@ -10,22 +10,22 @@ import com.tencent.supersonic.headless.server.persistence.repository.MetricRepos
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.List;
@Component
public class MetricRepositoryImpl implements MetricRepository {
private MetricDOMapper metricDOMapper;
private MetricDOCustomMapper metricDOCustomMapper;
private MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper;
public MetricRepositoryImpl(MetricDOMapper metricDOMapper,
MetricDOCustomMapper metricDOCustomMapper,
MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper) {
public MetricRepositoryImpl(
MetricDOMapper metricDOMapper,
MetricDOCustomMapper metricDOCustomMapper,
MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper) {
this.metricDOMapper = metricDOMapper;
this.metricDOCustomMapper = metricDOCustomMapper;
this.metricQueryDefaultConfigDOMapper = metricQueryDefaultConfigDOMapper;
@@ -95,9 +95,10 @@ public class MetricRepositoryImpl implements MetricRepository {
@Override
public MetricQueryDefaultConfigDO getDefaultQueryConfig(Long metricId, String userName) {
QueryWrapper<MetricQueryDefaultConfigDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(MetricQueryDefaultConfigDO::getMetricId, metricId)
queryWrapper
.lambda()
.eq(MetricQueryDefaultConfigDO::getMetricId, metricId)
.eq(MetricQueryDefaultConfigDO::getCreatedBy, userName);
return metricQueryDefaultConfigDOMapper.selectOne(queryWrapper);
}
}

7
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/ModelRepositoryImpl.java
View File

@@ -12,17 +12,15 @@ import org.springframework.util.CollectionUtils;
import java.util.List;
@Component
public class ModelRepositoryImpl implements ModelRepository {
private ModelDOMapper modelDOMapper;
private ModelDOCustomMapper modelDOCustomMapper;
public ModelRepositoryImpl(ModelDOMapper modelDOMapper,
ModelDOCustomMapper modelDOCustomMapper) {
public ModelRepositoryImpl(
ModelDOMapper modelDOMapper, ModelDOCustomMapper modelDOCustomMapper) {
this.modelDOMapper = modelDOMapper;
this.modelDOCustomMapper = modelDOCustomMapper;
}
@@ -71,5 +69,4 @@ public class ModelRepositoryImpl implements ModelRepository {
public void batchUpdate(List<ModelDO> modelDOS) {
modelDOCustomMapper.batchUpdateStatus(modelDOS);
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/QueryRuleRepositoryImpl.java
View File

@@ -57,4 +57,4 @@ public class QueryRuleRepositoryImpl implements QueryRuleRepository {
queryRuleDOList.addAll(queryRuleDOListSys);
return queryRuleDOList;
}
}
}

69
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java
View File

@@ -45,22 +45,34 @@ public class StatRepositoryImpl implements StatRepository {
List<ItemUseResp> result = new ArrayList<>();
List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
Map<String, Long> map = new ConcurrentHashMap<>();
statInfos.stream().forEach(stat -> {
String dimensions = stat.getDimensions();
String metrics = stat.getMetrics();
if (Objects.nonNull(stat.getDataSetId())) {
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(), stat.getDataSetId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(), stat.getDataSetId());
}
});
map.forEach((k, v) -> {
Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]);
String type = k.split(AT_SYMBOL + AT_SYMBOL)[1];
String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2];
result.add(new ItemUseResp(classId, type, nameEn, v));
});
statInfos.stream()
.forEach(
stat -> {
String dimensions = stat.getDimensions();
String metrics = stat.getMetrics();
if (Objects.nonNull(stat.getDataSetId())) {
updateStatMapInfo(
map,
dimensions,
TypeEnums.DIMENSION.name().toLowerCase(),
stat.getDataSetId());
updateStatMapInfo(
map,
metrics,
TypeEnums.METRIC.name().toLowerCase(),
stat.getDataSetId());
}
});
map.forEach(
(k, v) -> {
Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]);
String type = k.split(AT_SYMBOL + AT_SYMBOL)[1];
String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2];
result.add(new ItemUseResp(classId, type, nameEn, v));
});
return result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
return result.stream()
.sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
.collect(Collectors.toList());
}
@@ -69,19 +81,24 @@ public class StatRepositoryImpl implements StatRepository {
return statMapper.getStatInfo(itemUseCommend);
}
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type, Long dataSetId) {
private void updateStatMapInfo(
Map<String, Long> map, String dimensions, String type, Long dataSetId) {
if (StringUtils.isNotEmpty(dimensions)) {
try {
List<String> dimensionList = mapper.readValue(dimensions, new TypeReference<List<String>>() {
});
dimensionList.stream().forEach(dimension -> {
String key = dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL + dimension;
if (map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {
map.put(key, 1L);
}
});
List<String> dimensionList =
mapper.readValue(dimensions, new TypeReference<List<String>>() {});
dimensionList.stream()
.forEach(
dimension -> {
String key =
dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL
+ AT_SYMBOL + dimension;
if (map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {
map.put(key, 1L);
}
});
} catch (Exception e) {
log.warn("e:{}", e);
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagObjectRepositoryImpl.java
View File

@@ -60,4 +60,4 @@ public class TagObjectRepositoryImpl implements TagObjectRepository {
}
return tagObjectMapper.selectList(wrapper);
}
}
}

9
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java
View File

@@ -20,8 +20,7 @@ public class TagRepositoryImpl implements TagRepository {
private final TagMapper mapper;
private final TagCustomMapper tagCustomMapper;
public TagRepositoryImpl(TagMapper mapper,
TagCustomMapper tagCustomMapper) {
public TagRepositoryImpl(TagMapper mapper, TagCustomMapper tagCustomMapper) {
this.mapper = mapper;
this.tagCustomMapper = tagCustomMapper;
}
@@ -62,8 +61,10 @@ public class TagRepositoryImpl implements TagRepository {
if (CollectionUtils.isNotEmpty(tagDeleteReq.getIds())) {
tagCustomMapper.deleteBatchByIds(tagDeleteReq.getIds());
}
if (Objects.nonNull(tagDeleteReq.getTagDefineType()) && CollectionUtils.isNotEmpty(tagDeleteReq.getItemIds())) {
tagCustomMapper.deleteBatchByType(tagDeleteReq.getItemIds(), tagDeleteReq.getTagDefineType().name());
if (Objects.nonNull(tagDeleteReq.getTagDefineType())
&& CollectionUtils.isNotEmpty(tagDeleteReq.getItemIds())) {
tagCustomMapper.deleteBatchByType(
tagDeleteReq.getItemIds(), tagDeleteReq.getTagDefineType().name());
}
}
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClassFilter.java
View File

@@ -9,4 +9,4 @@ public class ClassFilter extends MetaFilter {
private String type;
private Long dataSetId;
private Long classId;
}
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ClickHouseParametersBuilder.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.pojo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DataDownload.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.pojo;
import lombok.Builder;
import lombok.Data;
import java.util.List;
@Data
@@ -11,5 +12,4 @@ public class DataDownload {
List<List<String>> headers;
List<List<String>> data;
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DatabaseParameter.java
View File

@@ -1,11 +1,12 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParameterFactory.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.server.pojo;
import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
import java.util.LinkedHashMap;
import java.util.Map;
public class DbParameterFactory {
private static Map<String, DbParametersBuilder> parametersBuilder;
@@ -25,5 +25,4 @@ public class DbParameterFactory {
public static Map<String, DbParametersBuilder> getMap() {
return parametersBuilder;
}
}
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DbParametersBuilder.java
View File

@@ -5,5 +5,4 @@ import java.util.List;
public interface DbParametersBuilder {
List<DatabaseParameter> build();
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionFilter.java
View File

@@ -3,10 +3,8 @@ package com.tencent.supersonic.headless.server.pojo;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import lombok.Data;
@Data
public class DimensionFilter extends MetaFilter {
private Integer isTag;
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/DimensionsFilter.java
View File

@@ -1,8 +1,9 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.List;
import lombok.Data;
import java.util.List;
@Data
public class DimensionsFilter {
@@ -11,5 +12,4 @@ public class DimensionsFilter {
private List<Long> dimensionIds;
private List<String> dimensionNames;
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/H2ParametersBuilder.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.server.pojo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Service
@Slf4j

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MetricsFilter.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.server.pojo;
import java.util.List;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import lombok.Data;
import java.util.List;
@Data
public class MetricsFilter extends MetaFilter {
@@ -13,5 +13,4 @@ public class MetricsFilter extends MetaFilter {
private List<Long> metricIds;
private List<String> metricNames;
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelCluster.java
View File

@@ -21,4 +21,4 @@ public class ModelCluster {
modelCluster.setContainsPartitionDimensions(containsPartitionDimensions);
return modelCluster;
}
}
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/ModelFilter.java
View File

@@ -14,9 +14,7 @@ public class ModelFilter extends MetaFilter {
private Boolean includesDetail = true;
public ModelFilter() {
}
public ModelFilter() {}
public ModelFilter(Boolean includesDetail) {
this.includesDetail = includesDetail;
@@ -26,5 +24,4 @@ public class ModelFilter extends MetaFilter {
this.includesDetail = includesDetail;
this.setIds(ids);
}
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/MysqlParametersBuilder.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.pojo;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/OtherParametersBuilder.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.pojo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/PostgresqlParametersBuilder.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.pojo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagObjectFilter.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.server.pojo;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import lombok.Data;
@@ -10,4 +9,4 @@ import java.util.List;
public class TagObjectFilter extends MetaFilter {
private List<Long> domainIds;
}
}

Some files were not shown because too many files have changed in this diff Show More