mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-14 05:43:51 +00:00
(improvement)[build] Use Spotless to customize the code formatting (#1750)
This commit is contained in:
@@ -7,4 +7,5 @@ import java.lang.annotation.Target;
|
||||
|
||||
@Target({ElementType.PARAMETER, ElementType.METHOD})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface ApiHeaderCheck {}
|
||||
public @interface ApiHeaderCheck {
|
||||
}
|
||||
|
||||
@@ -9,4 +9,5 @@ import java.lang.annotation.Target;
|
||||
@Target(ElementType.METHOD)
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Documented
|
||||
public @interface S2DataPermission {}
|
||||
public @interface S2DataPermission {
|
||||
}
|
||||
|
||||
@@ -30,7 +30,8 @@ public class ApiHeaderCheckAspect {
|
||||
|
||||
private static final String SIGNATURE = "signature";
|
||||
|
||||
@Autowired private AppService appService;
|
||||
@Autowired
|
||||
private AppService appService;
|
||||
|
||||
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck)")
|
||||
private void apiPermissionCheck() {}
|
||||
@@ -63,12 +64,8 @@ public class ApiHeaderCheckAspect {
|
||||
if (!AppStatus.ONLINE.equals(appDetailResp.getAppStatus())) {
|
||||
throw new InvalidArgumentException("该应用暂时为非在线状态");
|
||||
}
|
||||
Pair<Boolean, String> checkResult =
|
||||
SignatureUtils.isValidSignature(
|
||||
appId,
|
||||
appDetailResp.getAppSecret(),
|
||||
Long.parseLong(timestampStr),
|
||||
signature);
|
||||
Pair<Boolean, String> checkResult = SignatureUtils.isValidSignature(appId,
|
||||
appDetailResp.getAppSecret(), Long.parseLong(timestampStr), signature);
|
||||
if (!checkResult.first) {
|
||||
throw new InvalidArgumentException(checkResult.second);
|
||||
}
|
||||
|
||||
@@ -44,10 +44,10 @@ public class DimValueAspect {
|
||||
@Value("${s2.dimension.value.map.enable:true}")
|
||||
private Boolean dimensionValueMapEnable;
|
||||
|
||||
@Autowired private DimensionService dimensionService;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
|
||||
@Around(
|
||||
"execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))")
|
||||
@Around("execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))")
|
||||
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
|
||||
if (!dimensionValueMapEnable) {
|
||||
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
|
||||
@@ -108,22 +108,14 @@ public class DimValueAspect {
|
||||
}
|
||||
// consider '=' filter
|
||||
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())) {
|
||||
dimension.getDimValueMaps().stream()
|
||||
.forEach(
|
||||
dimValue -> {
|
||||
if (!CollectionUtils.isEmpty(dimValue.getAlias())
|
||||
&& dimValue.getAlias()
|
||||
.contains(
|
||||
expression
|
||||
.getFieldValue()
|
||||
.toString())) {
|
||||
getFiledNameToValueMap(
|
||||
filedNameToValueMap,
|
||||
expression.getFieldValue().toString(),
|
||||
dimValue.getTechName(),
|
||||
expression.getFieldName());
|
||||
}
|
||||
});
|
||||
dimension.getDimValueMaps().stream().forEach(dimValue -> {
|
||||
if (!CollectionUtils.isEmpty(dimValue.getAlias()) && dimValue.getAlias()
|
||||
.contains(expression.getFieldValue().toString())) {
|
||||
getFiledNameToValueMap(filedNameToValueMap,
|
||||
expression.getFieldValue().toString(), dimValue.getTechName(),
|
||||
expression.getFieldName());
|
||||
}
|
||||
});
|
||||
}
|
||||
// consider 'in' filter,each element needs to judge.
|
||||
replaceInCondition(expression, dimension, filedNameToValueMap);
|
||||
@@ -141,9 +133,7 @@ public class DimValueAspect {
|
||||
return queryResultWithColumns;
|
||||
}
|
||||
|
||||
public void replaceInCondition(
|
||||
FieldExpression expression,
|
||||
DimensionResp dimension,
|
||||
public void replaceInCondition(FieldExpression expression, DimensionResp dimension,
|
||||
Map<String, Map<String, String>> filedNameToValueMap) {
|
||||
if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) {
|
||||
String fieldValue = JsonUtil.toString(expression.getFieldValue());
|
||||
@@ -165,27 +155,20 @@ public class DimValueAspect {
|
||||
}
|
||||
}
|
||||
if (!revisedValues.equals(values)) {
|
||||
getFiledNameToValueMap(
|
||||
filedNameToValueMap,
|
||||
JsonUtil.toString(values),
|
||||
JsonUtil.toString(revisedValues),
|
||||
expression.getFieldName());
|
||||
getFiledNameToValueMap(filedNameToValueMap, JsonUtil.toString(values),
|
||||
JsonUtil.toString(revisedValues), expression.getFieldName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void getFiledNameToValueMap(
|
||||
Map<String, Map<String, String>> filedNameToValueMap,
|
||||
String oldValue,
|
||||
String newValue,
|
||||
String fieldName) {
|
||||
public void getFiledNameToValueMap(Map<String, Map<String, String>> filedNameToValueMap,
|
||||
String oldValue, String newValue, String fieldName) {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
map.put(oldValue, newValue);
|
||||
filedNameToValueMap.put(fieldName, map);
|
||||
}
|
||||
|
||||
private void rewriteDimValue(
|
||||
SemanticQueryResp semanticQueryResp,
|
||||
private void rewriteDimValue(SemanticQueryResp semanticQueryResp,
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
|
||||
if (!selectDimValueMap(semanticQueryResp.getColumns(), dimAndTechNameAndBizNamePair)) {
|
||||
return;
|
||||
@@ -209,8 +192,7 @@ public class DimValueAspect {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean selectDimValueMap(
|
||||
List<QueryColumn> columns,
|
||||
private boolean selectDimValueMap(List<QueryColumn> columns,
|
||||
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
|
||||
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)
|
||||
|| CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)) {
|
||||
@@ -225,8 +207,8 @@ public class DimValueAspect {
|
||||
return false;
|
||||
}
|
||||
|
||||
private void rewriteFilter(
|
||||
List<Filter> dimensionFilters, Map<String, Map<String, String>> aliasAndTechNamePair) {
|
||||
private void rewriteFilter(List<Filter> dimensionFilters,
|
||||
Map<String, Map<String, String>> aliasAndTechNamePair) {
|
||||
for (Filter filter : dimensionFilters) {
|
||||
if (Objects.isNull(filter)) {
|
||||
continue;
|
||||
@@ -283,18 +265,15 @@ public class DimValueAspect {
|
||||
continue;
|
||||
}
|
||||
if (StringUtils.isNotEmpty(dimValueMap.getBizName())) {
|
||||
aliasAndBizNameToTechName.put(
|
||||
dimValueMap.getBizName(), dimValueMap.getTechName());
|
||||
aliasAndBizNameToTechName.put(dimValueMap.getBizName(),
|
||||
dimValueMap.getTechName());
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(dimValueMap.getAlias())) {
|
||||
dimValueMap.getAlias().stream()
|
||||
.forEach(
|
||||
alias -> {
|
||||
if (StringUtils.isNotEmpty(alias)) {
|
||||
aliasAndBizNameToTechName.put(
|
||||
alias, dimValueMap.getTechName());
|
||||
}
|
||||
});
|
||||
dimValueMap.getAlias().stream().forEach(alias -> {
|
||||
if (StringUtils.isNotEmpty(alias)) {
|
||||
aliasAndBizNameToTechName.put(alias, dimValueMap.getTechName());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -339,8 +318,7 @@ public class DimValueAspect {
|
||||
}
|
||||
|
||||
private boolean needSkipDimension(DimensionResp dimension) {
|
||||
return Objects.isNull(dimension)
|
||||
|| StringUtils.isEmpty(dimension.getBizName())
|
||||
return Objects.isNull(dimension) || StringUtils.isEmpty(dimension.getBizName())
|
||||
|| CollectionUtils.isEmpty(dimension.getDimValueMaps());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,10 +52,14 @@ import java.util.stream.Collectors;
|
||||
@Slf4j
|
||||
public class S2DataPermissionAspect {
|
||||
|
||||
@Autowired private QueryStructUtils queryStructUtils;
|
||||
@Autowired private ModelService modelService;
|
||||
@Autowired private SchemaService schemaService;
|
||||
@Autowired private AuthService authService;
|
||||
@Autowired
|
||||
private QueryStructUtils queryStructUtils;
|
||||
@Autowired
|
||||
private ModelService modelService;
|
||||
@Autowired
|
||||
private SchemaService schemaService;
|
||||
@Autowired
|
||||
private AuthService authService;
|
||||
|
||||
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)")
|
||||
private void s2PermissionCheck() {}
|
||||
@@ -109,26 +113,19 @@ public class S2DataPermissionAspect {
|
||||
return result;
|
||||
}
|
||||
|
||||
private void checkColPermission(
|
||||
SemanticQueryReq semanticQueryReq,
|
||||
AuthorizedResourceResp authorizedResource,
|
||||
Set<Long> modelIds,
|
||||
private void checkColPermission(SemanticQueryReq semanticQueryReq,
|
||||
AuthorizedResourceResp authorizedResource, Set<Long> modelIds,
|
||||
SemanticSchemaResp semanticSchemaResp) {
|
||||
// get high sensitive fields in query
|
||||
Set<String> bizNamesInQueryReq = getBizNameInQueryReq(semanticQueryReq, semanticSchemaResp);
|
||||
Set<String> sensitiveBizNamesByModel =
|
||||
getHighSensitiveBizNamesByModelId(semanticSchemaResp);
|
||||
Set<String> sensitiveBizNameInQuery =
|
||||
bizNamesInQueryReq
|
||||
.parallelStream()
|
||||
.filter(sensitiveBizNamesByModel::contains)
|
||||
.collect(Collectors.toSet());
|
||||
Set<String> sensitiveBizNameInQuery = bizNamesInQueryReq.parallelStream()
|
||||
.filter(sensitiveBizNamesByModel::contains).collect(Collectors.toSet());
|
||||
|
||||
// get high sensitive field cur user has been authed
|
||||
Set<String> sensitiveBizNameUserAuthed =
|
||||
authorizedResource.getAuthResList().stream()
|
||||
.map(AuthRes::getName)
|
||||
.collect(Collectors.toSet());
|
||||
Set<String> sensitiveBizNameUserAuthed = authorizedResource.getAuthResList().stream()
|
||||
.map(AuthRes::getName).collect(Collectors.toSet());
|
||||
sensitiveBizNameInQuery.removeAll(sensitiveBizNameUserAuthed);
|
||||
if (!CollectionUtils.isEmpty(sensitiveBizNameInQuery)) {
|
||||
Set<String> sensitiveResNames =
|
||||
@@ -140,8 +137,8 @@ public class S2DataPermissionAspect {
|
||||
}
|
||||
}
|
||||
|
||||
private Set<Long> getModelIdInQuery(
|
||||
SemanticQueryReq semanticQueryReq, SemanticSchemaResp semanticSchemaResp) {
|
||||
private Set<Long> getModelIdInQuery(SemanticQueryReq semanticQueryReq,
|
||||
SemanticSchemaResp semanticSchemaResp) {
|
||||
if (semanticQueryReq instanceof QuerySqlReq) {
|
||||
QuerySqlReq querySqlReq = (QuerySqlReq) semanticQueryReq;
|
||||
return queryStructUtils.getModelIdFromSql(querySqlReq, semanticSchemaResp);
|
||||
@@ -153,8 +150,8 @@ public class S2DataPermissionAspect {
|
||||
return Sets.newHashSet();
|
||||
}
|
||||
|
||||
private void checkRowPermission(
|
||||
SemanticQueryReq queryReq, AuthorizedResourceResp authorizedResource) {
|
||||
private void checkRowPermission(SemanticQueryReq queryReq,
|
||||
AuthorizedResourceResp authorizedResource) {
|
||||
if (queryReq instanceof QuerySqlReq) {
|
||||
doRowPermission((QuerySqlReq) queryReq, authorizedResource);
|
||||
}
|
||||
@@ -163,8 +160,8 @@ public class S2DataPermissionAspect {
|
||||
}
|
||||
}
|
||||
|
||||
private Set<String> getBizNameInQueryReq(
|
||||
SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) {
|
||||
private Set<String> getBizNameInQueryReq(SemanticQueryReq queryReq,
|
||||
SemanticSchemaResp semanticSchemaResp) {
|
||||
if (queryReq instanceof QuerySqlReq) {
|
||||
return queryStructUtils.getBizNameFromSql((QuerySqlReq) queryReq, semanticSchemaResp);
|
||||
}
|
||||
@@ -181,8 +178,8 @@ public class S2DataPermissionAspect {
|
||||
return schemaService.fetchSemanticSchema(filter);
|
||||
}
|
||||
|
||||
private void doRowPermission(
|
||||
QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) {
|
||||
private void doRowPermission(QuerySqlReq querySqlReq,
|
||||
AuthorizedResourceResp authorizedResource) {
|
||||
log.debug("start doRowPermission logic");
|
||||
StringJoiner joiner = new StringJoiner(" OR ");
|
||||
List<String> dimensionFilters = new ArrayList<>();
|
||||
@@ -196,14 +193,11 @@ public class S2DataPermissionAspect {
|
||||
return;
|
||||
}
|
||||
|
||||
dimensionFilters.stream()
|
||||
.forEach(
|
||||
filter -> {
|
||||
if (StringUtils.isNotEmpty(filter)
|
||||
&& StringUtils.isNotEmpty(filter.trim())) {
|
||||
joiner.add(" ( " + filter + " ) ");
|
||||
}
|
||||
});
|
||||
dimensionFilters.stream().forEach(filter -> {
|
||||
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
|
||||
joiner.add(" ( " + filter + " ) ");
|
||||
}
|
||||
});
|
||||
try {
|
||||
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
|
||||
if (StringUtils.isNotEmpty(joiner.toString())) {
|
||||
@@ -217,8 +211,8 @@ public class S2DataPermissionAspect {
|
||||
}
|
||||
}
|
||||
|
||||
private void doRowPermission(
|
||||
QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
|
||||
private void doRowPermission(QueryStructReq queryStructReq,
|
||||
AuthorizedResourceResp authorizedResource) {
|
||||
log.debug("start doRowPermission logic");
|
||||
StringJoiner joiner = new StringJoiner(" OR ");
|
||||
List<String> dimensionFilters = new ArrayList<>();
|
||||
@@ -232,21 +226,17 @@ public class S2DataPermissionAspect {
|
||||
return;
|
||||
}
|
||||
|
||||
dimensionFilters.stream()
|
||||
.forEach(
|
||||
filter -> {
|
||||
if (StringUtils.isNotEmpty(filter)
|
||||
&& StringUtils.isNotEmpty(filter.trim())) {
|
||||
joiner.add(" ( " + filter + " ) ");
|
||||
}
|
||||
});
|
||||
dimensionFilters.stream().forEach(filter -> {
|
||||
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
|
||||
joiner.add(" ( " + filter + " ) ");
|
||||
}
|
||||
});
|
||||
|
||||
if (StringUtils.isNotEmpty(joiner.toString())) {
|
||||
log.info("before doRowPermission, queryStructReq:{}", queryStructReq);
|
||||
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString());
|
||||
List<Filter> filters =
|
||||
Objects.isNull(queryStructReq.getOriginalFilter())
|
||||
? new ArrayList<>()
|
||||
Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>()
|
||||
: queryStructReq.getOriginalFilter();
|
||||
filters.add(filter);
|
||||
queryStructReq.setDimensionFilters(filters);
|
||||
@@ -269,8 +259,7 @@ public class S2DataPermissionAspect {
|
||||
public void checkModelVisible(User user, Set<Long> modelIds) {
|
||||
List<Long> modelListVisible =
|
||||
modelService.getModelListWithAuth(user, null, AuthType.VISIBLE).stream()
|
||||
.map(ModelResp::getId)
|
||||
.collect(Collectors.toList());
|
||||
.map(ModelResp::getId).collect(Collectors.toList());
|
||||
List<Long> modelIdCopied = new ArrayList<>(modelIds);
|
||||
modelIdCopied.removeAll(modelListVisible);
|
||||
if (!CollectionUtils.isEmpty(modelIdCopied)) {
|
||||
@@ -281,9 +270,8 @@ public class S2DataPermissionAspect {
|
||||
if (modelResp == null) {
|
||||
throw new InvalidArgumentException("查询的模型不存在");
|
||||
}
|
||||
String message =
|
||||
String.format(
|
||||
"您没有模型[%s]权限,请联系管理员%s开通", modelResp.getName(), modelResp.getAdmins());
|
||||
String message = String.format("您没有模型[%s]权限,请联系管理员%s开通", modelResp.getName(),
|
||||
modelResp.getAdmins());
|
||||
throw new InvalidPermissionException(message);
|
||||
}
|
||||
}
|
||||
@@ -292,20 +280,14 @@ public class S2DataPermissionAspect {
|
||||
Set<String> highSensitiveCols = new HashSet<>();
|
||||
if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) {
|
||||
semanticSchemaResp.getDimensions().stream()
|
||||
.filter(
|
||||
dimSchemaResp ->
|
||||
SensitiveLevelEnum.HIGH
|
||||
.getCode()
|
||||
.equals(dimSchemaResp.getSensitiveLevel()))
|
||||
.filter(dimSchemaResp -> SensitiveLevelEnum.HIGH.getCode()
|
||||
.equals(dimSchemaResp.getSensitiveLevel()))
|
||||
.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(semanticSchemaResp.getMetrics())) {
|
||||
semanticSchemaResp.getMetrics().stream()
|
||||
.filter(
|
||||
metricSchemaResp ->
|
||||
SensitiveLevelEnum.HIGH
|
||||
.getCode()
|
||||
.equals(metricSchemaResp.getSensitiveLevel()))
|
||||
.filter(metricSchemaResp -> SensitiveLevelEnum.HIGH.getCode()
|
||||
.equals(metricSchemaResp.getSensitiveLevel()))
|
||||
.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
|
||||
}
|
||||
return highSensitiveCols;
|
||||
@@ -315,11 +297,8 @@ public class S2DataPermissionAspect {
|
||||
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
|
||||
queryAuthResReq.setModelIds(new ArrayList<>(modelIds));
|
||||
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
|
||||
log.info(
|
||||
"user:{}, domainId:{}, after queryAuthorizedResources:{}",
|
||||
user.getName(),
|
||||
modelIds,
|
||||
authorizedResource);
|
||||
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(),
|
||||
modelIds, authorizedResource);
|
||||
return authorizedResource;
|
||||
}
|
||||
|
||||
@@ -328,9 +307,7 @@ public class S2DataPermissionAspect {
|
||||
return authService.queryAuthorizedResources(queryAuthResReq, user);
|
||||
}
|
||||
|
||||
public void addHint(
|
||||
Set<Long> modelIds,
|
||||
SemanticQueryResp queryResultWithColumns,
|
||||
public void addHint(Set<Long> modelIds, SemanticQueryResp queryResultWithColumns,
|
||||
AuthorizedResourceResp authorizedResource) {
|
||||
List<DimensionFilter> filters = authorizedResource.getFilters();
|
||||
if (CollectionUtils.isEmpty(filters)) {
|
||||
@@ -342,20 +319,15 @@ public class S2DataPermissionAspect {
|
||||
ModelResp modelResp = modelService.getModel(modelIds.iterator().next());
|
||||
List<String> exprList = new ArrayList<>();
|
||||
List<String> descList = new ArrayList<>();
|
||||
filters.stream()
|
||||
.forEach(
|
||||
filter -> {
|
||||
if (StringUtils.isNotEmpty(filter.getDescription())) {
|
||||
descList.add(filter.getDescription());
|
||||
}
|
||||
exprList.add(filter.getExpressions().toString());
|
||||
});
|
||||
filters.stream().forEach(filter -> {
|
||||
if (StringUtils.isNotEmpty(filter.getDescription())) {
|
||||
descList.add(filter.getDescription());
|
||||
}
|
||||
exprList.add(filter.getExpressions().toString());
|
||||
});
|
||||
String promptInfo = "当前结果已经过行权限过滤,详细过滤条件如下:%s, 申请权限请联系管理员%s";
|
||||
String message =
|
||||
String.format(
|
||||
promptInfo,
|
||||
CollectionUtils.isEmpty(descList) ? exprList : descList,
|
||||
admins);
|
||||
String message = String.format(promptInfo,
|
||||
CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
|
||||
queryResultWithColumns.setQueryAuthorization(
|
||||
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
|
||||
}
|
||||
|
||||
@@ -23,45 +23,36 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
@Slf4j
|
||||
public class ChatQueryApiController {
|
||||
|
||||
@Autowired private ChatLayerService chatLayerService;
|
||||
@Autowired
|
||||
private ChatLayerService chatLayerService;
|
||||
|
||||
@Autowired private SemanticLayerService semanticLayerService;
|
||||
@Autowired
|
||||
private SemanticLayerService semanticLayerService;
|
||||
|
||||
@PostMapping("/chat/search")
|
||||
public Object search(
|
||||
@RequestBody QueryNLReq queryNLReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object search(@RequestBody QueryNLReq queryNLReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
queryNLReq.setUser(UserHolder.findUser(request, response));
|
||||
return chatLayerService.retrieve(queryNLReq);
|
||||
}
|
||||
|
||||
@PostMapping("/chat/map")
|
||||
public Object map(
|
||||
@RequestBody QueryNLReq queryNLReq,
|
||||
HttpServletRequest request,
|
||||
public Object map(@RequestBody QueryNLReq queryNLReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
queryNLReq.setUser(UserHolder.findUser(request, response));
|
||||
return chatLayerService.performMapping(queryNLReq);
|
||||
}
|
||||
|
||||
@PostMapping("/chat/parse")
|
||||
public Object parse(
|
||||
@RequestBody QueryNLReq queryNLReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object parse(@RequestBody QueryNLReq queryNLReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
queryNLReq.setUser(UserHolder.findUser(request, response));
|
||||
return chatLayerService.performParsing(queryNLReq);
|
||||
}
|
||||
|
||||
@PostMapping("/chat")
|
||||
public Object queryByNL(
|
||||
@RequestBody QueryNLReq queryNLReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object queryByNL(@RequestBody QueryNLReq queryNLReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
ParseResp parseResp = chatLayerService.performParsing(queryNLReq);
|
||||
if (parseResp.getState().equals(ParseResp.ParseState.COMPLETED)) {
|
||||
|
||||
@@ -21,15 +21,14 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
@Slf4j
|
||||
public class DataSetQueryApiController {
|
||||
|
||||
@Autowired private DataSetService dataSetService;
|
||||
@Autowired private SemanticLayerService semanticLayerService;
|
||||
@Autowired
|
||||
private DataSetService dataSetService;
|
||||
@Autowired
|
||||
private SemanticLayerService semanticLayerService;
|
||||
|
||||
@PostMapping("/dataSet")
|
||||
public Object queryByDataSet(
|
||||
@RequestBody QueryDataSetReq queryDataSetReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object queryByDataSet(@RequestBody QueryDataSetReq queryDataSetReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
SemanticQueryReq queryReq = dataSetService.convert(queryDataSetReq);
|
||||
return semanticLayerService.queryByReq(queryReq, user);
|
||||
|
||||
@@ -19,14 +19,12 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
@Slf4j
|
||||
public class MetaDiscoveryApiController {
|
||||
|
||||
@Autowired private ChatLayerService chatLayerService;
|
||||
@Autowired
|
||||
private ChatLayerService chatLayerService;
|
||||
|
||||
@PostMapping("map")
|
||||
public Object map(
|
||||
@RequestBody QueryMapReq queryMapReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object map(@RequestBody QueryMapReq queryMapReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
queryMapReq.setUser(user);
|
||||
return chatLayerService.map(queryMapReq);
|
||||
|
||||
@@ -24,39 +24,33 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
@Slf4j
|
||||
public class MetricQueryApiController {
|
||||
|
||||
@Autowired private SemanticLayerService semanticLayerService;
|
||||
@Autowired
|
||||
private SemanticLayerService semanticLayerService;
|
||||
|
||||
@Autowired private MetricService metricService;
|
||||
@Autowired
|
||||
private MetricService metricService;
|
||||
|
||||
@Autowired private DownloadService downloadService;
|
||||
@Autowired
|
||||
private DownloadService downloadService;
|
||||
|
||||
@PostMapping("/metric")
|
||||
public Object queryByMetric(
|
||||
@RequestBody QueryMetricReq queryMetricReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object queryByMetric(@RequestBody QueryMetricReq queryMetricReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
QueryStructReq queryStructReq = metricService.convert(queryMetricReq);
|
||||
return semanticLayerService.queryByReq(queryStructReq.convert(true), user);
|
||||
}
|
||||
|
||||
@PostMapping("/download/metric")
|
||||
public void downloadMetric(
|
||||
@RequestBody DownloadMetricReq downloadMetricReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public void downloadMetric(@RequestBody DownloadMetricReq downloadMetricReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
downloadService.downloadByStruct(downloadMetricReq, user, response);
|
||||
}
|
||||
|
||||
@PostMapping("/downloadBatch/metric")
|
||||
public void downloadBatch(
|
||||
@RequestBody BatchDownloadReq batchDownloadReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
downloadService.batchDownload(batchDownloadReq, user, response);
|
||||
}
|
||||
|
||||
@@ -30,16 +30,15 @@ import java.util.stream.Collectors;
|
||||
@Slf4j
|
||||
public class SqlQueryApiController {
|
||||
|
||||
@Autowired private SemanticLayerService semanticLayerService;
|
||||
@Autowired
|
||||
private SemanticLayerService semanticLayerService;
|
||||
|
||||
@Autowired private ChatLayerService chatLayerService;
|
||||
@Autowired
|
||||
private ChatLayerService chatLayerService;
|
||||
|
||||
@PostMapping("/sql")
|
||||
public Object queryBySql(
|
||||
@RequestBody QuerySqlReq querySqlReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object queryBySql(@RequestBody QuerySqlReq querySqlReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
String sql = querySqlReq.getSql();
|
||||
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
|
||||
@@ -48,63 +47,40 @@ public class SqlQueryApiController {
|
||||
}
|
||||
|
||||
@PostMapping("/sqls")
|
||||
public Object queryBySqls(
|
||||
@RequestBody QuerySqlsReq querySqlsReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object queryBySqls(@RequestBody QuerySqlsReq querySqlsReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
List<SemanticQueryReq> semanticQueryReqs =
|
||||
querySqlsReq.getSqls().stream()
|
||||
.map(
|
||||
sql -> {
|
||||
QuerySqlReq querySqlReq = new QuerySqlReq();
|
||||
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
|
||||
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
|
||||
chatLayerService.correct(querySqlReq, user);
|
||||
return querySqlReq;
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
List<SemanticQueryReq> semanticQueryReqs = querySqlsReq.getSqls().stream().map(sql -> {
|
||||
QuerySqlReq querySqlReq = new QuerySqlReq();
|
||||
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
|
||||
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
|
||||
chatLayerService.correct(querySqlReq, user);
|
||||
return querySqlReq;
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
List<CompletableFuture<SemanticQueryResp>> futures =
|
||||
semanticQueryReqs.stream()
|
||||
.map(
|
||||
querySqlReq ->
|
||||
CompletableFuture.supplyAsync(
|
||||
() -> {
|
||||
try {
|
||||
return semanticLayerService.queryByReq(
|
||||
querySqlReq, user);
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"querySqlReq:{},queryByReq error:",
|
||||
querySqlReq,
|
||||
e);
|
||||
return new SemanticQueryResp();
|
||||
}
|
||||
}))
|
||||
.collect(Collectors.toList());
|
||||
semanticQueryReqs.stream().map(querySqlReq -> CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
return semanticLayerService.queryByReq(querySqlReq, user);
|
||||
} catch (Exception e) {
|
||||
log.error("querySqlReq:{},queryByReq error:", querySqlReq, e);
|
||||
return new SemanticQueryResp();
|
||||
}
|
||||
})).collect(Collectors.toList());
|
||||
return futures.stream().map(CompletableFuture::join).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@PostMapping("/sqlsWithException")
|
||||
public Object queryBySqlsWithException(
|
||||
@RequestBody QuerySqlsReq querySqlsReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object queryBySqlsWithException(@RequestBody QuerySqlsReq querySqlsReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
List<SemanticQueryReq> semanticQueryReqs =
|
||||
querySqlsReq.getSqls().stream()
|
||||
.map(
|
||||
sql -> {
|
||||
QuerySqlReq querySqlReq = new QuerySqlReq();
|
||||
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
|
||||
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
|
||||
chatLayerService.correct(querySqlReq, user);
|
||||
return querySqlReq;
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
List<SemanticQueryReq> semanticQueryReqs = querySqlsReq.getSqls().stream().map(sql -> {
|
||||
QuerySqlReq querySqlReq = new QuerySqlReq();
|
||||
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
|
||||
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
|
||||
chatLayerService.correct(querySqlReq, user);
|
||||
return querySqlReq;
|
||||
}).collect(Collectors.toList());
|
||||
List<SemanticQueryResp> semanticQueryRespList = new ArrayList<>();
|
||||
try {
|
||||
for (SemanticQueryReq semanticQueryReq : semanticQueryReqs) {
|
||||
@@ -119,11 +95,8 @@ public class SqlQueryApiController {
|
||||
}
|
||||
|
||||
@PostMapping("/validate")
|
||||
public Object validate(
|
||||
@RequestBody QuerySqlReq querySqlReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object validate(@RequestBody QuerySqlReq querySqlReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
String sql = querySqlReq.getSql();
|
||||
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
|
||||
|
||||
@@ -19,14 +19,12 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
@Slf4j
|
||||
public class TagQueryApiController {
|
||||
|
||||
@Autowired private SemanticLayerService semanticLayerService;
|
||||
@Autowired
|
||||
private SemanticLayerService semanticLayerService;
|
||||
|
||||
@PostMapping("/tag")
|
||||
public Object queryByTag(
|
||||
@RequestBody QueryStructReq queryStructReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Object queryByTag(@RequestBody QueryStructReq queryStructReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return semanticLayerService.queryByReq(queryStructReq.convert(), user);
|
||||
}
|
||||
|
||||
@@ -53,20 +53,22 @@ import java.util.stream.Collectors;
|
||||
@Service
|
||||
@Slf4j
|
||||
public class S2ChatLayerService implements ChatLayerService {
|
||||
@Autowired private SchemaService schemaService;
|
||||
@Autowired private DataSetService dataSetService;
|
||||
@Autowired private RetrieveService retrieveService;
|
||||
@Autowired private ChatWorkflowEngine chatWorkflowEngine;
|
||||
@Autowired
|
||||
private SchemaService schemaService;
|
||||
@Autowired
|
||||
private DataSetService dataSetService;
|
||||
@Autowired
|
||||
private RetrieveService retrieveService;
|
||||
@Autowired
|
||||
private ChatWorkflowEngine chatWorkflowEngine;
|
||||
|
||||
@Override
|
||||
public MapResp performMapping(QueryNLReq queryNLReq) {
|
||||
MapResp mapResp = new MapResp(queryNLReq.getQueryText());
|
||||
ChatQueryContext queryCtx = buildChatQueryContext(queryNLReq);
|
||||
ComponentFactory.getSchemaMappers()
|
||||
.forEach(
|
||||
mapper -> {
|
||||
mapper.map(queryCtx);
|
||||
});
|
||||
ComponentFactory.getSchemaMappers().forEach(mapper -> {
|
||||
mapper.map(queryCtx);
|
||||
});
|
||||
mapResp.setMapInfo(queryCtx.getMapInfo());
|
||||
return mapResp;
|
||||
}
|
||||
@@ -96,17 +98,12 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
private ChatQueryContext buildChatQueryContext(QueryNLReq queryNLReq) {
|
||||
SemanticSchema semanticSchema = schemaService.getSemanticSchema(queryNLReq.getDataSetIds());
|
||||
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds();
|
||||
ChatQueryContext queryCtx =
|
||||
ChatQueryContext.builder()
|
||||
.queryFilters(queryNLReq.getQueryFilters())
|
||||
.semanticSchema(semanticSchema)
|
||||
.candidateQueries(new ArrayList<>())
|
||||
.mapInfo(new SchemaMapInfo())
|
||||
.modelIdToDataSetIds(modelIdToDataSetIds)
|
||||
.text2SQLType(queryNLReq.getText2SQLType())
|
||||
.mapModeEnum(queryNLReq.getMapModeEnum())
|
||||
.dataSetIds(queryNLReq.getDataSetIds())
|
||||
.build();
|
||||
ChatQueryContext queryCtx = ChatQueryContext.builder()
|
||||
.queryFilters(queryNLReq.getQueryFilters()).semanticSchema(semanticSchema)
|
||||
.candidateQueries(new ArrayList<>()).mapInfo(new SchemaMapInfo())
|
||||
.modelIdToDataSetIds(modelIdToDataSetIds).text2SQLType(queryNLReq.getText2SQLType())
|
||||
.mapModeEnum(queryNLReq.getMapModeEnum()).dataSetIds(queryNLReq.getDataSetIds())
|
||||
.build();
|
||||
BeanUtils.copyProperties(queryNLReq, queryCtx);
|
||||
return queryCtx;
|
||||
}
|
||||
@@ -146,14 +143,12 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
SchemaElement dataSet = semanticSchema.getDataSet(dataSetId);
|
||||
semanticParseInfo.setDataSet(dataSet);
|
||||
semanticParseInfo.setQueryConfig(semanticSchema.getQueryConfig(dataSetId));
|
||||
ComponentFactory.getSemanticCorrectors()
|
||||
.forEach(
|
||||
corrector -> {
|
||||
if (!(corrector instanceof GrammarCorrector
|
||||
|| (corrector instanceof SchemaCorrector))) {
|
||||
corrector.correct(queryCtx, semanticParseInfo);
|
||||
}
|
||||
});
|
||||
ComponentFactory.getSemanticCorrectors().forEach(corrector -> {
|
||||
if (!(corrector instanceof GrammarCorrector
|
||||
|| (corrector instanceof SchemaCorrector))) {
|
||||
corrector.correct(queryCtx, semanticParseInfo);
|
||||
}
|
||||
});
|
||||
log.info("chatQueryServiceImpl correct:{}", sqlInfo.getCorrectedS2SQL());
|
||||
return semanticParseInfo;
|
||||
}
|
||||
@@ -174,8 +169,8 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
return mapInfoResp;
|
||||
}
|
||||
|
||||
private Map<String, DataSetMapInfo> getDataSetInfo(
|
||||
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap, Integer topN) {
|
||||
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
|
||||
Map<Long, DataSetResp> dataSetMap, Integer topN) {
|
||||
Map<String, DataSetMapInfo> map = new HashMap<>();
|
||||
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
|
||||
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
|
||||
@@ -197,18 +192,15 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
return map;
|
||||
}
|
||||
|
||||
private Map<Long, List<SchemaElementMatch>> getMapFields(
|
||||
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap) {
|
||||
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
|
||||
Map<Long, DataSetResp> dataSetMap) {
|
||||
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
|
||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry :
|
||||
mapInfo.getDataSetElementMatches().entrySet()) {
|
||||
List<SchemaElementMatch> values =
|
||||
entry.getValue().stream()
|
||||
.filter(
|
||||
schemaElementMatch ->
|
||||
!SchemaElementType.TERM.equals(
|
||||
schemaElementMatch.getElement().getType()))
|
||||
.collect(Collectors.toList());
|
||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches()
|
||||
.entrySet()) {
|
||||
List<SchemaElementMatch> values = entry.getValue().stream()
|
||||
.filter(schemaElementMatch -> !SchemaElementType.TERM
|
||||
.equals(schemaElementMatch.getElement().getType()))
|
||||
.collect(Collectors.toList());
|
||||
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
|
||||
result.put(entry.getKey(), values);
|
||||
}
|
||||
@@ -216,15 +208,15 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
return result;
|
||||
}
|
||||
|
||||
private Map<Long, List<SchemaElementMatch>> getTopFields(
|
||||
Integer topN, SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap) {
|
||||
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN, SchemaMapInfo mapInfo,
|
||||
Map<Long, DataSetResp> dataSetMap) {
|
||||
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
|
||||
if (0 == topN) {
|
||||
return result;
|
||||
}
|
||||
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
|
||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry :
|
||||
mapInfo.getDataSetElementMatches().entrySet()) {
|
||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches()
|
||||
.entrySet()) {
|
||||
Long dataSetId = entry.getKey();
|
||||
List<SchemaElementMatch> values = entry.getValue();
|
||||
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
|
||||
@@ -233,23 +225,17 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
}
|
||||
String dataSetName = dataSetResp.getName();
|
||||
// topN dimensions
|
||||
Set<SchemaElementMatch> dimensions =
|
||||
semanticSchema.getDimensions(dataSetId).stream()
|
||||
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
|
||||
.limit(topN - 1)
|
||||
.map(mergeFunction())
|
||||
.collect(Collectors.toSet());
|
||||
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId).stream()
|
||||
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
|
||||
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
|
||||
|
||||
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
|
||||
dimensions.add(timeDimensionMatch);
|
||||
|
||||
// topN metrics
|
||||
Set<SchemaElementMatch> metrics =
|
||||
semanticSchema.getMetrics(dataSetId).stream()
|
||||
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
|
||||
.limit(topN)
|
||||
.map(mergeFunction())
|
||||
.collect(Collectors.toSet());
|
||||
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId).stream()
|
||||
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()).limit(topN)
|
||||
.map(mergeFunction()).collect(Collectors.toSet());
|
||||
|
||||
dimensions.addAll(metrics);
|
||||
result.put(dataSetId, new ArrayList<>(dimensions));
|
||||
@@ -257,8 +243,8 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
return result;
|
||||
}
|
||||
|
||||
private Map<String, List<SchemaElementMatch>> getTerms(
|
||||
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetNameMap) {
|
||||
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
|
||||
Map<Long, DataSetResp> dataSetNameMap) {
|
||||
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
|
||||
Map<Long, List<SchemaElementMatch>> dataSetElementMatches =
|
||||
mapInfo.getDataSetElementMatches();
|
||||
@@ -267,13 +253,10 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
if (dataSetResp == null) {
|
||||
continue;
|
||||
}
|
||||
List<SchemaElementMatch> terms =
|
||||
entry.getValue().stream()
|
||||
.filter(
|
||||
schemaElementMatch ->
|
||||
SchemaElementType.TERM.equals(
|
||||
schemaElementMatch.getElement().getType()))
|
||||
.collect(Collectors.toList());
|
||||
List<SchemaElementMatch> terms = entry.getValue().stream()
|
||||
.filter(schemaElementMatch -> SchemaElementType.TERM
|
||||
.equals(schemaElementMatch.getElement().getType()))
|
||||
.collect(Collectors.toList());
|
||||
termMap.put(dataSetResp.getName(), terms);
|
||||
}
|
||||
return termMap;
|
||||
@@ -287,34 +270,21 @@ public class S2ChatLayerService implements ChatLayerService {
|
||||
* @return
|
||||
*/
|
||||
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
|
||||
SchemaElement element =
|
||||
SchemaElement.builder()
|
||||
.dataSetId(dataSetId)
|
||||
.dataSetName(dataSetName)
|
||||
.type(SchemaElementType.DIMENSION)
|
||||
.bizName(TimeDimensionEnum.DAY.getName())
|
||||
.build();
|
||||
SchemaElement element = SchemaElement.builder().dataSetId(dataSetId)
|
||||
.dataSetName(dataSetName).type(SchemaElementType.DIMENSION)
|
||||
.bizName(TimeDimensionEnum.DAY.getName()).build();
|
||||
|
||||
SchemaElementMatch timeDimensionMatch =
|
||||
SchemaElementMatch.builder()
|
||||
.element(element)
|
||||
.detectWord(TimeDimensionEnum.DAY.getChName())
|
||||
.word(TimeDimensionEnum.DAY.getChName())
|
||||
.similarity(1L)
|
||||
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY)
|
||||
.build();
|
||||
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
|
||||
.detectWord(TimeDimensionEnum.DAY.getChName())
|
||||
.word(TimeDimensionEnum.DAY.getChName()).similarity(1L)
|
||||
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
|
||||
|
||||
return timeDimensionMatch;
|
||||
}
|
||||
|
||||
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
|
||||
return schemaElement ->
|
||||
SchemaElementMatch.builder()
|
||||
.element(schemaElement)
|
||||
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY)
|
||||
.word(schemaElement.getName())
|
||||
.similarity(1)
|
||||
.detectWord(schemaElement.getName())
|
||||
.build();
|
||||
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
|
||||
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName())
|
||||
.similarity(1).detectWord(schemaElement.getName()).build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,17 +85,11 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
private final QueryCache queryCache = ComponentFactory.getQueryCache();
|
||||
private final List<QueryExecutor> queryExecutors = ComponentFactory.getQueryExecutors();
|
||||
|
||||
public S2SemanticLayerService(
|
||||
StatUtils statUtils,
|
||||
QueryUtils queryUtils,
|
||||
QueryReqConverter queryReqConverter,
|
||||
SemanticSchemaManager semanticSchemaManager,
|
||||
DataSetService dataSetService,
|
||||
SchemaService schemaService,
|
||||
SemanticTranslator semanticTranslator,
|
||||
MetricDrillDownChecker metricDrillDownChecker,
|
||||
KnowledgeBaseService knowledgeBaseService,
|
||||
MetricService metricService,
|
||||
public S2SemanticLayerService(StatUtils statUtils, QueryUtils queryUtils,
|
||||
QueryReqConverter queryReqConverter, SemanticSchemaManager semanticSchemaManager,
|
||||
DataSetService dataSetService, SchemaService schemaService,
|
||||
SemanticTranslator semanticTranslator, MetricDrillDownChecker metricDrillDownChecker,
|
||||
KnowledgeBaseService knowledgeBaseService, MetricService metricService,
|
||||
DimensionService dimensionService) {
|
||||
this.statUtils = statUtils;
|
||||
this.queryUtils = queryUtils;
|
||||
@@ -119,11 +113,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
public SemanticTranslateResp translate(SemanticQueryReq queryReq, User user) throws Exception {
|
||||
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
|
||||
semanticTranslator.translate(queryStatement);
|
||||
return SemanticTranslateResp.builder()
|
||||
.querySQL(queryStatement.getSql())
|
||||
.isOk(queryStatement.isOk())
|
||||
.errMsg(queryStatement.getErrMsg())
|
||||
.build();
|
||||
return SemanticTranslateResp.builder().querySQL(queryStatement.getSql())
|
||||
.isOk(queryStatement.isOk()).errMsg(queryStatement.getErrMsg()).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -161,8 +152,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
for (QueryExecutor queryExecutor : queryExecutors) {
|
||||
if (queryExecutor.accept(queryStatement)) {
|
||||
queryResp = queryExecutor.execute(queryStatement);
|
||||
queryUtils.populateQueryColumns(
|
||||
queryResp, queryStatement.getSemanticSchemaResp());
|
||||
queryUtils.populateQueryColumns(queryResp,
|
||||
queryStatement.getSemanticSchemaResp());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,8 +199,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
return semanticQueryResp;
|
||||
}
|
||||
|
||||
private List<String> getDimensionValuesFromDict(
|
||||
DimensionValueReq dimensionValueReq, Set<Long> dataSetIds) {
|
||||
private List<String> getDimensionValuesFromDict(DimensionValueReq dimensionValueReq,
|
||||
Set<Long> dataSetIds) {
|
||||
if (StringUtils.isBlank(dimensionValueReq.getValue())) {
|
||||
return SearchService.getDimensionValue(dimensionValueReq);
|
||||
}
|
||||
@@ -217,28 +208,19 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
Map<Long, List<Long>> modelIdToDataSetIds = new HashMap<>();
|
||||
modelIdToDataSetIds.put(dimensionValueReq.getModelId(), new ArrayList<>(dataSetIds));
|
||||
|
||||
List<HanlpMapResult> hanlpMapResultList =
|
||||
knowledgeBaseService.prefixSearch(
|
||||
dimensionValueReq.getValue(), 2000, modelIdToDataSetIds, dataSetIds);
|
||||
List<HanlpMapResult> hanlpMapResultList = knowledgeBaseService
|
||||
.prefixSearch(dimensionValueReq.getValue(), 2000, modelIdToDataSetIds, dataSetIds);
|
||||
|
||||
HanlpHelper.transLetterOriginal(hanlpMapResultList);
|
||||
|
||||
return hanlpMapResultList.stream()
|
||||
.filter(
|
||||
o ->
|
||||
o.getNatures().stream()
|
||||
.map(NatureHelper::getElementID)
|
||||
.anyMatch(
|
||||
elementID ->
|
||||
dimensionValueReq
|
||||
.getElementID()
|
||||
.equals(elementID)))
|
||||
.map(MapResult::getName)
|
||||
.collect(Collectors.toList());
|
||||
.filter(o -> o.getNatures().stream().map(NatureHelper::getElementID)
|
||||
.anyMatch(elementID -> dimensionValueReq.getElementID().equals(elementID)))
|
||||
.map(MapResult::getName).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private SemanticQueryResp getDimensionValuesFromDb(
|
||||
DimensionValueReq dimensionValueReq, User user) {
|
||||
private SemanticQueryResp getDimensionValuesFromDb(DimensionValueReq dimensionValueReq,
|
||||
User user) {
|
||||
QuerySqlReq querySqlReq = buildQuerySqlReq(dimensionValueReq);
|
||||
return queryByReq(querySqlReq, user);
|
||||
}
|
||||
@@ -255,16 +237,13 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
return columns;
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> createResultList(
|
||||
DimensionValueReq dimensionValueReq, List<String> dimensionValues) {
|
||||
return dimensionValues.stream()
|
||||
.map(
|
||||
value -> {
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
map.put(dimensionValueReq.getBizName(), value);
|
||||
return map;
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
private List<Map<String, Object>> createResultList(DimensionValueReq dimensionValueReq,
|
||||
List<String> dimensionValues) {
|
||||
return dimensionValues.stream().map(value -> {
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
map.put(dimensionValueReq.getBizName(), value);
|
||||
return map;
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private DimensionResp getDimension(DimensionValueReq dimensionValueReq) {
|
||||
@@ -278,8 +257,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
return dimensionResp;
|
||||
}
|
||||
|
||||
public EntityInfo getEntityInfo(
|
||||
SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
|
||||
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema,
|
||||
User user) {
|
||||
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
|
||||
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
|
||||
if (parseInfo.getDimensionFilters().size() <= 0
|
||||
@@ -292,8 +271,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
if (StringUtils.isNotBlank(primaryKey)) {
|
||||
String entityId = "";
|
||||
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
|
||||
if (chatFilter != null
|
||||
&& chatFilter.getBizName() != null
|
||||
if (chatFilter != null && chatFilter.getBizName() != null
|
||||
&& chatFilter.getBizName().equals(primaryKey)) {
|
||||
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
|
||||
entityId = chatFilter.getValue().toString();
|
||||
@@ -377,8 +355,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
queryStatement =
|
||||
buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user);
|
||||
}
|
||||
if (Objects.nonNull(queryStatement)
|
||||
&& Objects.nonNull(semanticQueryReq.getSqlInfo())
|
||||
if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo())
|
||||
&& StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) {
|
||||
queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
|
||||
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
|
||||
@@ -402,8 +379,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
return queryStatement;
|
||||
}
|
||||
|
||||
private QueryStatement buildMultiStructQueryStatement(
|
||||
QueryMultiStructReq queryMultiStructReq, User user) throws Exception {
|
||||
private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq,
|
||||
User user) throws Exception {
|
||||
List<QueryStatement> sqlParsers = new ArrayList<>();
|
||||
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
|
||||
QueryStatement queryStatement = buildQueryStatement(queryStructReq, user);
|
||||
@@ -429,32 +406,20 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
QuerySqlReq querySqlReq = new QuerySqlReq();
|
||||
List<ModelResp> modelResps =
|
||||
schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
|
||||
DimensionResp dimensionResp =
|
||||
schemaService.getDimension(
|
||||
queryDimValueReq.getBizName(), queryDimValueReq.getModelId());
|
||||
DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getBizName(),
|
||||
queryDimValueReq.getModelId());
|
||||
ModelResp modelResp = modelResps.get(0);
|
||||
String sql =
|
||||
String.format(
|
||||
"select distinct %s from %s where 1=1",
|
||||
dimensionResp.getName(), modelResp.getName());
|
||||
String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(),
|
||||
modelResp.getName());
|
||||
List<Dim> timeDims = modelResp.getTimeDimension();
|
||||
if (CollectionUtils.isNotEmpty(timeDims)) {
|
||||
sql =
|
||||
String.format(
|
||||
"%s and %s >= '%s' and %s <= '%s'",
|
||||
sql,
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
queryDimValueReq.getDateInfo().getStartDate(),
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
queryDimValueReq.getDateInfo().getEndDate());
|
||||
sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
|
||||
TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(),
|
||||
TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate());
|
||||
}
|
||||
if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
|
||||
sql +=
|
||||
" AND "
|
||||
+ queryDimValueReq.getBizName()
|
||||
+ " LIKE '%"
|
||||
+ queryDimValueReq.getValue()
|
||||
+ "%'";
|
||||
sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%"
|
||||
+ queryDimValueReq.getValue() + "%'";
|
||||
}
|
||||
querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
|
||||
querySqlReq.setSql(sql);
|
||||
@@ -488,48 +453,32 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
|| detailTypeDefaultConfig.getDefaultDisplayInfo() == null) {
|
||||
return entityInfo;
|
||||
}
|
||||
List<DataInfo> dimensions =
|
||||
detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
|
||||
.map(
|
||||
id -> {
|
||||
SchemaElement element =
|
||||
dataSetSchema.getElement(
|
||||
SchemaElementType.DIMENSION, id);
|
||||
if (element == null) {
|
||||
return null;
|
||||
}
|
||||
return new DataInfo(
|
||||
element.getId().intValue(),
|
||||
element.getName(),
|
||||
element.getBizName(),
|
||||
null);
|
||||
})
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toList());
|
||||
List<DataInfo> metrics =
|
||||
detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
|
||||
.map(
|
||||
id -> {
|
||||
SchemaElement element =
|
||||
dataSetSchema.getElement(SchemaElementType.METRIC, id);
|
||||
if (element == null) {
|
||||
return null;
|
||||
}
|
||||
return new DataInfo(
|
||||
element.getId().intValue(),
|
||||
element.getName(),
|
||||
element.getBizName(),
|
||||
null);
|
||||
})
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toList());
|
||||
List<DataInfo> dimensions = detailTypeDefaultConfig.getDefaultDisplayInfo()
|
||||
.getDimensionIds().stream().map(id -> {
|
||||
SchemaElement element =
|
||||
dataSetSchema.getElement(SchemaElementType.DIMENSION, id);
|
||||
if (element == null) {
|
||||
return null;
|
||||
}
|
||||
return new DataInfo(element.getId().intValue(), element.getName(),
|
||||
element.getBizName(), null);
|
||||
}).filter(Objects::nonNull).collect(Collectors.toList());
|
||||
List<DataInfo> metrics = detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds()
|
||||
.stream().map(id -> {
|
||||
SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id);
|
||||
if (element == null) {
|
||||
return null;
|
||||
}
|
||||
return new DataInfo(element.getId().intValue(), element.getName(),
|
||||
element.getBizName(), null);
|
||||
}).filter(Objects::nonNull).collect(Collectors.toList());
|
||||
entityInfo.setDimensions(dimensions);
|
||||
entityInfo.setMetrics(metrics);
|
||||
return entityInfo;
|
||||
}
|
||||
|
||||
private void fillEntityInfoValue(
|
||||
EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
|
||||
private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema,
|
||||
User user) {
|
||||
SemanticQueryResp queryResultWithColumns =
|
||||
getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user);
|
||||
if (queryResultWithColumns != null) {
|
||||
@@ -540,19 +489,17 @@ public class S2SemanticLayerService implements SemanticLayerService {
|
||||
if (entry.getValue() == null || entryKey == null) {
|
||||
continue;
|
||||
}
|
||||
entityInfo.getDimensions().stream()
|
||||
.filter(i -> entryKey.equals(i.getBizName()))
|
||||
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
|
||||
.forEach(i -> i.setValue(entry.getValue().toString()));
|
||||
entityInfo.getMetrics().stream()
|
||||
.filter(i -> entryKey.equals(i.getBizName()))
|
||||
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
|
||||
.forEach(i -> i.setValue(entry.getValue().toString()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private SemanticQueryResp getQueryResultWithSchemaResp(
|
||||
EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
|
||||
private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
|
||||
DataSetSchema dataSetSchema, User user) {
|
||||
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
||||
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
|
||||
semanticParseInfo.setQueryType(QueryType.DETAIL);
|
||||
|
||||
@@ -21,9 +21,11 @@ import java.util.List;
|
||||
@Slf4j
|
||||
public class MetaEmbeddingListener implements ApplicationListener<DataEvent> {
|
||||
|
||||
@Autowired private EmbeddingConfig embeddingConfig;
|
||||
@Autowired
|
||||
private EmbeddingConfig embeddingConfig;
|
||||
|
||||
@Autowired private EmbeddingService embeddingService;
|
||||
@Autowired
|
||||
private EmbeddingService embeddingService;
|
||||
|
||||
@Value("${s2.embedding.operation.sleep.time:3000}")
|
||||
private Integer embeddingOperationSleepTime;
|
||||
|
||||
@@ -22,28 +22,24 @@ public class SchemaDictUpdateListener implements ApplicationListener<DataEvent>
|
||||
if (CollectionUtils.isEmpty(dataEvent.getDataItems())) {
|
||||
return;
|
||||
}
|
||||
dataEvent
|
||||
.getDataItems()
|
||||
.forEach(
|
||||
dataItem -> {
|
||||
DictWord dictWord = new DictWord();
|
||||
dictWord.setWord(dataItem.getName());
|
||||
String sign = DictWordType.NATURE_SPILT;
|
||||
String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
|
||||
String nature =
|
||||
sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
|
||||
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
|
||||
dictWord.setNature(nature);
|
||||
dictWord.setNatureWithFrequency(natureWithFrequency);
|
||||
if (EventType.ADD.equals(dataEvent.getEventType())) {
|
||||
HanlpHelper.addToCustomDictionary(dictWord);
|
||||
} else if (EventType.DELETE.equals(dataEvent.getEventType())) {
|
||||
HanlpHelper.removeFromCustomDictionary(dictWord);
|
||||
} else if (EventType.UPDATE.equals(dataEvent.getEventType())) {
|
||||
HanlpHelper.removeFromCustomDictionary(dictWord);
|
||||
dictWord.setWord(dataItem.getNewName());
|
||||
HanlpHelper.addToCustomDictionary(dictWord);
|
||||
}
|
||||
});
|
||||
dataEvent.getDataItems().forEach(dataItem -> {
|
||||
DictWord dictWord = new DictWord();
|
||||
dictWord.setWord(dataItem.getName());
|
||||
String sign = DictWordType.NATURE_SPILT;
|
||||
String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
|
||||
String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
|
||||
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
|
||||
dictWord.setNature(nature);
|
||||
dictWord.setNatureWithFrequency(natureWithFrequency);
|
||||
if (EventType.ADD.equals(dataEvent.getEventType())) {
|
||||
HanlpHelper.addToCustomDictionary(dictWord);
|
||||
} else if (EventType.DELETE.equals(dataEvent.getEventType())) {
|
||||
HanlpHelper.removeFromCustomDictionary(dictWord);
|
||||
} else if (EventType.UPDATE.equals(dataEvent.getEventType())) {
|
||||
HanlpHelper.removeFromCustomDictionary(dictWord);
|
||||
dictWord.setWord(dataItem.getNewName());
|
||||
HanlpHelper.addToCustomDictionary(dictWord);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,14 +22,9 @@ public class DimensionYamlManager {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
return dimensions.stream()
|
||||
.filter(
|
||||
dimension ->
|
||||
!dimension
|
||||
.getType()
|
||||
.name()
|
||||
.equalsIgnoreCase(IdentifyType.primary.name()))
|
||||
.map(DimensionYamlManager::convert2DimensionYamlTpl)
|
||||
.collect(Collectors.toList());
|
||||
.filter(dimension -> !dimension.getType().name()
|
||||
.equalsIgnoreCase(IdentifyType.primary.name()))
|
||||
.map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) {
|
||||
|
||||
@@ -28,8 +28,8 @@ import java.util.stream.Collectors;
|
||||
@Slf4j
|
||||
public class ModelYamlManager {
|
||||
|
||||
public static synchronized DataModelYamlTpl convert2YamlObj(
|
||||
ModelResp modelResp, DatabaseResp databaseResp) {
|
||||
public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp,
|
||||
DatabaseResp databaseResp) {
|
||||
ModelDetail modelDetail = modelResp.getModelDetail();
|
||||
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
|
||||
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
|
||||
@@ -37,18 +37,12 @@ public class ModelYamlManager {
|
||||
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
|
||||
dataModelYamlTpl.setType(databaseResp.getType());
|
||||
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);
|
||||
dataModelYamlTpl.setIdentifiers(
|
||||
modelDetail.getIdentifiers().stream()
|
||||
.map(ModelYamlManager::convert)
|
||||
.collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setDimensions(
|
||||
modelDetail.getDimensions().stream()
|
||||
.map(ModelYamlManager::convert)
|
||||
.collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setMeasures(
|
||||
modelDetail.getMeasures().stream()
|
||||
.map(ModelYamlManager::convert)
|
||||
.collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setIdentifiers(modelDetail.getIdentifiers().stream()
|
||||
.map(ModelYamlManager::convert).collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setDimensions(modelDetail.getDimensions().stream()
|
||||
.map(ModelYamlManager::convert).collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setMeasures(modelDetail.getMeasures().stream()
|
||||
.map(ModelYamlManager::convert).collect(Collectors.toList()));
|
||||
dataModelYamlTpl.setName(modelResp.getBizName());
|
||||
dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
|
||||
if (modelDetail.getQueryType().equalsIgnoreCase(ModelDefineType.SQL_QUERY.getName())) {
|
||||
|
||||
@@ -64,12 +64,8 @@ public class SemanticSchemaManager {
|
||||
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
|
||||
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
|
||||
Map<Long, String> modelIdName = new HashMap<>();
|
||||
schemaService.getSchemaYamlTpl(
|
||||
semanticSchemaResp,
|
||||
dimensionYamlTpls,
|
||||
dataModelYamlTpls,
|
||||
metricYamlTpls,
|
||||
modelIdName);
|
||||
schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls, dataModelYamlTpls,
|
||||
metricYamlTpls, modelIdName);
|
||||
DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp();
|
||||
semanticModel.setDatabase(DatabaseConverter.convert(databaseResp));
|
||||
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) {
|
||||
@@ -78,11 +74,8 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
if (!dataModelYamlTpls.isEmpty()) {
|
||||
Map<String, DataSource> dataSourceMap =
|
||||
dataModelYamlTpls.stream()
|
||||
.map(SemanticSchemaManager::getDatasource)
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
DataSource::getName, item -> item, (k1, k2) -> k1));
|
||||
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect(
|
||||
Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
|
||||
semanticModel.setDatasourceMap(dataSourceMap);
|
||||
}
|
||||
if (!dimensionYamlTpls.isEmpty()) {
|
||||
@@ -114,8 +107,8 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
if (Objects.nonNull(semanticModel.getDatasourceMap())
|
||||
&& !semanticModel.getDatasourceMap().isEmpty()) {
|
||||
for (Map.Entry<String, DataSource> entry :
|
||||
semanticModel.getDatasourceMap().entrySet()) {
|
||||
for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap()
|
||||
.entrySet()) {
|
||||
List<Dimension> modelDimensions = new ArrayList<>();
|
||||
if (!semanticModel.getDimensionMap().containsKey(entry.getKey())) {
|
||||
semanticModel.getDimensionMap().put(entry.getKey(), modelDimensions);
|
||||
@@ -133,18 +126,16 @@ public class SemanticSchemaManager {
|
||||
return semanticModel;
|
||||
}
|
||||
|
||||
private void addTagModel(
|
||||
TagResp tagResp, List<Dimension> modelDimensions, List<Metric> modelMetrics)
|
||||
throws Exception {
|
||||
private void addTagModel(TagResp tagResp, List<Dimension> modelDimensions,
|
||||
List<Metric> modelMetrics) throws Exception {
|
||||
TagDefineType tagDefineType = TagDefineType.valueOf(tagResp.getTagDefineType());
|
||||
switch (tagDefineType) {
|
||||
case FIELD:
|
||||
case DIMENSION:
|
||||
if (TagDefineType.DIMENSION.equals(tagResp.getTagDefineType())) {
|
||||
Optional<Dimension> modelDimension =
|
||||
modelDimensions.stream()
|
||||
// .filter(d -> d.getBizName().equals(tagResp.getExpr()))
|
||||
.findFirst();
|
||||
Optional<Dimension> modelDimension = modelDimensions.stream()
|
||||
// .filter(d -> d.getBizName().equals(tagResp.getExpr()))
|
||||
.findFirst();
|
||||
if (modelDimension.isPresent()) {
|
||||
modelDimension.get().setName(tagResp.getBizName());
|
||||
return;
|
||||
@@ -152,7 +143,7 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
Dimension dimension = Dimension.builder().build();
|
||||
dimension.setType("");
|
||||
// dimension.setExpr(tagResp.getExpr());
|
||||
// dimension.setExpr(tagResp.getExpr());
|
||||
dimension.setName(tagResp.getBizName());
|
||||
dimension.setOwners("");
|
||||
dimension.setBizName(tagResp.getBizName());
|
||||
@@ -165,10 +156,9 @@ public class SemanticSchemaManager {
|
||||
modelDimensions.add(dimension);
|
||||
return;
|
||||
case METRIC:
|
||||
Optional<Metric> modelMetric =
|
||||
modelMetrics.stream()
|
||||
// .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr()))
|
||||
.findFirst();
|
||||
Optional<Metric> modelMetric = modelMetrics.stream()
|
||||
// .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr()))
|
||||
.findFirst();
|
||||
if (modelMetric.isPresent()) {
|
||||
modelMetric.get().setName(tagResp.getBizName());
|
||||
} else {
|
||||
@@ -189,37 +179,22 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
|
||||
public static DataSource getDatasource(final DataModelYamlTpl d) {
|
||||
DataSource datasource =
|
||||
DataSource.builder()
|
||||
.id(d.getId())
|
||||
.sourceId(d.getSourceId())
|
||||
.type(d.getType())
|
||||
.sqlQuery(d.getSqlQuery())
|
||||
.name(d.getName())
|
||||
.tableQuery(d.getTableQuery())
|
||||
.identifiers(getIdentify(d.getIdentifiers()))
|
||||
.measures(getMeasureParams(d.getMeasures()))
|
||||
.dimensions(getDimensions(d.getDimensions()))
|
||||
.build();
|
||||
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId())
|
||||
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
|
||||
.tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
|
||||
.measures(getMeasureParams(d.getMeasures()))
|
||||
.dimensions(getDimensions(d.getDimensions())).build();
|
||||
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
|
||||
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
|
||||
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
|
||||
}
|
||||
if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
|
||||
Set<String> measures =
|
||||
datasource.getMeasures().stream()
|
||||
.map(mm -> mm.getName())
|
||||
.collect(Collectors.toSet());
|
||||
Set<String> measures = datasource.getMeasures().stream().map(mm -> mm.getName())
|
||||
.collect(Collectors.toSet());
|
||||
for (Field f : d.getFields()) {
|
||||
if (!measures.contains(f.getFieldName())) {
|
||||
datasource
|
||||
.getMeasures()
|
||||
.add(
|
||||
Measure.builder()
|
||||
.expr(f.getFieldName())
|
||||
.name(f.getFieldName())
|
||||
.agg("")
|
||||
.build());
|
||||
datasource.getMeasures().add(Measure.builder().expr(f.getFieldName())
|
||||
.name(f.getFieldName()).agg("").build());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -227,10 +202,9 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
|
||||
private static String getDataSourceAggTime(List<Dimension> dimensions) {
|
||||
Optional<Dimension> timeDimension =
|
||||
dimensions.stream()
|
||||
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
|
||||
.findFirst();
|
||||
Optional<Dimension> timeDimension = dimensions.stream()
|
||||
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
|
||||
.findFirst();
|
||||
if (timeDimension.isPresent()
|
||||
&& Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
|
||||
return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
|
||||
@@ -340,8 +314,8 @@ public class SemanticSchemaManager {
|
||||
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl) {
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
|
||||
if (dimensionTimeTypeParamsTpl != null) {
|
||||
dimensionTimeTypeParams.setTimeGranularity(
|
||||
dimensionTimeTypeParamsTpl.getTimeGranularity());
|
||||
dimensionTimeTypeParams
|
||||
.setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity());
|
||||
dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary());
|
||||
}
|
||||
return dimensionTimeTypeParams;
|
||||
@@ -358,38 +332,27 @@ public class SemanticSchemaManager {
|
||||
return identifies;
|
||||
}
|
||||
|
||||
private static List<JoinRelation> getJoinRelation(
|
||||
List<ModelRela> modelRelas, Map<Long, String> modelIdName) {
|
||||
private static List<JoinRelation> getJoinRelation(List<ModelRela> modelRelas,
|
||||
Map<Long, String> modelIdName) {
|
||||
List<JoinRelation> joinRelations = new ArrayList<>();
|
||||
modelRelas.stream()
|
||||
.forEach(
|
||||
r -> {
|
||||
if (modelIdName.containsKey(r.getFromModelId())
|
||||
&& modelIdName.containsKey(r.getToModelId())) {
|
||||
JoinRelation joinRelation =
|
||||
JoinRelation.builder()
|
||||
.left(modelIdName.get(r.getFromModelId()))
|
||||
.right(modelIdName.get(r.getToModelId()))
|
||||
.joinType(r.getJoinType())
|
||||
.build();
|
||||
List<Triple<String, String, String>> conditions = new ArrayList<>();
|
||||
r.getJoinConditions().stream()
|
||||
.forEach(
|
||||
rr -> {
|
||||
if (FilterOperatorEnum.isValueCompare(
|
||||
rr.getOperator())) {
|
||||
conditions.add(
|
||||
Triple.of(
|
||||
rr.getLeftField(),
|
||||
rr.getOperator().getValue(),
|
||||
rr.getRightField()));
|
||||
}
|
||||
});
|
||||
joinRelation.setId(r.getId());
|
||||
joinRelation.setJoinCondition(conditions);
|
||||
joinRelations.add(joinRelation);
|
||||
}
|
||||
});
|
||||
modelRelas.stream().forEach(r -> {
|
||||
if (modelIdName.containsKey(r.getFromModelId())
|
||||
&& modelIdName.containsKey(r.getToModelId())) {
|
||||
JoinRelation joinRelation = JoinRelation.builder()
|
||||
.left(modelIdName.get(r.getFromModelId()))
|
||||
.right(modelIdName.get(r.getToModelId())).joinType(r.getJoinType()).build();
|
||||
List<Triple<String, String, String>> conditions = new ArrayList<>();
|
||||
r.getJoinConditions().stream().forEach(rr -> {
|
||||
if (FilterOperatorEnum.isValueCompare(rr.getOperator())) {
|
||||
conditions.add(Triple.of(rr.getLeftField(), rr.getOperator().getValue(),
|
||||
rr.getRightField()));
|
||||
}
|
||||
});
|
||||
joinRelation.setId(r.getId());
|
||||
joinRelation.setJoinCondition(conditions);
|
||||
joinRelations.add(joinRelation);
|
||||
}
|
||||
});
|
||||
return joinRelations;
|
||||
}
|
||||
|
||||
@@ -405,8 +368,7 @@ public class SemanticSchemaManager {
|
||||
String dataSourceName = datasourceYamlTpl.getName();
|
||||
Optional<Entry<String, DataSource>> datasourceYamlTplMap =
|
||||
schema.getDatasource().entrySet().stream()
|
||||
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName))
|
||||
.findFirst();
|
||||
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
|
||||
if (datasourceYamlTplMap.isPresent()) {
|
||||
datasourceYamlTplMap.get().setValue(datasourceYamlTpl);
|
||||
} else {
|
||||
@@ -415,14 +377,12 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
}
|
||||
|
||||
public static void update(
|
||||
SemanticSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
|
||||
throws Exception {
|
||||
public static void update(SemanticSchema schema, String datasourceBizName,
|
||||
List<Dimension> dimensionYamlTpls) throws Exception {
|
||||
if (schema != null) {
|
||||
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap =
|
||||
schema.getDimension().entrySet().stream()
|
||||
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName))
|
||||
.findFirst();
|
||||
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema
|
||||
.getDimension().entrySet().stream()
|
||||
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
|
||||
if (datasourceYamlTplMap.isPresent()) {
|
||||
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
|
||||
} else {
|
||||
@@ -433,8 +393,8 @@ public class SemanticSchemaManager {
|
||||
}
|
||||
}
|
||||
|
||||
private static void updateDimension(
|
||||
List<Dimension> dimensionYamlTpls, List<Dimension> dimensions) {
|
||||
private static void updateDimension(List<Dimension> dimensionYamlTpls,
|
||||
List<Dimension> dimensions) {
|
||||
if (CollectionUtils.isEmpty(dimensionYamlTpls)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -139,8 +139,8 @@ public class DatabaseDOExample {
|
||||
criteria.add(new Criterion(condition, value));
|
||||
}
|
||||
|
||||
protected void addCriterion(
|
||||
String condition, Object value1, Object value2, String property) {
|
||||
protected void addCriterion(String condition, Object value1, Object value2,
|
||||
String property) {
|
||||
if (value1 == null || value2 == null) {
|
||||
throw new RuntimeException("Between values for " + property + " cannot be null");
|
||||
}
|
||||
@@ -969,8 +969,8 @@ public class DatabaseDOExample {
|
||||
this(condition, value, null);
|
||||
}
|
||||
|
||||
protected Criterion(
|
||||
String condition, Object value, Object secondValue, String typeHandler) {
|
||||
protected Criterion(String condition, Object value, Object secondValue,
|
||||
String typeHandler) {
|
||||
super();
|
||||
this.condition = condition;
|
||||
this.value = value;
|
||||
|
||||
@@ -139,8 +139,8 @@ public class DomainDOExample {
|
||||
criteria.add(new Criterion(condition, value));
|
||||
}
|
||||
|
||||
protected void addCriterion(
|
||||
String condition, Object value1, Object value2, String property) {
|
||||
protected void addCriterion(String condition, Object value1, Object value2,
|
||||
String property) {
|
||||
if (value1 == null || value2 == null) {
|
||||
throw new RuntimeException("Between values for " + property + " cannot be null");
|
||||
}
|
||||
@@ -1117,8 +1117,8 @@ public class DomainDOExample {
|
||||
this(condition, value, null);
|
||||
}
|
||||
|
||||
protected Criterion(
|
||||
String condition, Object value, Object secondValue, String typeHandler) {
|
||||
protected Criterion(String condition, Object value, Object secondValue,
|
||||
String typeHandler) {
|
||||
super();
|
||||
this.condition = condition;
|
||||
this.value = value;
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.AppDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface AppMapper extends BaseMapper<AppDO> {}
|
||||
public interface AppMapper extends BaseMapper<AppDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface CanvasDOMapper extends BaseMapper<CanvasDO> {}
|
||||
public interface CanvasDOMapper extends BaseMapper<CanvasDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ClassDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface ClassMapper extends BaseMapper<ClassDO> {}
|
||||
public interface ClassMapper extends BaseMapper<ClassDO> {
|
||||
}
|
||||
|
||||
@@ -11,4 +11,5 @@ import org.apache.ibatis.annotations.Mapper;
|
||||
* @since 2023-11-09 03:49:33
|
||||
*/
|
||||
@Mapper
|
||||
public interface CollectMapper extends BaseMapper<CollectDO> {}
|
||||
public interface CollectMapper extends BaseMapper<CollectDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface DataSetDOMapper extends BaseMapper<DataSetDO> {}
|
||||
public interface DataSetDOMapper extends BaseMapper<DataSetDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface DatabaseDOMapper extends BaseMapper<DatabaseDO> {}
|
||||
public interface DatabaseDOMapper extends BaseMapper<DatabaseDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface DictConfMapper extends BaseMapper<DictConfDO> {}
|
||||
public interface DictConfMapper extends BaseMapper<DictConfDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface DictTaskMapper extends BaseMapper<DictTaskDO> {}
|
||||
public interface DictTaskMapper extends BaseMapper<DictTaskDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface DimensionDOMapper extends BaseMapper<DimensionDO> {}
|
||||
public interface DimensionDOMapper extends BaseMapper<DimensionDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface DomainDOMapper extends BaseMapper<DomainDO> {}
|
||||
public interface DomainDOMapper extends BaseMapper<DomainDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface MetricDOMapper extends BaseMapper<MetricDO> {}
|
||||
public interface MetricDOMapper extends BaseMapper<MetricDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQuery
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface MetricQueryDefaultConfigDOMapper extends BaseMapper<MetricQueryDefaultConfigDO> {}
|
||||
public interface MetricQueryDefaultConfigDOMapper extends BaseMapper<MetricQueryDefaultConfigDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface ModelDOMapper extends BaseMapper<ModelDO> {}
|
||||
public interface ModelDOMapper extends BaseMapper<ModelDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ModelRelaDO
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface ModelRelaDOMapper extends BaseMapper<ModelRelaDO> {}
|
||||
public interface ModelRelaDOMapper extends BaseMapper<ModelRelaDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.QueryRuleDO
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface QueryRuleMapper extends BaseMapper<QueryRuleDO> {}
|
||||
public interface QueryRuleMapper extends BaseMapper<QueryRuleDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface TagMapper extends BaseMapper<TagDO> {}
|
||||
public interface TagMapper extends BaseMapper<TagDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TagObjectDO
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface TagObjectMapper extends BaseMapper<TagObjectDO> {}
|
||||
public interface TagObjectMapper extends BaseMapper<TagObjectDO> {
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TermDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface TermMapper extends BaseMapper<TermDO> {}
|
||||
public interface TermMapper extends BaseMapper<TermDO> {
|
||||
}
|
||||
|
||||
@@ -26,7 +26,8 @@ public class DateInfoRepositoryImpl implements DateInfoRepository {
|
||||
|
||||
private ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
@Autowired private DateInfoMapper dateInfoMapper;
|
||||
@Autowired
|
||||
private DateInfoMapper dateInfoMapper;
|
||||
|
||||
@Override
|
||||
public Integer upsertDateInfo(List<DateInfoReq> dateInfoCommends) {
|
||||
@@ -36,22 +37,19 @@ public class DateInfoRepositoryImpl implements DateInfoRepository {
|
||||
return 0;
|
||||
}
|
||||
|
||||
dateInfoCommends.stream()
|
||||
.forEach(
|
||||
commend -> {
|
||||
DateInfoDO dateInfoDO = new DateInfoDO();
|
||||
BeanUtils.copyProperties(commend, dateInfoDO);
|
||||
try {
|
||||
dateInfoDO.setUnavailableDateList(
|
||||
mapper.writeValueAsString(
|
||||
commend.getUnavailableDateList()));
|
||||
dateInfoDO.setCreatedBy(Constants.ADMIN_LOWER);
|
||||
dateInfoDO.setUpdatedBy(Constants.ADMIN_LOWER);
|
||||
} catch (JsonProcessingException e) {
|
||||
log.info("e,", e);
|
||||
}
|
||||
dateInfoDOList.add(dateInfoDO);
|
||||
});
|
||||
dateInfoCommends.stream().forEach(commend -> {
|
||||
DateInfoDO dateInfoDO = new DateInfoDO();
|
||||
BeanUtils.copyProperties(commend, dateInfoDO);
|
||||
try {
|
||||
dateInfoDO.setUnavailableDateList(
|
||||
mapper.writeValueAsString(commend.getUnavailableDateList()));
|
||||
dateInfoDO.setCreatedBy(Constants.ADMIN_LOWER);
|
||||
dateInfoDO.setUpdatedBy(Constants.ADMIN_LOWER);
|
||||
} catch (JsonProcessingException e) {
|
||||
log.info("e,", e);
|
||||
}
|
||||
dateInfoDOList.add(dateInfoDO);
|
||||
});
|
||||
|
||||
return batchUpsert(dateInfoDOList);
|
||||
}
|
||||
|
||||
@@ -36,11 +36,8 @@ public class DictRepositoryImpl implements DictRepository {
|
||||
private final DictUtils dictConverter;
|
||||
private final DimensionService dimensionService;
|
||||
|
||||
public DictRepositoryImpl(
|
||||
DictTaskMapper dictTaskMapper,
|
||||
DictConfMapper dictConfMapper,
|
||||
DictUtils dictConverter,
|
||||
DimensionService dimensionService) {
|
||||
public DictRepositoryImpl(DictTaskMapper dictTaskMapper, DictConfMapper dictConfMapper,
|
||||
DictUtils dictConverter, DimensionService dimensionService) {
|
||||
this.dictTaskMapper = dictTaskMapper;
|
||||
this.dictConfMapper = dictConfMapper;
|
||||
this.dictConverter = dictConverter;
|
||||
@@ -90,11 +87,9 @@ public class DictRepositoryImpl implements DictRepository {
|
||||
QueryWrapper<DictTaskDO> wrapper = new QueryWrapper<>();
|
||||
wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId());
|
||||
wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType());
|
||||
List<DictTaskDO> dictTaskDOList =
|
||||
dictTaskMapper.selectList(wrapper).stream()
|
||||
.sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
|
||||
.limit(dictTaskNum)
|
||||
.collect(Collectors.toList());
|
||||
List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper).stream()
|
||||
.sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
|
||||
.limit(dictTaskNum).collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(dictTaskDOList)) {
|
||||
return taskResp;
|
||||
}
|
||||
@@ -114,10 +109,8 @@ public class DictRepositoryImpl implements DictRepository {
|
||||
@Override
|
||||
public Long editDictConf(DictConfDO dictConfDO) {
|
||||
DictItemFilter filter =
|
||||
DictItemFilter.builder()
|
||||
.type(TypeEnums.valueOf(dictConfDO.getType()))
|
||||
.itemId(dictConfDO.getItemId())
|
||||
.build();
|
||||
DictItemFilter.builder().type(TypeEnums.valueOf(dictConfDO.getType()))
|
||||
.itemId(dictConfDO.getItemId()).build();
|
||||
|
||||
List<DictConfDO> dictConfDOList = getDictConfDOList(filter);
|
||||
if (CollectionUtils.isEmpty(dictConfDOList)) {
|
||||
|
||||
@@ -17,8 +17,8 @@ public class DimensionRepositoryImpl implements DimensionRepository {
|
||||
|
||||
private DimensionDOCustomMapper dimensionDOCustomMapper;
|
||||
|
||||
public DimensionRepositoryImpl(
|
||||
DimensionDOMapper dimensionDOMapper, DimensionDOCustomMapper dimensionDOCustomMapper) {
|
||||
public DimensionRepositoryImpl(DimensionDOMapper dimensionDOMapper,
|
||||
DimensionDOCustomMapper dimensionDOCustomMapper) {
|
||||
this.dimensionDOMapper = dimensionDOMapper;
|
||||
this.dimensionDOCustomMapper = dimensionDOCustomMapper;
|
||||
}
|
||||
|
||||
@@ -22,8 +22,7 @@ public class MetricRepositoryImpl implements MetricRepository {
|
||||
|
||||
private MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper;
|
||||
|
||||
public MetricRepositoryImpl(
|
||||
MetricDOMapper metricDOMapper,
|
||||
public MetricRepositoryImpl(MetricDOMapper metricDOMapper,
|
||||
MetricDOCustomMapper metricDOCustomMapper,
|
||||
MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper) {
|
||||
this.metricDOMapper = metricDOMapper;
|
||||
@@ -95,9 +94,7 @@ public class MetricRepositoryImpl implements MetricRepository {
|
||||
@Override
|
||||
public MetricQueryDefaultConfigDO getDefaultQueryConfig(Long metricId, String userName) {
|
||||
QueryWrapper<MetricQueryDefaultConfigDO> queryWrapper = new QueryWrapper<>();
|
||||
queryWrapper
|
||||
.lambda()
|
||||
.eq(MetricQueryDefaultConfigDO::getMetricId, metricId)
|
||||
queryWrapper.lambda().eq(MetricQueryDefaultConfigDO::getMetricId, metricId)
|
||||
.eq(MetricQueryDefaultConfigDO::getCreatedBy, userName);
|
||||
return metricQueryDefaultConfigDOMapper.selectOne(queryWrapper);
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ public class ModelRepositoryImpl implements ModelRepository {
|
||||
|
||||
private ModelDOCustomMapper modelDOCustomMapper;
|
||||
|
||||
public ModelRepositoryImpl(
|
||||
ModelDOMapper modelDOMapper, ModelDOCustomMapper modelDOCustomMapper) {
|
||||
public ModelRepositoryImpl(ModelDOMapper modelDOMapper,
|
||||
ModelDOCustomMapper modelDOCustomMapper) {
|
||||
this.modelDOMapper = modelDOMapper;
|
||||
this.modelDOCustomMapper = modelDOCustomMapper;
|
||||
}
|
||||
|
||||
@@ -45,34 +45,24 @@ public class StatRepositoryImpl implements StatRepository {
|
||||
List<ItemUseResp> result = new ArrayList<>();
|
||||
List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
|
||||
Map<String, Long> map = new ConcurrentHashMap<>();
|
||||
statInfos.stream()
|
||||
.forEach(
|
||||
stat -> {
|
||||
String dimensions = stat.getDimensions();
|
||||
String metrics = stat.getMetrics();
|
||||
if (Objects.nonNull(stat.getDataSetId())) {
|
||||
updateStatMapInfo(
|
||||
map,
|
||||
dimensions,
|
||||
TypeEnums.DIMENSION.name().toLowerCase(),
|
||||
stat.getDataSetId());
|
||||
updateStatMapInfo(
|
||||
map,
|
||||
metrics,
|
||||
TypeEnums.METRIC.name().toLowerCase(),
|
||||
stat.getDataSetId());
|
||||
}
|
||||
});
|
||||
map.forEach(
|
||||
(k, v) -> {
|
||||
Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]);
|
||||
String type = k.split(AT_SYMBOL + AT_SYMBOL)[1];
|
||||
String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2];
|
||||
result.add(new ItemUseResp(classId, type, nameEn, v));
|
||||
});
|
||||
statInfos.stream().forEach(stat -> {
|
||||
String dimensions = stat.getDimensions();
|
||||
String metrics = stat.getMetrics();
|
||||
if (Objects.nonNull(stat.getDataSetId())) {
|
||||
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(),
|
||||
stat.getDataSetId());
|
||||
updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(),
|
||||
stat.getDataSetId());
|
||||
}
|
||||
});
|
||||
map.forEach((k, v) -> {
|
||||
Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]);
|
||||
String type = k.split(AT_SYMBOL + AT_SYMBOL)[1];
|
||||
String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2];
|
||||
result.add(new ItemUseResp(classId, type, nameEn, v));
|
||||
});
|
||||
|
||||
return result.stream()
|
||||
.sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
|
||||
return result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@@ -81,24 +71,21 @@ public class StatRepositoryImpl implements StatRepository {
|
||||
return statMapper.getStatInfo(itemUseCommend);
|
||||
}
|
||||
|
||||
private void updateStatMapInfo(
|
||||
Map<String, Long> map, String dimensions, String type, Long dataSetId) {
|
||||
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type,
|
||||
Long dataSetId) {
|
||||
if (StringUtils.isNotEmpty(dimensions)) {
|
||||
try {
|
||||
List<String> dimensionList =
|
||||
mapper.readValue(dimensions, new TypeReference<List<String>>() {});
|
||||
dimensionList.stream()
|
||||
.forEach(
|
||||
dimension -> {
|
||||
String key =
|
||||
dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL
|
||||
+ AT_SYMBOL + dimension;
|
||||
if (map.containsKey(key)) {
|
||||
map.put(key, map.get(key) + 1);
|
||||
} else {
|
||||
map.put(key, 1L);
|
||||
}
|
||||
});
|
||||
dimensionList.stream().forEach(dimension -> {
|
||||
String key = dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL
|
||||
+ dimension;
|
||||
if (map.containsKey(key)) {
|
||||
map.put(key, map.get(key) + 1);
|
||||
} else {
|
||||
map.put(key, 1L);
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.warn("e:{}", e);
|
||||
}
|
||||
|
||||
@@ -63,8 +63,8 @@ public class TagRepositoryImpl implements TagRepository {
|
||||
}
|
||||
if (Objects.nonNull(tagDeleteReq.getTagDefineType())
|
||||
&& CollectionUtils.isNotEmpty(tagDeleteReq.getItemIds())) {
|
||||
tagCustomMapper.deleteBatchByType(
|
||||
tagDeleteReq.getItemIds(), tagDeleteReq.getTagDefineType().name());
|
||||
tagCustomMapper.deleteBatchByType(tagDeleteReq.getItemIds(),
|
||||
tagDeleteReq.getTagDefineType().name());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,24 +16,18 @@ public class EntityInfoProcessor implements ResultProcessor {
|
||||
|
||||
@Override
|
||||
public void process(ParseResp parseResp, ChatQueryContext chatQueryContext) {
|
||||
parseResp
|
||||
.getSelectedParses()
|
||||
.forEach(
|
||||
parseInfo -> {
|
||||
String queryMode = parseInfo.getQueryMode();
|
||||
if (!QueryManager.isDetailQuery(queryMode)
|
||||
&& !QueryManager.isMetricQuery(queryMode)) {
|
||||
return;
|
||||
}
|
||||
parseResp.getSelectedParses().forEach(parseInfo -> {
|
||||
String queryMode = parseInfo.getQueryMode();
|
||||
if (!QueryManager.isDetailQuery(queryMode) && !QueryManager.isMetricQuery(queryMode)) {
|
||||
return;
|
||||
}
|
||||
|
||||
SemanticLayerService semanticService =
|
||||
ContextUtils.getBean(SemanticLayerService.class);
|
||||
DataSetSchema dataSetSchema =
|
||||
semanticService.getDataSetSchema(parseInfo.getDataSetId());
|
||||
EntityInfo entityInfo =
|
||||
semanticService.getEntityInfo(
|
||||
parseInfo, dataSetSchema, chatQueryContext.getUser());
|
||||
parseInfo.setEntityInfo(entityInfo);
|
||||
});
|
||||
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
|
||||
DataSetSchema dataSetSchema =
|
||||
semanticService.getDataSetSchema(parseInfo.getDataSetId());
|
||||
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema,
|
||||
chatQueryContext.getUser());
|
||||
parseInfo.setEntityInfo(entityInfo);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,39 +82,33 @@ public class ParseInfoProcessor implements ResultProcessor {
|
||||
} else if (QueryType.DETAIL.equals(parseInfo.getQueryType())) {
|
||||
List<String> selectFields = SqlSelectHelper.getSelectFields(s2SQL);
|
||||
List<String> selectDimensions = filterDateField(dsSchema, selectFields);
|
||||
parseInfo.setDimensions(
|
||||
matchSchemaElements(selectDimensions, dsSchema.getDimensions()));
|
||||
parseInfo
|
||||
.setDimensions(matchSchemaElements(selectDimensions, dsSchema.getDimensions()));
|
||||
}
|
||||
}
|
||||
|
||||
private Set<SchemaElement> matchSchemaElements(
|
||||
List<String> allFields, Set<SchemaElement> elements) {
|
||||
return elements.stream()
|
||||
.filter(
|
||||
schemaElement -> {
|
||||
if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
|
||||
return allFields.contains(schemaElement.getName());
|
||||
}
|
||||
Set<String> allFieldsSet = new HashSet<>(allFields);
|
||||
Set<String> aliasSet = new HashSet<>(schemaElement.getAlias());
|
||||
List<String> intersection =
|
||||
allFieldsSet.stream()
|
||||
.filter(aliasSet::contains)
|
||||
.collect(Collectors.toList());
|
||||
return allFields.contains(schemaElement.getName())
|
||||
|| !CollectionUtils.isEmpty(intersection);
|
||||
})
|
||||
.collect(Collectors.toSet());
|
||||
private Set<SchemaElement> matchSchemaElements(List<String> allFields,
|
||||
Set<SchemaElement> elements) {
|
||||
return elements.stream().filter(schemaElement -> {
|
||||
if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
|
||||
return allFields.contains(schemaElement.getName());
|
||||
}
|
||||
Set<String> allFieldsSet = new HashSet<>(allFields);
|
||||
Set<String> aliasSet = new HashSet<>(schemaElement.getAlias());
|
||||
List<String> intersection =
|
||||
allFieldsSet.stream().filter(aliasSet::contains).collect(Collectors.toList());
|
||||
return allFields.contains(schemaElement.getName())
|
||||
|| !CollectionUtils.isEmpty(intersection);
|
||||
}).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private List<String> filterDateField(DataSetSchema dataSetSchema, List<String> allFields) {
|
||||
return allFields.stream()
|
||||
.filter(entry -> !isPartitionDimension(dataSetSchema, entry))
|
||||
return allFields.stream().filter(entry -> !isPartitionDimension(dataSetSchema, entry))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<QueryFilter> extractDimensionFilter(
|
||||
DataSetSchema dsSchema, List<FieldExpression> fieldExpressions) {
|
||||
private List<QueryFilter> extractDimensionFilter(DataSetSchema dsSchema,
|
||||
List<FieldExpression> fieldExpressions) {
|
||||
|
||||
Map<String, SchemaElement> fieldNameToElement = getNameToElement(dsSchema);
|
||||
List<QueryFilter> result = Lists.newArrayList();
|
||||
@@ -139,15 +133,11 @@ public class ParseInfoProcessor implements ResultProcessor {
|
||||
return result;
|
||||
}
|
||||
|
||||
private DateConf extractDateFilter(
|
||||
List<FieldExpression> fieldExpressions, DataSetSchema dataSetSchema) {
|
||||
List<FieldExpression> dateExpressions =
|
||||
fieldExpressions.stream()
|
||||
.filter(
|
||||
expression ->
|
||||
isPartitionDimension(
|
||||
dataSetSchema, expression.getFieldName()))
|
||||
.collect(Collectors.toList());
|
||||
private DateConf extractDateFilter(List<FieldExpression> fieldExpressions,
|
||||
DataSetSchema dataSetSchema) {
|
||||
List<FieldExpression> dateExpressions = fieldExpressions.stream().filter(
|
||||
expression -> isPartitionDimension(dataSetSchema, expression.getFieldName()))
|
||||
.collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(dateExpressions)) {
|
||||
return null;
|
||||
}
|
||||
@@ -164,20 +154,14 @@ public class ParseInfoProcessor implements ResultProcessor {
|
||||
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
|
||||
return dateInfo;
|
||||
}
|
||||
if (containOperators(
|
||||
firstExpression,
|
||||
firstOperator,
|
||||
FilterOperatorEnum.GREATER_THAN,
|
||||
if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.GREATER_THAN,
|
||||
FilterOperatorEnum.GREATER_THAN_EQUALS)) {
|
||||
dateInfo.setStartDate(firstExpression.getFieldValue().toString());
|
||||
if (hasSecondDate(dateExpressions)) {
|
||||
dateInfo.setEndDate(dateExpressions.get(1).getFieldValue().toString());
|
||||
}
|
||||
}
|
||||
if (containOperators(
|
||||
firstExpression,
|
||||
firstOperator,
|
||||
FilterOperatorEnum.MINOR_THAN,
|
||||
if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.MINOR_THAN,
|
||||
FilterOperatorEnum.MINOR_THAN_EQUALS)) {
|
||||
dateInfo.setEndDate(firstExpression.getFieldValue().toString());
|
||||
if (hasSecondDate(dateExpressions)) {
|
||||
@@ -191,17 +175,14 @@ public class ParseInfoProcessor implements ResultProcessor {
|
||||
if (TimeDimensionEnum.containsTimeDimension(sqlFieldName)) {
|
||||
return true;
|
||||
}
|
||||
if (Objects.isNull(dataSetSchema)
|
||||
|| Objects.isNull(dataSetSchema.getPartitionDimension())
|
||||
if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension())
|
||||
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
|
||||
return false;
|
||||
}
|
||||
return sqlFieldName.equalsIgnoreCase(dataSetSchema.getPartitionDimension().getName());
|
||||
}
|
||||
|
||||
private boolean containOperators(
|
||||
FieldExpression expression,
|
||||
FilterOperatorEnum firstOperator,
|
||||
private boolean containOperators(FieldExpression expression, FilterOperatorEnum firstOperator,
|
||||
FilterOperatorEnum... operatorEnums) {
|
||||
return (Arrays.asList(operatorEnums).contains(firstOperator)
|
||||
&& Objects.nonNull(expression.getFieldValue()));
|
||||
@@ -220,21 +201,16 @@ public class ParseInfoProcessor implements ResultProcessor {
|
||||
allElements.addAll(dimensions);
|
||||
allElements.addAll(metrics);
|
||||
// support alias
|
||||
return allElements.stream()
|
||||
.flatMap(
|
||||
schemaElement -> {
|
||||
Set<Pair<String, SchemaElement>> result = new HashSet<>();
|
||||
result.add(Pair.of(schemaElement.getName(), schemaElement));
|
||||
List<String> aliasList = schemaElement.getAlias();
|
||||
if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) {
|
||||
for (String alias : aliasList) {
|
||||
result.add(Pair.of(alias, schemaElement));
|
||||
}
|
||||
}
|
||||
return result.stream();
|
||||
})
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Pair::getLeft, Pair::getRight, (value1, value2) -> value2));
|
||||
return allElements.stream().flatMap(schemaElement -> {
|
||||
Set<Pair<String, SchemaElement>> result = new HashSet<>();
|
||||
result.add(Pair.of(schemaElement.getName(), schemaElement));
|
||||
List<String> aliasList = schemaElement.getAlias();
|
||||
if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) {
|
||||
for (String alias : aliasList) {
|
||||
result.add(Pair.of(alias, schemaElement));
|
||||
}
|
||||
}
|
||||
return result.stream();
|
||||
}).collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (value1, value2) -> value2));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,28 +25,27 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
@RequestMapping("/api/semantic/app")
|
||||
public class AppController {
|
||||
|
||||
@Autowired private AppService appService;
|
||||
@Autowired
|
||||
private AppService appService;
|
||||
|
||||
@PostMapping
|
||||
public boolean save(
|
||||
@RequestBody AppReq app, HttpServletRequest request, HttpServletResponse response) {
|
||||
public boolean save(@RequestBody AppReq app, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
appService.save(app, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PutMapping
|
||||
public boolean update(
|
||||
@RequestBody AppReq app, HttpServletRequest request, HttpServletResponse response) {
|
||||
public boolean update(@RequestBody AppReq app, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
appService.update(app, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PutMapping("/online/{id}")
|
||||
public boolean online(
|
||||
@PathVariable("id") Integer id,
|
||||
HttpServletRequest request,
|
||||
public boolean online(@PathVariable("id") Integer id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
appService.online(id, user);
|
||||
@@ -54,9 +53,7 @@ public class AppController {
|
||||
}
|
||||
|
||||
@PutMapping("/offline/{id}")
|
||||
public boolean offline(
|
||||
@PathVariable("id") Integer id,
|
||||
HttpServletRequest request,
|
||||
public boolean offline(@PathVariable("id") Integer id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
appService.offline(id, user);
|
||||
@@ -64,9 +61,7 @@ public class AppController {
|
||||
}
|
||||
|
||||
@DeleteMapping("/{id}")
|
||||
public boolean delete(
|
||||
@PathVariable("id") Integer id,
|
||||
HttpServletRequest request,
|
||||
public boolean delete(@PathVariable("id") Integer id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
appService.delete(id, user);
|
||||
@@ -74,19 +69,15 @@ public class AppController {
|
||||
}
|
||||
|
||||
@GetMapping("/{id}")
|
||||
public AppDetailResp getApp(
|
||||
@PathVariable("id") Integer id,
|
||||
HttpServletRequest request,
|
||||
public AppDetailResp getApp(@PathVariable("id") Integer id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return appService.getApp(id, user);
|
||||
}
|
||||
|
||||
@PostMapping("/page")
|
||||
public PageInfo<AppResp> pageApp(
|
||||
@RequestBody AppQueryReq appQueryReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public PageInfo<AppResp> pageApp(@RequestBody AppQueryReq appQueryReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return appService.pageApp(appQueryReq, user);
|
||||
}
|
||||
|
||||
@@ -24,13 +24,12 @@ import java.util.List;
|
||||
@RequestMapping("/api/semantic/viewInfo")
|
||||
public class CanvasController {
|
||||
|
||||
@Autowired private CanvasService canvasService;
|
||||
@Autowired
|
||||
private CanvasService canvasService;
|
||||
|
||||
@PostMapping("/createOrUpdateViewInfo")
|
||||
public CanvasDO createOrUpdateCanvas(
|
||||
@RequestBody CanvasReq canvasReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public CanvasDO createOrUpdateCanvas(@RequestBody CanvasReq canvasReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return canvasService.createOrUpdateCanvas(canvasReq, user);
|
||||
}
|
||||
@@ -46,10 +45,8 @@ public class CanvasController {
|
||||
}
|
||||
|
||||
@GetMapping("/getDomainSchemaRela/{domainId}")
|
||||
public List<CanvasSchemaResp> getDomainSchema(
|
||||
@PathVariable("domainId") Long domainId,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public List<CanvasSchemaResp> getDomainSchema(@PathVariable("domainId") Long domainId,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return canvasService.getCanvasSchema(domainId, user);
|
||||
}
|
||||
|
||||
@@ -40,9 +40,7 @@ public class ClassController {
|
||||
* @return
|
||||
*/
|
||||
@PostMapping("/create")
|
||||
public ClassResp create(
|
||||
@RequestBody @Valid ClassReq classReq,
|
||||
HttpServletRequest request,
|
||||
public ClassResp create(@RequestBody @Valid ClassReq classReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return classService.create(classReq, user);
|
||||
@@ -57,9 +55,7 @@ public class ClassController {
|
||||
* @return
|
||||
*/
|
||||
@PutMapping("/update")
|
||||
public ClassResp update(
|
||||
@RequestBody @Valid ClassReq classReq,
|
||||
HttpServletRequest request,
|
||||
public ClassResp update(@RequestBody @Valid ClassReq classReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return classService.update(classReq, user);
|
||||
@@ -75,12 +71,8 @@ public class ClassController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@DeleteMapping("delete/{id}/{force}")
|
||||
public Boolean delete(
|
||||
@PathVariable("id") Long id,
|
||||
@PathVariable("force") Boolean force,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Boolean delete(@PathVariable("id") Long id, @PathVariable("force") Boolean force,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return classService.delete(id, force, user);
|
||||
}
|
||||
@@ -95,9 +87,7 @@ public class ClassController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@GetMapping("delete/{id}/{force}")
|
||||
public List<ClassResp> get(
|
||||
@RequestBody @Valid ClassFilter filter,
|
||||
HttpServletRequest request,
|
||||
public List<ClassResp> get(@RequestBody @Valid ClassFilter filter, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return classService.getClassList(filter, user);
|
||||
|
||||
@@ -26,27 +26,23 @@ public class CollectController {
|
||||
}
|
||||
|
||||
@PostMapping("/createCollectionIndicators")
|
||||
public boolean createCollectionIndicators(
|
||||
@RequestBody CollectDO collectDO,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public boolean createCollectionIndicators(@RequestBody CollectDO collectDO,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return collectService.collect(user, collectDO);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@DeleteMapping("/deleteCollectionIndicators/{id}")
|
||||
public boolean deleteCollectionIndicators(
|
||||
@PathVariable Long id, HttpServletRequest request, HttpServletResponse response) {
|
||||
public boolean deleteCollectionIndicators(@PathVariable Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return collectService.unCollect(user, id);
|
||||
}
|
||||
|
||||
@PostMapping("/deleteCollectionIndicators")
|
||||
public boolean deleteCollectionIndicators(
|
||||
@RequestBody CollectDO collectDO,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public boolean deleteCollectionIndicators(@RequestBody CollectDO collectDO,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return collectService.unCollect(user, collectDO);
|
||||
}
|
||||
|
||||
@@ -26,21 +26,18 @@ import java.util.List;
|
||||
@RequestMapping("/api/semantic/dataSet")
|
||||
public class DataSetController {
|
||||
|
||||
@Autowired private DataSetService dataSetService;
|
||||
@Autowired
|
||||
private DataSetService dataSetService;
|
||||
|
||||
@PostMapping
|
||||
public DataSetResp save(
|
||||
@RequestBody DataSetReq dataSetReq,
|
||||
HttpServletRequest request,
|
||||
public DataSetResp save(@RequestBody DataSetReq dataSetReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return dataSetService.save(dataSetReq, user);
|
||||
}
|
||||
|
||||
@PutMapping
|
||||
public DataSetResp update(
|
||||
@RequestBody DataSetReq dataSetReq,
|
||||
HttpServletRequest request,
|
||||
public DataSetResp update(@RequestBody DataSetReq dataSetReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return dataSetService.update(dataSetReq, user);
|
||||
@@ -59,8 +56,8 @@ public class DataSetController {
|
||||
}
|
||||
|
||||
@DeleteMapping("/{id}")
|
||||
public Boolean delete(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
|
||||
public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
dataSetService.delete(id, user);
|
||||
return true;
|
||||
|
||||
@@ -37,33 +37,29 @@ public class DatabaseController {
|
||||
}
|
||||
|
||||
@PostMapping("/testConnect")
|
||||
public boolean testConnect(
|
||||
@RequestBody DatabaseReq databaseReq,
|
||||
HttpServletRequest request,
|
||||
public boolean testConnect(@RequestBody DatabaseReq databaseReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return databaseService.testConnect(databaseReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/createOrUpdateDatabase")
|
||||
public DatabaseResp createOrUpdateDatabase(
|
||||
@RequestBody DatabaseReq databaseReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public DatabaseResp createOrUpdateDatabase(@RequestBody DatabaseReq databaseReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return databaseService.createOrUpdateDatabase(databaseReq, user);
|
||||
}
|
||||
|
||||
@GetMapping("/{id}")
|
||||
public DatabaseResp getDatabase(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
|
||||
public DatabaseResp getDatabase(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return databaseService.getDatabase(id, user);
|
||||
}
|
||||
|
||||
@GetMapping("/getDatabaseList")
|
||||
public List<DatabaseResp> getDatabaseList(
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
public List<DatabaseResp> getDatabaseList(HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return databaseService.getDatabaseList(user);
|
||||
}
|
||||
@@ -75,10 +71,8 @@ public class DatabaseController {
|
||||
}
|
||||
|
||||
@PostMapping("/executeSql")
|
||||
public SemanticQueryResp executeSql(
|
||||
@RequestBody SqlExecuteReq sqlExecuteReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user);
|
||||
}
|
||||
@@ -89,17 +83,14 @@ public class DatabaseController {
|
||||
}
|
||||
|
||||
@RequestMapping("/getTables")
|
||||
public List<String> getTables(
|
||||
@RequestParam("databaseId") Long databaseId, @RequestParam("db") String db)
|
||||
throws SQLException {
|
||||
public List<String> getTables(@RequestParam("databaseId") Long databaseId,
|
||||
@RequestParam("db") String db) throws SQLException {
|
||||
return databaseService.getTables(databaseId, db);
|
||||
}
|
||||
|
||||
@RequestMapping("/getColumnsByName")
|
||||
public List<DBColumn> getColumnsByName(
|
||||
@RequestParam("databaseId") Long databaseId,
|
||||
@RequestParam("db") String db,
|
||||
@RequestParam("table") String table)
|
||||
public List<DBColumn> getColumnsByName(@RequestParam("databaseId") Long databaseId,
|
||||
@RequestParam("db") String db, @RequestParam("table") String table)
|
||||
throws SQLException {
|
||||
return databaseService.getColumns(databaseId, db, table);
|
||||
}
|
||||
@@ -110,8 +101,8 @@ public class DatabaseController {
|
||||
}
|
||||
|
||||
@GetMapping("/getDatabaseParameters")
|
||||
public Map<String, List<DatabaseParameter>> getDatabaseParameters(
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
public Map<String, List<DatabaseParameter>> getDatabaseParameters(HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return databaseService.getDatabaseParameters(user);
|
||||
}
|
||||
|
||||
@@ -34,9 +34,11 @@ import java.util.List;
|
||||
@RequestMapping("/api/semantic/dimension")
|
||||
public class DimensionController {
|
||||
|
||||
@Autowired private DimensionService dimensionService;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
|
||||
@Autowired private SemanticLayerService queryService;
|
||||
@Autowired
|
||||
private SemanticLayerService queryService;
|
||||
|
||||
/**
|
||||
* 创建维度
|
||||
@@ -44,60 +46,46 @@ public class DimensionController {
|
||||
* @param dimensionReq
|
||||
*/
|
||||
@PostMapping("/createDimension")
|
||||
public DimensionResp createDimension(
|
||||
@RequestBody DimensionReq dimensionReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public DimensionResp createDimension(@RequestBody DimensionReq dimensionReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return dimensionService.createDimension(dimensionReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/updateDimension")
|
||||
public Boolean updateDimension(
|
||||
@RequestBody DimensionReq dimensionReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Boolean updateDimension(@RequestBody DimensionReq dimensionReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
dimensionService.updateDimension(dimensionReq, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/batchUpdateStatus")
|
||||
public Boolean batchUpdateStatus(
|
||||
@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
dimensionService.batchUpdateStatus(metaBatchReq, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/batchUpdateSensitiveLevel")
|
||||
public Boolean batchUpdateSensitiveLevel(
|
||||
@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public Boolean batchUpdateSensitiveLevel(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
dimensionService.batchUpdateSensitiveLevel(metaBatchReq, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/mockDimensionAlias")
|
||||
public List<String> mockMetricAlias(
|
||||
@RequestBody DimensionReq dimensionReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public List<String> mockMetricAlias(@RequestBody DimensionReq dimensionReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return dimensionService.mockAlias(dimensionReq, "dimension", user);
|
||||
}
|
||||
|
||||
@PostMapping("/mockDimensionValuesAlias")
|
||||
public List<DimValueMap> mockDimensionValuesAlias(
|
||||
@RequestBody DimensionReq dimensionReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public List<DimValueMap> mockDimensionValuesAlias(@RequestBody DimensionReq dimensionReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return dimensionService.mockDimensionValueAlias(dimensionReq, user);
|
||||
}
|
||||
@@ -115,8 +103,7 @@ public class DimensionController {
|
||||
}
|
||||
|
||||
@GetMapping("/{modelId}/{dimensionName}")
|
||||
public DimensionResp getDimensionDescByNameAndId(
|
||||
@PathVariable("modelId") Long modelId,
|
||||
public DimensionResp getDimensionDescByNameAndId(@PathVariable("modelId") Long modelId,
|
||||
@PathVariable("dimensionName") String dimensionBizName) {
|
||||
return dimensionService.getDimension(dimensionBizName, modelId);
|
||||
}
|
||||
@@ -127,17 +114,15 @@ public class DimensionController {
|
||||
}
|
||||
|
||||
@PostMapping("/queryDimValue")
|
||||
public SemanticQueryResp queryDimValue(
|
||||
@RequestBody DimensionValueReq dimensionValueReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public SemanticQueryResp queryDimValue(@RequestBody DimensionValueReq dimensionValueReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryService.queryDimensionValue(dimensionValueReq, user);
|
||||
}
|
||||
|
||||
@DeleteMapping("deleteDimension/{id}")
|
||||
public Boolean deleteDimension(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
|
||||
public Boolean deleteDimension(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
dimensionService.deleteDimension(id, user);
|
||||
return true;
|
||||
|
||||
@@ -32,19 +32,15 @@ public class DomainController {
|
||||
}
|
||||
|
||||
@PostMapping("/createDomain")
|
||||
public DomainResp createDomain(
|
||||
@RequestBody DomainReq domainReq,
|
||||
HttpServletRequest request,
|
||||
public DomainResp createDomain(@RequestBody DomainReq domainReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return domainService.createDomain(domainReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/updateDomain")
|
||||
public DomainResp updateDomain(
|
||||
@RequestBody DomainUpdateReq domainUpdateReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public DomainResp updateDomain(@RequestBody DomainUpdateReq domainUpdateReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return domainService.updateDomain(domainUpdateReq, user);
|
||||
}
|
||||
@@ -56,8 +52,8 @@ public class DomainController {
|
||||
}
|
||||
|
||||
@GetMapping("/getDomainList")
|
||||
public List<DomainResp> getDomainList(
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
public List<DomainResp> getDomainList(HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return domainService.getDomainListWithAdminAuth(user);
|
||||
}
|
||||
@@ -69,9 +65,7 @@ public class DomainController {
|
||||
|
||||
@GetMapping("/getDomainListByIds/{domainIds}")
|
||||
public List<DomainResp> getDomainListByIds(@PathVariable("domainIds") String domainIds) {
|
||||
return domainService.getDomainList(
|
||||
Arrays.stream(domainIds.split(","))
|
||||
.map(Long::parseLong)
|
||||
.collect(Collectors.toList()));
|
||||
return domainService.getDomainList(Arrays.stream(domainIds.split(",")).map(Long::parseLong)
|
||||
.collect(Collectors.toList()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,17 +34,23 @@ import java.util.List;
|
||||
@RequestMapping("/api/semantic/knowledge")
|
||||
public class KnowledgeController {
|
||||
|
||||
@Autowired private DictTaskService taskService;
|
||||
@Autowired
|
||||
private DictTaskService taskService;
|
||||
|
||||
@Autowired private DictConfService confService;
|
||||
@Autowired
|
||||
private DictConfService confService;
|
||||
|
||||
@Autowired private MetaEmbeddingTask metaEmbeddingTask;
|
||||
@Autowired
|
||||
private MetaEmbeddingTask metaEmbeddingTask;
|
||||
|
||||
@Autowired private DictionaryReloadTask dictionaryReloadTask;
|
||||
@Autowired
|
||||
private DictionaryReloadTask dictionaryReloadTask;
|
||||
|
||||
@Autowired private ExemplarService exemplarService;
|
||||
@Autowired
|
||||
private ExemplarService exemplarService;
|
||||
|
||||
@Autowired private EmbeddingService embeddingService;
|
||||
@Autowired
|
||||
private EmbeddingService embeddingService;
|
||||
|
||||
/**
|
||||
* addDictConf-新增item的字典配置 Add configuration information for dictionary entries
|
||||
@@ -52,10 +58,8 @@ public class KnowledgeController {
|
||||
* @param dictItemReq
|
||||
*/
|
||||
@PostMapping("/conf")
|
||||
public DictItemResp addDictConf(
|
||||
@RequestBody @Valid DictItemReq dictItemReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public DictItemResp addDictConf(@RequestBody @Valid DictItemReq dictItemReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return confService.addDictConf(dictItemReq, user);
|
||||
}
|
||||
@@ -66,10 +70,8 @@ public class KnowledgeController {
|
||||
* @param dictItemReq
|
||||
*/
|
||||
@PutMapping("/conf")
|
||||
public DictItemResp editDictConf(
|
||||
@RequestBody @Valid DictItemReq dictItemReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public DictItemResp editDictConf(@RequestBody @Valid DictItemReq dictItemReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return confService.editDictConf(dictItemReq, user);
|
||||
}
|
||||
@@ -80,10 +82,8 @@ public class KnowledgeController {
|
||||
* @param filter
|
||||
*/
|
||||
@PostMapping("/conf/query")
|
||||
public List<DictItemResp> queryDictConf(
|
||||
@RequestBody @Valid DictItemFilter filter,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public List<DictItemResp> queryDictConf(@RequestBody @Valid DictItemFilter filter,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return confService.queryDictConf(filter, user);
|
||||
}
|
||||
@@ -94,9 +94,7 @@ public class KnowledgeController {
|
||||
* @param taskReq
|
||||
*/
|
||||
@PostMapping("/task")
|
||||
public Long addDictTask(
|
||||
@RequestBody DictSingleTaskReq taskReq,
|
||||
HttpServletRequest request,
|
||||
public Long addDictTask(@RequestBody DictSingleTaskReq taskReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return taskService.addDictTask(taskReq, user);
|
||||
@@ -108,9 +106,7 @@ public class KnowledgeController {
|
||||
* @param taskReq
|
||||
*/
|
||||
@PutMapping("/task/delete")
|
||||
public Long deleteDictTask(
|
||||
@RequestBody DictSingleTaskReq taskReq,
|
||||
HttpServletRequest request,
|
||||
public Long deleteDictTask(@RequestBody DictSingleTaskReq taskReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return taskService.deleteDictTask(taskReq, user);
|
||||
@@ -128,10 +124,8 @@ public class KnowledgeController {
|
||||
* @param taskReq
|
||||
*/
|
||||
@PostMapping("/task/search")
|
||||
public DictTaskResp queryLatestDictTask(
|
||||
@RequestBody DictSingleTaskReq taskReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public DictTaskResp queryLatestDictTask(@RequestBody DictSingleTaskReq taskReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return taskService.queryLatestDictTask(taskReq, user);
|
||||
}
|
||||
@@ -161,10 +155,8 @@ public class KnowledgeController {
|
||||
* @param dictValueReq
|
||||
*/
|
||||
@PostMapping("/dict/data")
|
||||
public PageInfo<DictValueResp> queryDictValue(
|
||||
@RequestBody @Valid DictValueReq dictValueReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public PageInfo<DictValueResp> queryDictValue(@RequestBody @Valid DictValueReq dictValueReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return taskService.queryDictValue(dictValueReq, user);
|
||||
}
|
||||
@@ -175,10 +167,8 @@ public class KnowledgeController {
|
||||
* @param dictValueReq
|
||||
*/
|
||||
@PostMapping("/dict/file")
|
||||
public String queryDictFilePath(
|
||||
@RequestBody @Valid DictValueReq dictValueReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public String queryDictFilePath(@RequestBody @Valid DictValueReq dictValueReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return taskService.queryDictFilePath(dictValueReq, user);
|
||||
}
|
||||
|
||||
@@ -40,39 +40,29 @@ public class MetricController {
|
||||
}
|
||||
|
||||
@PostMapping("/createMetric")
|
||||
public MetricResp createMetric(
|
||||
@RequestBody MetricReq metricReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public MetricResp createMetric(@RequestBody MetricReq metricReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return metricService.createMetric(metricReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/updateMetric")
|
||||
public MetricResp updateMetric(
|
||||
@RequestBody MetricReq metricReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public MetricResp updateMetric(@RequestBody MetricReq metricReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return metricService.updateMetric(metricReq, user);
|
||||
}
|
||||
|
||||
@PostMapping("/batchUpdateStatus")
|
||||
public Boolean batchUpdateStatus(
|
||||
@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.batchUpdateStatus(metaBatchReq, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/batchPublish")
|
||||
public Boolean batchPublish(
|
||||
@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
public Boolean batchPublish(@RequestBody MetaBatchReq metaBatchReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.batchPublish(metaBatchReq.getIds(), user);
|
||||
@@ -80,40 +70,32 @@ public class MetricController {
|
||||
}
|
||||
|
||||
@PostMapping("/batchUnPublish")
|
||||
public Boolean batchUnPublish(
|
||||
@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public Boolean batchUnPublish(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.batchUnPublish(metaBatchReq.getIds(), user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/batchUpdateClassifications")
|
||||
public Boolean batchUpdateClassifications(
|
||||
@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public Boolean batchUpdateClassifications(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.batchUpdateClassifications(metaBatchReq, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/batchUpdateSensitiveLevel")
|
||||
public Boolean batchUpdateSensitiveLevel(
|
||||
@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public Boolean batchUpdateSensitiveLevel(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.batchUpdateSensitiveLevel(metaBatchReq, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/mockMetricAlias")
|
||||
public List<String> mockMetricAlias(
|
||||
@RequestBody MetricBaseReq metricReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public List<String> mockMetricAlias(@RequestBody MetricBaseReq metricReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return metricService.mockAlias(metricReq, "indicator", user);
|
||||
}
|
||||
@@ -130,32 +112,29 @@ public class MetricController {
|
||||
}
|
||||
|
||||
@PostMapping("/queryMetric")
|
||||
public PageInfo<MetricResp> queryMetric(
|
||||
@RequestBody PageMetricReq pageMetricReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public PageInfo<MetricResp> queryMetric(@RequestBody PageMetricReq pageMetricReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return metricService.queryMetricMarket(pageMetricReq, user);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
@GetMapping("getMetric/{modelId}/{bizName}")
|
||||
public MetricResp getMetric(
|
||||
@PathVariable("modelId") Long modelId, @PathVariable("bizName") String bizName) {
|
||||
public MetricResp getMetric(@PathVariable("modelId") Long modelId,
|
||||
@PathVariable("bizName") String bizName) {
|
||||
return metricService.getMetric(modelId, bizName);
|
||||
}
|
||||
|
||||
@GetMapping("getMetric/{id}")
|
||||
public MetricResp getMetric(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
|
||||
public MetricResp getMetric(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return metricService.getMetric(id, user);
|
||||
}
|
||||
|
||||
@DeleteMapping("deleteMetric/{id}")
|
||||
public Boolean deleteMetric(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Boolean deleteMetric(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.deleteMetric(id, user);
|
||||
return true;
|
||||
@@ -186,8 +165,7 @@ public class MetricController {
|
||||
|
||||
@PostMapping("/saveMetricQueryDefaultConfig")
|
||||
public boolean saveMetricQueryDefaultConfig(
|
||||
@RequestBody MetricQueryDefaultConfig queryDefaultConfig,
|
||||
HttpServletRequest request,
|
||||
@RequestBody MetricQueryDefaultConfig queryDefaultConfig, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
metricService.saveMetricQueryDefaultConfig(queryDefaultConfig, user);
|
||||
@@ -196,8 +174,7 @@ public class MetricController {
|
||||
|
||||
@RequestMapping("getMetricQueryDefaultConfig/{metricId}")
|
||||
public MetricQueryDefaultConfig getMetricQueryDefaultConfig(
|
||||
@PathVariable("metricId") Long metricId,
|
||||
HttpServletRequest request,
|
||||
@PathVariable("metricId") Long metricId, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return metricService.getMetricQueryDefaultConfig(metricId, user);
|
||||
|
||||
@@ -39,31 +39,23 @@ public class ModelController {
|
||||
}
|
||||
|
||||
@PostMapping("/createModel")
|
||||
public Boolean createModel(
|
||||
@RequestBody ModelReq modelReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Boolean createModel(@RequestBody ModelReq modelReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
modelService.createModel(modelReq, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@PostMapping("/updateModel")
|
||||
public Boolean updateModel(
|
||||
@RequestBody ModelReq modelReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Boolean updateModel(@RequestBody ModelReq modelReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
modelService.updateModel(modelReq, user);
|
||||
return true;
|
||||
}
|
||||
|
||||
@DeleteMapping("/deleteModel/{modelId}")
|
||||
public Boolean deleteModel(
|
||||
@PathVariable("modelId") Long modelId,
|
||||
HttpServletRequest request,
|
||||
public Boolean deleteModel(@PathVariable("modelId") Long modelId, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
modelService.deleteModel(modelId, user);
|
||||
@@ -71,10 +63,8 @@ public class ModelController {
|
||||
}
|
||||
|
||||
@GetMapping("/getModelList/{domainId}")
|
||||
public List<ModelResp> getModelList(
|
||||
@PathVariable("domainId") Long domainId,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public List<ModelResp> getModelList(@PathVariable("domainId") Long domainId,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN);
|
||||
}
|
||||
@@ -86,10 +76,8 @@ public class ModelController {
|
||||
|
||||
@GetMapping("/getModelListByIds/{modelIds}")
|
||||
public List<ModelResp> getModelListByIds(@PathVariable("modelIds") String modelIds) {
|
||||
List<Long> ids =
|
||||
Arrays.stream(modelIds.split(","))
|
||||
.map(Long::parseLong)
|
||||
.collect(Collectors.toList());
|
||||
List<Long> ids = Arrays.stream(modelIds.split(",")).map(Long::parseLong)
|
||||
.collect(Collectors.toList());
|
||||
ModelFilter modelFilter = new ModelFilter();
|
||||
modelFilter.setIds(ids);
|
||||
return modelService.getModelList(modelFilter);
|
||||
@@ -106,10 +94,8 @@ public class ModelController {
|
||||
}
|
||||
|
||||
@PostMapping("/batchUpdateStatus")
|
||||
public Boolean batchUpdateStatus(
|
||||
@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
modelService.batchUpdateStatus(metaBatchReq, user);
|
||||
return true;
|
||||
|
||||
@@ -19,7 +19,8 @@ import java.util.List;
|
||||
@RequestMapping("/api/semantic/modelRela")
|
||||
public class ModelRelaController {
|
||||
|
||||
@Autowired private ModelRelaService modelRelaService;
|
||||
@Autowired
|
||||
private ModelRelaService modelRelaService;
|
||||
|
||||
@PostMapping
|
||||
public boolean save(@RequestBody ModelRela modelRela, User user) {
|
||||
|
||||
@@ -39,10 +39,8 @@ public class QueryRuleController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/create")
|
||||
public QueryRuleResp create(
|
||||
@RequestBody @Validated QueryRuleReq queryRuleReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public QueryRuleResp create(@RequestBody @Validated QueryRuleReq queryRuleReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryRuleService.addQueryRule(queryRuleReq, user);
|
||||
}
|
||||
@@ -57,10 +55,8 @@ public class QueryRuleController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/update")
|
||||
public QueryRuleResp update(
|
||||
@RequestBody @Validated QueryRuleReq queryRuleReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public QueryRuleResp update(@RequestBody @Validated QueryRuleReq queryRuleReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryRuleService.updateQueryRule(queryRuleReq, user);
|
||||
}
|
||||
@@ -74,8 +70,8 @@ public class QueryRuleController {
|
||||
* @return
|
||||
*/
|
||||
@DeleteMapping("delete/{id}")
|
||||
public Boolean delete(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
|
||||
public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryRuleService.dropQueryRule(id, user);
|
||||
}
|
||||
@@ -88,10 +84,8 @@ public class QueryRuleController {
|
||||
* @return
|
||||
*/
|
||||
@PostMapping("query")
|
||||
public List<QueryRuleResp> query(
|
||||
@RequestBody @Validated QueryRuleFilter queryRuleFilter,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
public List<QueryRuleResp> query(@RequestBody @Validated QueryRuleFilter queryRuleFilter,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return queryRuleService.getQueryRuleList(queryRuleFilter, user);
|
||||
}
|
||||
|
||||
@@ -21,20 +21,19 @@ import java.util.List;
|
||||
@RequestMapping("/api/semantic/schema")
|
||||
public class SchemaController {
|
||||
|
||||
@Autowired private SchemaService schemaService;
|
||||
@Autowired
|
||||
private SchemaService schemaService;
|
||||
|
||||
@GetMapping("/domain/list")
|
||||
public List<DomainResp> getDomainList(
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
public List<DomainResp> getDomainList(HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return schemaService.getDomainList(user);
|
||||
}
|
||||
|
||||
@GetMapping("/model/list")
|
||||
public List<ModelResp> getModelList(
|
||||
@RequestParam("domainId") Long domainId,
|
||||
@RequestParam("authType") String authType,
|
||||
HttpServletRequest request,
|
||||
public List<ModelResp> getModelList(@RequestParam("domainId") Long domainId,
|
||||
@RequestParam("authType") String authType, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return schemaService.getModelList(user, AuthType.valueOf(authType), domainId);
|
||||
|
||||
@@ -49,9 +49,8 @@ public class TagController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/create")
|
||||
public TagResp create(
|
||||
@RequestBody TagReq tagReq, HttpServletRequest request, HttpServletResponse response)
|
||||
throws Exception {
|
||||
public TagResp create(@RequestBody TagReq tagReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagMetaService.create(tagReq, user);
|
||||
}
|
||||
@@ -66,11 +65,8 @@ public class TagController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/create/batch")
|
||||
public Integer createBatch(
|
||||
@RequestBody @Valid List<TagReq> tagReqList,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Integer createBatch(@RequestBody @Valid List<TagReq> tagReqList,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagMetaService.createBatch(tagReqList, user);
|
||||
}
|
||||
@@ -85,11 +81,8 @@ public class TagController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/delete/batch")
|
||||
public Boolean deleteBatch(
|
||||
@RequestBody @Valid List<TagDeleteReq> tagDeleteReqList,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Boolean deleteBatch(@RequestBody @Valid List<TagDeleteReq> tagDeleteReqList,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagMetaService.deleteBatch(tagDeleteReqList, user);
|
||||
}
|
||||
@@ -104,8 +97,8 @@ public class TagController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@DeleteMapping("delete/{id}")
|
||||
public Boolean delete(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
|
||||
public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
tagMetaService.delete(id, user);
|
||||
return true;
|
||||
@@ -120,8 +113,8 @@ public class TagController {
|
||||
* @return
|
||||
*/
|
||||
@GetMapping("getTag/{id}")
|
||||
public TagResp getTag(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
|
||||
public TagResp getTag(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagMetaService.getTag(id, user);
|
||||
}
|
||||
@@ -147,11 +140,8 @@ public class TagController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/value/distribution")
|
||||
public ItemValueResp queryTagValue(
|
||||
@RequestBody ItemValueReq itemValueReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public ItemValueResp queryTagValue(@RequestBody ItemValueReq itemValueReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagQueryService.queryTagValue(itemValueReq, user);
|
||||
}
|
||||
@@ -166,11 +156,8 @@ public class TagController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/queryTag/market")
|
||||
public PageInfo<TagResp> queryTagMarketPage(
|
||||
@RequestBody TagFilterPageReq tagMarketPageReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public PageInfo<TagResp> queryTagMarketPage(@RequestBody TagFilterPageReq tagMarketPageReq,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagMetaService.queryTagMarketPage(tagMarketPageReq, user);
|
||||
}
|
||||
|
||||
@@ -38,11 +38,8 @@ public class TagObjectController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/create")
|
||||
public TagObjectResp create(
|
||||
@RequestBody TagObjectReq tagObjectReq,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public TagObjectResp create(@RequestBody TagObjectReq tagObjectReq, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagObjectService.create(tagObjectReq, user);
|
||||
}
|
||||
@@ -56,9 +53,7 @@ public class TagObjectController {
|
||||
* @return
|
||||
*/
|
||||
@PostMapping("/update")
|
||||
public TagObjectResp update(
|
||||
@RequestBody TagObjectReq tagObjectReq,
|
||||
HttpServletRequest request,
|
||||
public TagObjectResp update(@RequestBody TagObjectReq tagObjectReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagObjectService.update(tagObjectReq, user);
|
||||
@@ -74,9 +69,8 @@ public class TagObjectController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@DeleteMapping("delete/{id}")
|
||||
public Boolean delete(
|
||||
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response)
|
||||
throws Exception {
|
||||
public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request,
|
||||
HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
tagObjectService.delete(id, user, true);
|
||||
return true;
|
||||
@@ -92,11 +86,8 @@ public class TagObjectController {
|
||||
* @throws Exception
|
||||
*/
|
||||
@PostMapping("/query")
|
||||
public List<TagObjectResp> queryTagObject(
|
||||
@RequestBody TagObjectFilter filter,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws Exception {
|
||||
public List<TagObjectResp> queryTagObject(@RequestBody TagObjectFilter filter,
|
||||
HttpServletRequest request, HttpServletResponse response) throws Exception {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return tagObjectService.getTagObjects(filter, user);
|
||||
}
|
||||
|
||||
@@ -25,12 +25,11 @@ import java.util.List;
|
||||
@RequestMapping("/api/semantic/term")
|
||||
public class TermController {
|
||||
|
||||
@Autowired private TermService termService;
|
||||
@Autowired
|
||||
private TermService termService;
|
||||
|
||||
@PostMapping("/saveOrUpdate")
|
||||
public boolean saveOrUpdate(
|
||||
@RequestBody TermReq termReq,
|
||||
HttpServletRequest request,
|
||||
public boolean saveOrUpdate(@RequestBody TermReq termReq, HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
termService.saveOrUpdate(termReq, user);
|
||||
@@ -38,8 +37,7 @@ public class TermController {
|
||||
}
|
||||
|
||||
@GetMapping
|
||||
public List<TermResp> getTerms(
|
||||
@RequestParam("domainId") Long domainId,
|
||||
public List<TermResp> getTerms(@RequestParam("domainId") Long domainId,
|
||||
@RequestParam(name = "queryKey", required = false) String queryKey) {
|
||||
return termService.getTerms(domainId, queryKey);
|
||||
}
|
||||
|
||||
@@ -8,9 +8,8 @@ import com.tencent.supersonic.headless.api.pojo.request.DownloadMetricReq;
|
||||
|
||||
public interface DownloadService {
|
||||
|
||||
void downloadByStruct(
|
||||
DownloadMetricReq downloadStructReq, User user, HttpServletResponse response)
|
||||
throws Exception;
|
||||
void downloadByStruct(DownloadMetricReq downloadStructReq, User user,
|
||||
HttpServletResponse response) throws Exception;
|
||||
|
||||
void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response)
|
||||
throws Exception;
|
||||
|
||||
@@ -57,11 +57,9 @@ public interface SchemaService {
|
||||
|
||||
List<ItemResp> getDomainDataSetTree();
|
||||
|
||||
void getSchemaYamlTpl(
|
||||
SemanticSchemaResp semanticSchemaResp,
|
||||
void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
|
||||
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
|
||||
List<DataModelYamlTpl> dataModelYamlTplList,
|
||||
List<MetricYamlTpl> metricYamlTplList,
|
||||
List<DataModelYamlTpl> dataModelYamlTplList, List<MetricYamlTpl> metricYamlTplList,
|
||||
Map<Long, String> modelIdName);
|
||||
|
||||
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
|
||||
|
||||
@@ -44,8 +44,8 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
|
||||
|
||||
private DimensionService dimensionService;
|
||||
|
||||
public AppServiceImpl(
|
||||
AppMapper appMapper, MetricService metricService, DimensionService dimensionService) {
|
||||
public AppServiceImpl(AppMapper appMapper, MetricService metricService,
|
||||
DimensionService dimensionService) {
|
||||
this.appMapper = appMapper;
|
||||
this.metricService = metricService;
|
||||
this.dimensionService = dimensionService;
|
||||
@@ -110,16 +110,13 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
|
||||
PageHelper.startPage(appQueryReq.getCurrent(), appQueryReq.getPageSize())
|
||||
.doSelectPageInfo(() -> queryApp(appQueryReq));
|
||||
PageInfo<AppResp> appPageInfo = PageUtils.pageInfo2PageInfoVo(appDOPageInfo);
|
||||
Map<Long, MetricResp> metricResps =
|
||||
metricService.getMetrics(new MetaFilter()).stream()
|
||||
.collect(Collectors.toMap(MetricResp::getId, m -> m));
|
||||
Map<Long, DimensionResp> dimensionResps =
|
||||
dimensionService.getDimensions(new MetaFilter()).stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getId, m -> m));
|
||||
appPageInfo.setList(
|
||||
appDOPageInfo.getList().stream()
|
||||
.map(appDO -> convert(appDO, dimensionResps, metricResps, user))
|
||||
.collect(Collectors.toList()));
|
||||
Map<Long, MetricResp> metricResps = metricService.getMetrics(new MetaFilter()).stream()
|
||||
.collect(Collectors.toMap(MetricResp::getId, m -> m));
|
||||
Map<Long, DimensionResp> dimensionResps = dimensionService.getDimensions(new MetaFilter())
|
||||
.stream().collect(Collectors.toMap(DimensionResp::getId, m -> m));
|
||||
appPageInfo.setList(appDOPageInfo.getList().stream()
|
||||
.map(appDO -> convert(appDO, dimensionResps, metricResps, user))
|
||||
.collect(Collectors.toList()));
|
||||
return appPageInfo;
|
||||
}
|
||||
|
||||
@@ -141,12 +138,10 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
|
||||
@Override
|
||||
public AppDetailResp getApp(Integer id, User user) {
|
||||
AppDO appDO = getAppDO(id);
|
||||
Map<Long, MetricResp> metricResps =
|
||||
metricService.getMetrics(new MetaFilter()).stream()
|
||||
.collect(Collectors.toMap(MetricResp::getId, m -> m));
|
||||
Map<Long, DimensionResp> dimensionResps =
|
||||
dimensionService.getDimensions(new MetaFilter()).stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getId, m -> m));
|
||||
Map<Long, MetricResp> metricResps = metricService.getMetrics(new MetaFilter()).stream()
|
||||
.collect(Collectors.toMap(MetricResp::getId, m -> m));
|
||||
Map<Long, DimensionResp> dimensionResps = dimensionService.getDimensions(new MetaFilter())
|
||||
.stream().collect(Collectors.toMap(DimensionResp::getId, m -> m));
|
||||
checkAuth(appDO, user);
|
||||
return convertDetail(appDO, dimensionResps, metricResps);
|
||||
}
|
||||
@@ -179,11 +174,8 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
|
||||
&& appDO.getOwner().contains(user.getName());
|
||||
}
|
||||
|
||||
private AppResp convert(
|
||||
AppDO appDO,
|
||||
Map<Long, DimensionResp> dimensionMap,
|
||||
Map<Long, MetricResp> metricMap,
|
||||
User user) {
|
||||
private AppResp convert(AppDO appDO, Map<Long, DimensionResp> dimensionMap,
|
||||
Map<Long, MetricResp> metricMap, User user) {
|
||||
AppResp app = new AppResp();
|
||||
BeanMapper.mapper(appDO, app);
|
||||
AppConfig appConfig = JSONObject.parseObject(appDO.getConfig(), AppConfig.class);
|
||||
@@ -198,8 +190,8 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
|
||||
return convertDetail(appDO, new HashMap<>(), new HashMap<>());
|
||||
}
|
||||
|
||||
private AppDetailResp convertDetail(
|
||||
AppDO appDO, Map<Long, DimensionResp> dimensionMap, Map<Long, MetricResp> metricMap) {
|
||||
private AppDetailResp convertDetail(AppDO appDO, Map<Long, DimensionResp> dimensionMap,
|
||||
Map<Long, MetricResp> metricMap) {
|
||||
AppDetailResp app = new AppDetailResp();
|
||||
BeanMapper.mapper(appDO, app);
|
||||
AppConfig appConfig = JSONObject.parseObject(appDO.getConfig(), AppConfig.class);
|
||||
@@ -209,50 +201,24 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
|
||||
return app;
|
||||
}
|
||||
|
||||
private void fillItemName(
|
||||
AppConfig appConfig,
|
||||
Map<Long, DimensionResp> dimensionMap,
|
||||
private void fillItemName(AppConfig appConfig, Map<Long, DimensionResp> dimensionMap,
|
||||
Map<Long, MetricResp> metricMap) {
|
||||
appConfig
|
||||
.getItems()
|
||||
.forEach(
|
||||
metricItem -> {
|
||||
metricItem.setName(
|
||||
metricMap
|
||||
.getOrDefault(metricItem.getId(), new MetricResp())
|
||||
.getName());
|
||||
metricItem.setBizName(
|
||||
metricMap
|
||||
.getOrDefault(metricItem.getId(), new MetricResp())
|
||||
.getBizName());
|
||||
metricItem.setCreatedBy(
|
||||
metricMap
|
||||
.getOrDefault(metricItem.getId(), new MetricResp())
|
||||
.getCreatedBy());
|
||||
metricItem
|
||||
.getRelateItems()
|
||||
.forEach(
|
||||
dimensionItem -> {
|
||||
dimensionItem.setName(
|
||||
dimensionMap
|
||||
.getOrDefault(
|
||||
dimensionItem.getId(),
|
||||
new DimensionResp())
|
||||
.getName());
|
||||
dimensionItem.setBizName(
|
||||
dimensionMap
|
||||
.getOrDefault(
|
||||
dimensionItem.getId(),
|
||||
new DimensionResp())
|
||||
.getBizName());
|
||||
dimensionItem.setCreatedBy(
|
||||
dimensionMap
|
||||
.getOrDefault(
|
||||
dimensionItem.getId(),
|
||||
new DimensionResp())
|
||||
.getCreatedBy());
|
||||
});
|
||||
});
|
||||
appConfig.getItems().forEach(metricItem -> {
|
||||
metricItem.setName(
|
||||
metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getName());
|
||||
metricItem.setBizName(
|
||||
metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getBizName());
|
||||
metricItem.setCreatedBy(
|
||||
metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getCreatedBy());
|
||||
metricItem.getRelateItems().forEach(dimensionItem -> {
|
||||
dimensionItem.setName(dimensionMap
|
||||
.getOrDefault(dimensionItem.getId(), new DimensionResp()).getName());
|
||||
dimensionItem.setBizName(dimensionMap
|
||||
.getOrDefault(dimensionItem.getId(), new DimensionResp()).getBizName());
|
||||
dimensionItem.setCreatedBy(dimensionMap
|
||||
.getOrDefault(dimensionItem.getId(), new DimensionResp()).getCreatedBy());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private String getUniqueId() {
|
||||
|
||||
@@ -27,11 +27,14 @@ import java.util.List;
|
||||
public class CanvasServiceImpl extends ServiceImpl<CanvasDOMapper, CanvasDO>
|
||||
implements CanvasService {
|
||||
|
||||
@Autowired private ModelService modelService;
|
||||
@Autowired
|
||||
private ModelService modelService;
|
||||
|
||||
@Autowired private DimensionService dimensionService;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
|
||||
@Autowired private MetricService metricService;
|
||||
@Autowired
|
||||
private MetricService metricService;
|
||||
|
||||
@Override
|
||||
public List<CanvasDO> getCanvasList(Long domainId) {
|
||||
|
||||
@@ -19,7 +19,8 @@ import java.util.List;
|
||||
public class CollectServiceImpl implements CollectService {
|
||||
|
||||
public static final String type = "metric";
|
||||
@Resource private CollectMapper collectMapper;
|
||||
@Resource
|
||||
private CollectMapper collectMapper;
|
||||
|
||||
@Override
|
||||
public Boolean collect(User user, CollectDO collectReq) {
|
||||
|
||||
@@ -58,13 +58,20 @@ import java.util.stream.Collectors;
|
||||
public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
||||
implements DataSetService {
|
||||
|
||||
@Autowired private DomainService domainService;
|
||||
@Autowired
|
||||
private DomainService domainService;
|
||||
|
||||
@Lazy @Autowired private DimensionService dimensionService;
|
||||
@Lazy
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
|
||||
@Lazy @Autowired private MetricService metricService;
|
||||
@Lazy
|
||||
@Autowired
|
||||
private MetricService metricService;
|
||||
|
||||
@Lazy @Autowired private TagMetaService tagMetaService;
|
||||
@Lazy
|
||||
@Autowired
|
||||
private TagMetaService tagMetaService;
|
||||
|
||||
@Override
|
||||
public DataSetResp save(DataSetReq dataSetReq, User user) {
|
||||
@@ -152,24 +159,21 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
||||
List<DataSetResp> dataSetFilterByAuth = getDataSetFilterByAuth(dataSetResps, user);
|
||||
dataSetRespSet.addAll(dataSetFilterByAuth);
|
||||
if (domainId != null && domainId > 0) {
|
||||
dataSetRespSet =
|
||||
dataSetRespSet.stream()
|
||||
.filter(modelResp -> modelResp.getDomainId().equals(domainId))
|
||||
.collect(Collectors.toSet());
|
||||
dataSetRespSet = dataSetRespSet.stream()
|
||||
.filter(modelResp -> modelResp.getDomainId().equals(domainId))
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
return dataSetRespSet.stream()
|
||||
.sorted(Comparator.comparingLong(DataSetResp::getId))
|
||||
return dataSetRespSet.stream().sorted(Comparator.comparingLong(DataSetResp::getId))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<DataSetResp> getDataSetFilterByAuth(List<DataSetResp> dataSetResps, User user) {
|
||||
return dataSetResps.stream()
|
||||
.filter(dataSetResp -> checkAdminPermission(user, dataSetResp))
|
||||
return dataSetResps.stream().filter(dataSetResp -> checkAdminPermission(user, dataSetResp))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<DataSetResp> getDataSetFilterByDomainAuth(
|
||||
List<DataSetResp> dataSetResps, User user) {
|
||||
private List<DataSetResp> getDataSetFilterByDomainAuth(List<DataSetResp> dataSetResps,
|
||||
User user) {
|
||||
Set<DomainResp> domainResps = domainService.getDomainAuthSet(user, AuthType.ADMIN);
|
||||
if (CollectionUtils.isEmpty(domainResps)) {
|
||||
return Lists.newArrayList();
|
||||
@@ -190,14 +194,10 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
||||
dataSetResp.setQueryConfig(
|
||||
JSONObject.parseObject(dataSetDO.getQueryConfig(), QueryConfig.class));
|
||||
}
|
||||
dataSetResp.setAdmins(
|
||||
StringUtils.isBlank(dataSetDO.getAdmin())
|
||||
? Lists.newArrayList()
|
||||
: Arrays.asList(dataSetDO.getAdmin().split(",")));
|
||||
dataSetResp.setAdminOrgs(
|
||||
StringUtils.isBlank(dataSetDO.getAdminOrg())
|
||||
? Lists.newArrayList()
|
||||
: Arrays.asList(dataSetDO.getAdminOrg().split(",")));
|
||||
dataSetResp.setAdmins(StringUtils.isBlank(dataSetDO.getAdmin()) ? Lists.newArrayList()
|
||||
: Arrays.asList(dataSetDO.getAdmin().split(",")));
|
||||
dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) ? Lists.newArrayList()
|
||||
: Arrays.asList(dataSetDO.getAdminOrg().split(",")));
|
||||
dataSetResp.setTypeEnum(TypeEnums.DATASET);
|
||||
List<TagItem> dimensionItems =
|
||||
tagMetaService.getTagItems(dataSetResp.dimensionIds(), TagDefineType.DIMENSION);
|
||||
@@ -246,14 +246,10 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
||||
metaFilter.setIds(dataSetIds);
|
||||
List<DataSetResp> dataSetList = getDataSetList(metaFilter);
|
||||
return dataSetList.stream()
|
||||
.flatMap(
|
||||
dataSetResp ->
|
||||
dataSetResp.getAllModels().stream()
|
||||
.map(modelId -> Pair.of(modelId, dataSetResp.getId())))
|
||||
.collect(
|
||||
Collectors.groupingBy(
|
||||
Pair::getLeft,
|
||||
Collectors.mapping(Pair::getRight, Collectors.toList())));
|
||||
.flatMap(dataSetResp -> dataSetResp.getAllModels().stream()
|
||||
.map(modelId -> Pair.of(modelId, dataSetResp.getId())))
|
||||
.collect(Collectors.groupingBy(Pair::getLeft,
|
||||
Collectors.mapping(Pair::getRight, Collectors.toList())));
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -285,14 +281,9 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
|
||||
}
|
||||
|
||||
private <T, R> List<String> findDuplicates(List<T> list, Function<T, R> keyExtractor) {
|
||||
return list.stream()
|
||||
.collect(Collectors.groupingBy(keyExtractor, Collectors.counting()))
|
||||
.entrySet()
|
||||
.stream()
|
||||
.filter(entry -> entry.getValue() > 1)
|
||||
.map(Map.Entry::getKey)
|
||||
.map(Object::toString)
|
||||
.collect(Collectors.toList());
|
||||
return list.stream().collect(Collectors.groupingBy(keyExtractor, Collectors.counting()))
|
||||
.entrySet().stream().filter(entry -> entry.getValue() > 1).map(Map.Entry::getKey)
|
||||
.map(Object::toString).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public Long getDataSetIdFromSql(String sql, User user) {
|
||||
|
||||
@@ -45,9 +45,12 @@ import java.util.stream.Collectors;
|
||||
public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseDO>
|
||||
implements DatabaseService {
|
||||
|
||||
@Autowired private SqlUtils sqlUtils;
|
||||
@Autowired
|
||||
private SqlUtils sqlUtils;
|
||||
|
||||
@Lazy @Autowired private ModelService datasourceService;
|
||||
@Lazy
|
||||
@Autowired
|
||||
private ModelService datasourceService;
|
||||
|
||||
@Override
|
||||
public boolean testConnect(DatabaseReq databaseReq, User user) {
|
||||
@@ -84,19 +87,18 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
|
||||
}
|
||||
|
||||
private void fillPermission(List<DatabaseResp> databaseResps, User user) {
|
||||
databaseResps.forEach(
|
||||
databaseResp -> {
|
||||
if (databaseResp.getAdmins().contains(user.getName())
|
||||
|| user.getName().equalsIgnoreCase(databaseResp.getCreatedBy())
|
||||
|| user.isSuperAdmin()) {
|
||||
databaseResp.setHasPermission(true);
|
||||
databaseResp.setHasEditPermission(true);
|
||||
databaseResp.setHasUsePermission(true);
|
||||
}
|
||||
if (databaseResp.getViewers().contains(user.getName())) {
|
||||
databaseResp.setHasUsePermission(true);
|
||||
}
|
||||
});
|
||||
databaseResps.forEach(databaseResp -> {
|
||||
if (databaseResp.getAdmins().contains(user.getName())
|
||||
|| user.getName().equalsIgnoreCase(databaseResp.getCreatedBy())
|
||||
|| user.isSuperAdmin()) {
|
||||
databaseResp.setHasPermission(true);
|
||||
databaseResp.setHasEditPermission(true);
|
||||
databaseResp.setHasUsePermission(true);
|
||||
}
|
||||
if (databaseResp.getViewers().contains(user.getName())) {
|
||||
databaseResp.setHasUsePermission(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -135,9 +137,8 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
|
||||
}
|
||||
checkPermission(databaseResp, user);
|
||||
String sql = sqlExecuteReq.getSql();
|
||||
sql =
|
||||
SqlVariableParseUtils.parse(
|
||||
sql, sqlExecuteReq.getSqlVariables(), Lists.newArrayList());
|
||||
sql = SqlVariableParseUtils.parse(sql, sqlExecuteReq.getSqlVariables(),
|
||||
Lists.newArrayList());
|
||||
return executeSql(sql, databaseResp);
|
||||
}
|
||||
|
||||
@@ -161,10 +162,8 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
|
||||
}
|
||||
// Add default parameters for unknown databases
|
||||
if (!CollectionUtils.isEmpty(databaseList)) {
|
||||
List<String> databaseTypeList =
|
||||
databaseList.stream()
|
||||
.map(databaseResp -> databaseResp.getType())
|
||||
.collect(Collectors.toList());
|
||||
List<String> databaseTypeList = databaseList.stream()
|
||||
.map(databaseResp -> databaseResp.getType()).collect(Collectors.toList());
|
||||
DefaultParametersBuilder defaultParametersBuilder = new DefaultParametersBuilder();
|
||||
for (String dbType : databaseTypeList) {
|
||||
if (!parametersBuilderMap.containsKey(dbType)) {
|
||||
@@ -174,8 +173,7 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
|
||||
}
|
||||
// Add the OTHER type at the end
|
||||
if (parametersBuilderMap.containsKey(EngineType.OTHER.getName())) {
|
||||
result.put(
|
||||
EngineType.OTHER.getName(),
|
||||
result.put(EngineType.OTHER.getName(),
|
||||
parametersBuilderMap.get(EngineType.OTHER.getName()).build());
|
||||
}
|
||||
return result;
|
||||
@@ -240,14 +238,11 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
|
||||
private void checkPermission(DatabaseResp databaseResp, User user) {
|
||||
List<String> admins = databaseResp.getAdmins();
|
||||
List<String> viewers = databaseResp.getViewers();
|
||||
if (!admins.contains(user.getName())
|
||||
&& !viewers.contains(user.getName())
|
||||
if (!admins.contains(user.getName()) && !viewers.contains(user.getName())
|
||||
&& !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName())
|
||||
&& !user.isSuperAdmin()) {
|
||||
String message =
|
||||
String.format(
|
||||
"您暂无当前数据库%s权限, 请联系数据库创建人:%s开通",
|
||||
databaseResp.getName(), databaseResp.getCreatedBy());
|
||||
String message = String.format("您暂无当前数据库%s权限, 请联系数据库创建人:%s开通", databaseResp.getName(),
|
||||
databaseResp.getCreatedBy());
|
||||
throw new RuntimeException(message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,12 +46,8 @@ public class DictTaskServiceImpl implements DictTaskService {
|
||||
private final FileHandler fileHandler;
|
||||
private final DictWordService dictWordService;
|
||||
|
||||
public DictTaskServiceImpl(
|
||||
DictRepository dictRepository,
|
||||
DictUtils dictConverter,
|
||||
DictUtils dictUtils,
|
||||
FileHandler fileHandler,
|
||||
DictWordService dictWordService) {
|
||||
public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter,
|
||||
DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService) {
|
||||
this.dictRepository = dictRepository;
|
||||
this.dictConverter = dictConverter;
|
||||
this.dictUtils = dictUtils;
|
||||
@@ -80,11 +76,8 @@ public class DictTaskServiceImpl implements DictTaskService {
|
||||
}
|
||||
|
||||
private DictItemResp fetchDictItemResp(DictSingleTaskReq taskReq) {
|
||||
DictItemFilter dictItemFilter =
|
||||
DictItemFilter.builder()
|
||||
.itemId(taskReq.getItemId())
|
||||
.type(taskReq.getType())
|
||||
.build();
|
||||
DictItemFilter dictItemFilter = DictItemFilter.builder().itemId(taskReq.getItemId())
|
||||
.type(taskReq.getType()).build();
|
||||
List<DictItemResp> dictItemRespList = dictRepository.queryDictConf(dictItemFilter);
|
||||
if (!CollectionUtils.isEmpty(dictItemRespList)) {
|
||||
return dictItemRespList.get(0);
|
||||
@@ -159,14 +152,9 @@ public class DictTaskServiceImpl implements DictTaskService {
|
||||
|
||||
@Override
|
||||
public PageInfo<DictValueResp> queryDictValue(DictValueReq dictValueReq, User user) {
|
||||
String fileName =
|
||||
String.format(
|
||||
"dic_value_%d_%s_%s",
|
||||
dictValueReq.getModelId(),
|
||||
dictValueReq.getType().name(),
|
||||
dictValueReq.getItemId())
|
||||
+ Constants.DOT
|
||||
+ dictFileType;
|
||||
String fileName = String.format("dic_value_%d_%s_%s", dictValueReq.getModelId(),
|
||||
dictValueReq.getType().name(), dictValueReq.getItemId()) + Constants.DOT
|
||||
+ dictFileType;
|
||||
PageInfo<DictValueResp> dictValueRespList =
|
||||
fileHandler.queryDictValue(fileName, dictValueReq);
|
||||
return dictValueRespList;
|
||||
@@ -174,14 +162,9 @@ public class DictTaskServiceImpl implements DictTaskService {
|
||||
|
||||
@Override
|
||||
public String queryDictFilePath(DictValueReq dictValueReq, User user) {
|
||||
String fileName =
|
||||
String.format(
|
||||
"dic_value_%d_%s_%s",
|
||||
dictValueReq.getModelId(),
|
||||
dictValueReq.getType().name(),
|
||||
dictValueReq.getItemId())
|
||||
+ Constants.DOT
|
||||
+ dictFileType;
|
||||
String fileName = String.format("dic_value_%d_%s_%s", dictValueReq.getModelId(),
|
||||
dictValueReq.getType().name(), dictValueReq.getItemId()) + Constants.DOT
|
||||
+ dictFileType;
|
||||
return fileHandler.queryDictFilePath(fileName);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,8 +22,10 @@ import java.util.stream.Collectors;
|
||||
@Slf4j
|
||||
public class DictWordService {
|
||||
|
||||
@Autowired private SchemaService schemaService;
|
||||
@Autowired private KnowledgeBaseService knowledgeBaseService;
|
||||
@Autowired
|
||||
private SchemaService schemaService;
|
||||
@Autowired
|
||||
private KnowledgeBaseService knowledgeBaseService;
|
||||
|
||||
private List<DictWord> preDictWords = new ArrayList<>();
|
||||
|
||||
@@ -37,8 +39,8 @@ public class DictWordService {
|
||||
long startTime = System.currentTimeMillis();
|
||||
List<DictWord> dictWords = getAllDictWords();
|
||||
List<DictWord> preDictWords = getPreDictWords();
|
||||
if (org.apache.commons.collections.CollectionUtils.isEqualCollection(
|
||||
dictWords, preDictWords)) {
|
||||
if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords,
|
||||
preDictWords)) {
|
||||
log.debug("Dictionary hasn't been reloaded.");
|
||||
return;
|
||||
}
|
||||
@@ -61,8 +63,8 @@ public class DictWordService {
|
||||
return words;
|
||||
}
|
||||
|
||||
private void addWordsByType(
|
||||
DictWordType value, List<SchemaElement> metas, List<DictWord> natures) {
|
||||
private void addWordsByType(DictWordType value, List<SchemaElement> metas,
|
||||
List<DictWord> natures) {
|
||||
metas = distinct(metas);
|
||||
List<DictWord> natureList = WordBuilderFactory.get(value).getDictWords(metas);
|
||||
log.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
|
||||
@@ -87,8 +89,6 @@ public class DictWordService {
|
||||
return metas.stream()
|
||||
.collect(
|
||||
Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1))
|
||||
.values()
|
||||
.stream()
|
||||
.collect(Collectors.toList());
|
||||
.values().stream().collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,15 +74,12 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
|
||||
private TagMetaService tagMetaService;
|
||||
|
||||
@Autowired private ApplicationEventPublisher eventPublisher;
|
||||
@Autowired
|
||||
private ApplicationEventPublisher eventPublisher;
|
||||
|
||||
public DimensionServiceImpl(
|
||||
DimensionRepository dimensionRepository,
|
||||
ModelService modelService,
|
||||
AliasGenerateHelper aliasGenerateHelper,
|
||||
DatabaseService databaseService,
|
||||
ModelRelaService modelRelaService,
|
||||
DataSetService dataSetService,
|
||||
public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService,
|
||||
AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService,
|
||||
ModelRelaService modelRelaService, DataSetService dataSetService,
|
||||
TagMetaService tagMetaService) {
|
||||
this.modelService = modelService;
|
||||
this.dimensionRepository = dimensionRepository;
|
||||
@@ -110,48 +107,36 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
}
|
||||
Long modelId = dimensionReqs.get(0).getModelId();
|
||||
List<DimensionResp> dimensionResps = getDimensions(modelId);
|
||||
Map<String, DimensionResp> bizNameMap =
|
||||
dimensionResps.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, DimensionResp> nameMap =
|
||||
dimensionResps.stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, DimensionResp> nameMap = dimensionResps.stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
|
||||
|
||||
List<DimensionReq> dimensionToInsert = Lists.newArrayList();
|
||||
dimensionReqs.stream()
|
||||
.forEach(
|
||||
dimension -> {
|
||||
if (!bizNameMap.containsKey(dimension.getBizName())
|
||||
&& !nameMap.containsKey(dimension.getName())) {
|
||||
dimensionToInsert.add(dimension);
|
||||
} else {
|
||||
DimensionResp dimensionRespByBizName =
|
||||
bizNameMap.get(dimension.getBizName());
|
||||
DimensionResp dimensionRespByName =
|
||||
nameMap.get(dimension.getName());
|
||||
if (null != dimensionRespByBizName
|
||||
&& isChange(dimension, dimensionRespByBizName)) {
|
||||
dimension.setId(dimensionRespByBizName.getId());
|
||||
this.updateDimension(dimension, user);
|
||||
} else {
|
||||
if (null != dimensionRespByName
|
||||
&& isChange(dimension, dimensionRespByName)) {
|
||||
dimension.setId(dimensionRespByName.getId());
|
||||
this.updateDimension(dimension, user);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
dimensionReqs.stream().forEach(dimension -> {
|
||||
if (!bizNameMap.containsKey(dimension.getBizName())
|
||||
&& !nameMap.containsKey(dimension.getName())) {
|
||||
dimensionToInsert.add(dimension);
|
||||
} else {
|
||||
DimensionResp dimensionRespByBizName = bizNameMap.get(dimension.getBizName());
|
||||
DimensionResp dimensionRespByName = nameMap.get(dimension.getName());
|
||||
if (null != dimensionRespByBizName && isChange(dimension, dimensionRespByBizName)) {
|
||||
dimension.setId(dimensionRespByBizName.getId());
|
||||
this.updateDimension(dimension, user);
|
||||
} else {
|
||||
if (null != dimensionRespByName && isChange(dimension, dimensionRespByName)) {
|
||||
dimension.setId(dimensionRespByName.getId());
|
||||
this.updateDimension(dimension, user);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
if (CollectionUtils.isEmpty(dimensionToInsert)) {
|
||||
return;
|
||||
}
|
||||
List<DimensionDO> dimensionDOS =
|
||||
dimensionToInsert.stream()
|
||||
.peek(dimension -> dimension.createdBy(user.getName()))
|
||||
.map(DimensionConverter::convert2DimensionDO)
|
||||
.collect(Collectors.toList());
|
||||
dimensionToInsert.stream().peek(dimension -> dimension.createdBy(user.getName()))
|
||||
.map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
|
||||
dimensionRepository.createDimensionBatch(dimensionDOS);
|
||||
sendEventBatch(dimensionDOS, EventType.ADD);
|
||||
}
|
||||
@@ -166,13 +151,9 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
dimensionRepository.updateDimension(dimensionDO);
|
||||
if (!oldName.equals(dimensionDO.getName())) {
|
||||
sendEvent(
|
||||
DataItem.builder()
|
||||
.modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
|
||||
.newName(dimensionReq.getName())
|
||||
.name(oldName)
|
||||
.type(TypeEnums.DIMENSION)
|
||||
.id(dimensionDO.getId() + Constants.UNDERLINE)
|
||||
.build(),
|
||||
DataItem.builder().modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
|
||||
.newName(dimensionReq.getName()).name(oldName).type(TypeEnums.DIMENSION)
|
||||
.id(dimensionDO.getId() + Constants.UNDERLINE).build(),
|
||||
EventType.UPDATE);
|
||||
}
|
||||
}
|
||||
@@ -188,15 +169,11 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
if (CollectionUtils.isEmpty(dimensionDOS)) {
|
||||
return;
|
||||
}
|
||||
dimensionDOS =
|
||||
dimensionDOS.stream()
|
||||
.peek(
|
||||
dimensionDO -> {
|
||||
dimensionDO.setStatus(metaBatchReq.getStatus());
|
||||
dimensionDO.setUpdatedAt(new Date());
|
||||
dimensionDO.setUpdatedBy(user.getName());
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
dimensionDOS = dimensionDOS.stream().peek(dimensionDO -> {
|
||||
dimensionDO.setStatus(metaBatchReq.getStatus());
|
||||
dimensionDO.setUpdatedAt(new Date());
|
||||
dimensionDO.setUpdatedBy(user.getName());
|
||||
}).collect(Collectors.toList());
|
||||
dimensionRepository.batchUpdateStatus(dimensionDOS);
|
||||
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
||||
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
||||
@@ -303,8 +280,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
return getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
|
||||
}
|
||||
|
||||
private List<DimensionResp> filterByField(
|
||||
List<DimensionResp> dimensionResps, List<String> fields) {
|
||||
private List<DimensionResp> filterByField(List<DimensionResp> dimensionResps,
|
||||
List<String> fields) {
|
||||
List<DimensionResp> dimensionFiltered = Lists.newArrayList();
|
||||
for (DimensionResp dimensionResp : dimensionResps) {
|
||||
for (String field : fields) {
|
||||
@@ -338,26 +315,17 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
|
||||
List<DimensionResp> dimensionResps = Lists.newArrayList();
|
||||
if (!CollectionUtils.isEmpty(dimensionDOS)) {
|
||||
dimensionResps =
|
||||
dimensionDOS.stream()
|
||||
.map(
|
||||
dimensionDO ->
|
||||
DimensionConverter.convert2DimensionResp(
|
||||
dimensionDO, modelMap))
|
||||
.collect(Collectors.toList());
|
||||
dimensionResps = dimensionDOS.stream().map(
|
||||
dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelMap))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
return dimensionResps;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
|
||||
String mockAlias =
|
||||
aliasGenerateHelper.generateAlias(
|
||||
mockType,
|
||||
dimensionReq.getName(),
|
||||
dimensionReq.getBizName(),
|
||||
"",
|
||||
dimensionReq.getDescription());
|
||||
String mockAlias = aliasGenerateHelper.generateAlias(mockType, dimensionReq.getName(),
|
||||
dimensionReq.getBizName(), "", dimensionReq.getDescription());
|
||||
String ret = aliasGenerateHelper.extractJsonStringFromAiMessage(mockAlias);
|
||||
return JSONObject.parseObject(ret, new TypeReference<List<String>>() {});
|
||||
}
|
||||
@@ -373,13 +341,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
}
|
||||
DatabaseResp database = databaseService.getDatabase(modelResp.getDatabaseId());
|
||||
|
||||
String sql =
|
||||
"select ai_talk."
|
||||
+ dimensionReq.getBizName()
|
||||
+ " from ("
|
||||
+ sqlQuery
|
||||
+ ") as ai_talk group by ai_talk."
|
||||
+ dimensionReq.getBizName();
|
||||
String sql = "select ai_talk." + dimensionReq.getBizName() + " from (" + sqlQuery
|
||||
+ ") as ai_talk group by ai_talk." + dimensionReq.getBizName();
|
||||
SemanticQueryResp semanticQueryResp = databaseService.executeSql(sql, database);
|
||||
List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
|
||||
List<String> valueList = new ArrayList<>();
|
||||
@@ -403,11 +366,9 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
dimValueMap.setBizName("");
|
||||
}
|
||||
try {
|
||||
dimValueMap.setAlias(
|
||||
jsonObject
|
||||
.getJSONObject("alias")
|
||||
.getJSONArray(stringObjectMap.get(dimensionReq.getBizName()) + "")
|
||||
.toJavaList(String.class));
|
||||
dimValueMap.setAlias(jsonObject.getJSONObject("alias")
|
||||
.getJSONArray(stringObjectMap.get(dimensionReq.getBizName()) + "")
|
||||
.toJavaList(String.class));
|
||||
} catch (Exception exception) {
|
||||
dimValueMap.setAlias(null);
|
||||
}
|
||||
@@ -420,39 +381,29 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
private void checkExist(List<DimensionReq> dimensionReqs) {
|
||||
Long modelId = dimensionReqs.get(0).getModelId();
|
||||
List<DimensionResp> dimensionResps = getDimensions(modelId);
|
||||
Map<String, DimensionResp> bizNameMap =
|
||||
dimensionResps.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, DimensionResp> nameMap =
|
||||
dimensionResps.stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, DimensionResp> nameMap = dimensionResps.stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
|
||||
for (DimensionReq dimensionReq : dimensionReqs) {
|
||||
String forbiddenCharacters =
|
||||
NameCheckUtils.findForbiddenCharacters(dimensionReq.getName());
|
||||
if (StringUtils.isNotBlank(forbiddenCharacters)) {
|
||||
throw new InvalidArgumentException(
|
||||
String.format(
|
||||
"名称包含特殊字符, 请修改: %s,特殊字符: %s",
|
||||
dimensionReq.getName(), forbiddenCharacters));
|
||||
throw new InvalidArgumentException(String.format("名称包含特殊字符, 请修改: %s,特殊字符: %s",
|
||||
dimensionReq.getName(), forbiddenCharacters));
|
||||
}
|
||||
if (bizNameMap.containsKey(dimensionReq.getBizName())) {
|
||||
DimensionResp dimensionResp = bizNameMap.get(dimensionReq.getBizName());
|
||||
if (!dimensionResp.getId().equals(dimensionReq.getId())) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"该主题域下存在相同的维度字段名:%s 创建人:%s",
|
||||
dimensionReq.getBizName(), dimensionResp.getCreatedBy()));
|
||||
throw new RuntimeException(String.format("该主题域下存在相同的维度字段名:%s 创建人:%s",
|
||||
dimensionReq.getBizName(), dimensionResp.getCreatedBy()));
|
||||
}
|
||||
}
|
||||
if (nameMap.containsKey(dimensionReq.getName())) {
|
||||
DimensionResp dimensionResp = nameMap.get(dimensionReq.getName());
|
||||
if (!dimensionResp.getId().equals(dimensionReq.getId())) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"该主题域下存在相同的维度名:%s 创建人:%s",
|
||||
dimensionReq.getName(), dimensionResp.getCreatedBy()));
|
||||
throw new RuntimeException(String.format("该主题域下存在相同的维度名:%s 创建人:%s",
|
||||
dimensionReq.getName(), dimensionResp.getCreatedBy()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -478,19 +429,12 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
}
|
||||
|
||||
private DataEvent getDataEvent(List<DimensionDO> dimensionDOS, EventType eventType) {
|
||||
List<DataItem> dataItems =
|
||||
dimensionDOS.stream()
|
||||
.map(
|
||||
dimensionDO ->
|
||||
DataItem.builder()
|
||||
.id(dimensionDO.getId() + Constants.UNDERLINE)
|
||||
.name(dimensionDO.getName())
|
||||
.modelId(
|
||||
dimensionDO.getModelId()
|
||||
+ Constants.UNDERLINE)
|
||||
.type(TypeEnums.DIMENSION)
|
||||
.build())
|
||||
.collect(Collectors.toList());
|
||||
List<DataItem> dataItems = dimensionDOS.stream()
|
||||
.map(dimensionDO -> DataItem.builder().id(dimensionDO.getId() + Constants.UNDERLINE)
|
||||
.name(dimensionDO.getName())
|
||||
.modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
|
||||
.type(TypeEnums.DIMENSION).build())
|
||||
.collect(Collectors.toList());
|
||||
return new DataEvent(this, dataItems, eventType);
|
||||
}
|
||||
|
||||
@@ -499,10 +443,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
||||
}
|
||||
|
||||
private boolean isChange(DimensionReq dimensionReq, DimensionResp dimensionResp) {
|
||||
boolean isExtChange =
|
||||
!new EqualsBuilder()
|
||||
.append(dimensionReq.getExt(), dimensionResp.getExt())
|
||||
.isEquals();
|
||||
boolean isExtChange = !new EqualsBuilder()
|
||||
.append(dimensionReq.getExt(), dimensionResp.getExt()).isEquals();
|
||||
boolean isTypeParamChange =
|
||||
!Objects.equals(dimensionReq.getTypeParams(), dimensionResp.getTypeParams());
|
||||
return isExtChange || isTypeParamChange;
|
||||
|
||||
@@ -41,9 +41,7 @@ public class DomainServiceImpl implements DomainService {
|
||||
private final ModelService modelService;
|
||||
private final UserService userService;
|
||||
|
||||
public DomainServiceImpl(
|
||||
DomainRepository domainRepository,
|
||||
@Lazy ModelService modelService,
|
||||
public DomainServiceImpl(DomainRepository domainRepository, @Lazy ModelService modelService,
|
||||
UserService userService) {
|
||||
this.domainRepository = domainRepository;
|
||||
this.modelService = modelService;
|
||||
@@ -89,8 +87,7 @@ public class DomainServiceImpl implements DomainService {
|
||||
|
||||
@Override
|
||||
public List<DomainResp> getDomainList(List<Long> domainIds) {
|
||||
return getDomainList().stream()
|
||||
.filter(domainDO -> domainIds.contains(domainDO.getId()))
|
||||
return getDomainList().stream().filter(domainDO -> domainIds.contains(domainDO.getId()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@@ -113,10 +110,8 @@ public class DomainServiceImpl implements DomainService {
|
||||
domainResp.setHasModel(true);
|
||||
}
|
||||
}
|
||||
return new ArrayList<>(domainWithAuthAll)
|
||||
.stream()
|
||||
.sorted(Comparator.comparingLong(DomainResp::getId))
|
||||
.collect(Collectors.toList());
|
||||
return new ArrayList<>(domainWithAuthAll).stream()
|
||||
.sorted(Comparator.comparingLong(DomainResp::getId)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -125,19 +120,16 @@ public class DomainServiceImpl implements DomainService {
|
||||
Set<String> orgIds = userService.getUserAllOrgId(user.getName());
|
||||
Set<DomainResp> domainWithAuth = Sets.newHashSet();
|
||||
if (authTypeEnum.equals(AuthType.ADMIN)) {
|
||||
domainWithAuth =
|
||||
domainResps.stream()
|
||||
.filter(domainResp -> checkAdminPermission(orgIds, user, domainResp))
|
||||
.collect(Collectors.toSet());
|
||||
return domainWithAuth.stream()
|
||||
.peek(domainResp -> domainResp.setHasEditPermission(true))
|
||||
domainWithAuth = domainResps.stream()
|
||||
.filter(domainResp -> checkAdminPermission(orgIds, user, domainResp))
|
||||
.collect(Collectors.toSet());
|
||||
return domainWithAuth.stream().peek(domainResp -> domainResp.setHasEditPermission(true))
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
if (authTypeEnum.equals(AuthType.VISIBLE)) {
|
||||
domainWithAuth =
|
||||
domainResps.stream()
|
||||
.filter(domainResp -> checkViewPermission(orgIds, user, domainResp))
|
||||
.collect(Collectors.toSet());
|
||||
domainWithAuth = domainResps.stream()
|
||||
.filter(domainResp -> checkViewPermission(orgIds, user, domainResp))
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
return domainWithAuth;
|
||||
@@ -217,9 +209,8 @@ public class DomainServiceImpl implements DomainService {
|
||||
public Map<Long, String> getDomainFullPathMap() {
|
||||
Map<Long, String> domainFullPathMap = new HashMap<>();
|
||||
List<DomainDO> domainDOList = domainRepository.getDomainList();
|
||||
Map<Long, DomainDO> domainDOMap =
|
||||
domainDOList.stream()
|
||||
.collect(Collectors.toMap(DomainDO::getId, a -> a, (k1, k2) -> k1));
|
||||
Map<Long, DomainDO> domainDOMap = domainDOList.stream()
|
||||
.collect(Collectors.toMap(DomainDO::getId, a -> a, (k1, k2) -> k1));
|
||||
for (DomainDO domainDO : domainDOList) {
|
||||
final Long domainId = domainDO.getId();
|
||||
StringBuilder fullPath = new StringBuilder(domainDO.getBizName() + "/");
|
||||
|
||||
@@ -69,9 +69,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
|
||||
private SemanticLayerService queryService;
|
||||
|
||||
public DownloadServiceImpl(
|
||||
MetricService metricService,
|
||||
DimensionService dimensionService,
|
||||
public DownloadServiceImpl(MetricService metricService, DimensionService dimensionService,
|
||||
SemanticLayerService queryService) {
|
||||
this.metricService = metricService;
|
||||
this.dimensionService = dimensionService;
|
||||
@@ -79,9 +77,8 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void downloadByStruct(
|
||||
DownloadMetricReq downloadMetricReq, User user, HttpServletResponse response)
|
||||
throws Exception {
|
||||
public void downloadByStruct(DownloadMetricReq downloadMetricReq, User user,
|
||||
HttpServletResponse response) throws Exception {
|
||||
String fileName =
|
||||
String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), dateFormat));
|
||||
File file = FileUtils.createTmpFile(fileName);
|
||||
@@ -91,14 +88,10 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
queryService.queryByReq(queryStructReq.convert(true), user);
|
||||
DataDownload dataDownload =
|
||||
buildDataDownload(queryResult, queryStructReq, downloadMetricReq.isTransform());
|
||||
EasyExcel.write(file)
|
||||
.sheet("Sheet1")
|
||||
.head(dataDownload.getHeaders())
|
||||
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders())
|
||||
.doWrite(dataDownload.getData());
|
||||
} catch (RuntimeException e) {
|
||||
EasyExcel.write(file)
|
||||
.sheet("Sheet1")
|
||||
.head(buildErrMessageHead())
|
||||
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
|
||||
.doWrite(buildErrMessageData(e.getMessage()));
|
||||
return;
|
||||
}
|
||||
@@ -106,9 +99,8 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchDownload(
|
||||
BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response)
|
||||
throws Exception {
|
||||
public void batchDownload(BatchDownloadReq batchDownloadReq, User user,
|
||||
HttpServletResponse response) throws Exception {
|
||||
String fileName =
|
||||
String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), dateFormat));
|
||||
File file = FileUtils.createTmpFile(fileName);
|
||||
@@ -127,16 +119,13 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
metaFilter.setIds(metricIds);
|
||||
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
|
||||
Map<String, List<MetricResp>> metricMap = getMetricMap(metricResps);
|
||||
List<Long> dimensionIds =
|
||||
metricResps.stream()
|
||||
.map(metricResp -> metricService.getDrillDownDimension(metricResp.getId()))
|
||||
.flatMap(Collection::stream)
|
||||
.map(DrillDownDimension::getDimensionId)
|
||||
.collect(Collectors.toList());
|
||||
List<Long> dimensionIds = metricResps.stream()
|
||||
.map(metricResp -> metricService.getDrillDownDimension(metricResp.getId()))
|
||||
.flatMap(Collection::stream).map(DrillDownDimension::getDimensionId)
|
||||
.collect(Collectors.toList());
|
||||
metaFilter.setIds(dimensionIds);
|
||||
Map<Long, DimensionResp> dimensionRespMap =
|
||||
dimensionService.getDimensions(metaFilter).stream()
|
||||
.collect(Collectors.toMap(DimensionResp::getId, d -> d));
|
||||
Map<Long, DimensionResp> dimensionRespMap = dimensionService.getDimensions(metaFilter)
|
||||
.stream().collect(Collectors.toMap(DimensionResp::getId, d -> d));
|
||||
ExcelWriter excelWriter = EasyExcel.write(file).build();
|
||||
int sheetCount = 1;
|
||||
for (List<MetricResp> metrics : metricMap.values()) {
|
||||
@@ -152,18 +141,13 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
QuerySqlReq querySqlReq = queryStructReq.convert();
|
||||
querySqlReq.setNeedAuth(true);
|
||||
SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user);
|
||||
DataDownload dataDownload =
|
||||
buildDataDownload(
|
||||
queryResult, queryStructReq, batchDownloadReq.isTransform());
|
||||
WriteSheet writeSheet =
|
||||
EasyExcel.writerSheet("Sheet" + sheetCount)
|
||||
.head(dataDownload.getHeaders())
|
||||
.build();
|
||||
DataDownload dataDownload = buildDataDownload(queryResult, queryStructReq,
|
||||
batchDownloadReq.isTransform());
|
||||
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
|
||||
.head(dataDownload.getHeaders()).build();
|
||||
excelWriter.write(dataDownload.getData(), writeSheet);
|
||||
} catch (RuntimeException e) {
|
||||
EasyExcel.write(file)
|
||||
.sheet("Sheet1")
|
||||
.head(buildErrMessageHead())
|
||||
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
|
||||
.doWrite(buildErrMessageData(e.getMessage()));
|
||||
return;
|
||||
}
|
||||
@@ -220,11 +204,8 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
return data;
|
||||
}
|
||||
|
||||
private List<List<String>> buildData(
|
||||
List<List<String>> headers,
|
||||
Map<String, String> nameMap,
|
||||
List<Map<String, Object>> dataTransformed,
|
||||
String metricName) {
|
||||
private List<List<String>> buildData(List<List<String>> headers, Map<String, String> nameMap,
|
||||
List<Map<String, Object>> dataTransformed, String metricName) {
|
||||
List<List<String>> data = Lists.newArrayList();
|
||||
for (Map<String, Object> map : dataTransformed) {
|
||||
List<String> row = Lists.newArrayList();
|
||||
@@ -246,27 +227,20 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
return data;
|
||||
}
|
||||
|
||||
private DataDownload buildDataDownload(
|
||||
SemanticQueryResp queryResult, QueryStructReq queryStructReq, boolean isTransform) {
|
||||
private DataDownload buildDataDownload(SemanticQueryResp queryResult,
|
||||
QueryStructReq queryStructReq, boolean isTransform) {
|
||||
List<QueryColumn> metricColumns = queryResult.getMetricColumns();
|
||||
List<QueryColumn> dimensionColumns = queryResult.getDimensionColumns();
|
||||
if (isTransform && !CollectionUtils.isEmpty(metricColumns)) {
|
||||
QueryColumn metric = metricColumns.get(0);
|
||||
List<String> groups = queryStructReq.getGroups();
|
||||
List<Map<String, Object>> dataTransformed =
|
||||
DataTransformUtils.transform(
|
||||
queryResult.getResultList(),
|
||||
metric.getNameEn(),
|
||||
groups,
|
||||
queryStructReq.getDateInfo());
|
||||
DataTransformUtils.transform(queryResult.getResultList(), metric.getNameEn(),
|
||||
groups, queryStructReq.getDateInfo());
|
||||
List<List<String>> headers =
|
||||
buildHeader(dimensionColumns, queryStructReq.getDateInfo().getDateList());
|
||||
List<List<String>> data =
|
||||
buildData(
|
||||
headers,
|
||||
getDimensionNameMap(dimensionColumns),
|
||||
dataTransformed,
|
||||
metric.getName());
|
||||
List<List<String>> data = buildData(headers, getDimensionNameMap(dimensionColumns),
|
||||
dataTransformed, metric.getName());
|
||||
return DataDownload.builder().headers(headers).data(data).build();
|
||||
} else {
|
||||
List<List<String>> data = buildData(queryResult);
|
||||
@@ -275,19 +249,15 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
}
|
||||
}
|
||||
|
||||
private QueryStructReq buildDownloadReq(
|
||||
List<DimensionResp> dimensionResps,
|
||||
MetricResp metricResp,
|
||||
BatchDownloadReq batchDownloadReq) {
|
||||
private QueryStructReq buildDownloadReq(List<DimensionResp> dimensionResps,
|
||||
MetricResp metricResp, BatchDownloadReq batchDownloadReq) {
|
||||
DateConf dateConf = batchDownloadReq.getDateInfo();
|
||||
Set<Long> modelIds =
|
||||
dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
|
||||
modelIds.add(metricResp.getModelId());
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setGroups(
|
||||
dimensionResps.stream()
|
||||
.map(DimensionResp::getBizName)
|
||||
.collect(Collectors.toList()));
|
||||
queryStructReq.setGroups(dimensionResps.stream().map(DimensionResp::getBizName)
|
||||
.collect(Collectors.toList()));
|
||||
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
|
||||
Aggregator aggregator = new Aggregator();
|
||||
aggregator.setColumn(metricResp.getBizName());
|
||||
@@ -325,19 +295,15 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
.collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn));
|
||||
}
|
||||
|
||||
private List<DimensionResp> getMetricRelaDimensions(
|
||||
MetricResp metricResp, Map<Long, DimensionResp> dimensionRespMap) {
|
||||
if (metricResp.getRelateDimension() == null
|
||||
|| CollectionUtils.isEmpty(
|
||||
metricResp.getRelateDimension().getDrillDownDimensions())) {
|
||||
private List<DimensionResp> getMetricRelaDimensions(MetricResp metricResp,
|
||||
Map<Long, DimensionResp> dimensionRespMap) {
|
||||
if (metricResp.getRelateDimension() == null || CollectionUtils
|
||||
.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
return metricResp.getRelateDimension().getDrillDownDimensions().stream()
|
||||
.map(
|
||||
drillDownDimension ->
|
||||
dimensionRespMap.get(drillDownDimension.getDimensionId()))
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toList());
|
||||
return metricResp.getRelateDimension().getDrillDownDimensions().stream().map(
|
||||
drillDownDimension -> dimensionRespMap.get(drillDownDimension.getDimensionId()))
|
||||
.filter(Objects::nonNull).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private void downloadFile(HttpServletResponse response, File file, String filename) {
|
||||
@@ -345,8 +311,7 @@ public class DownloadServiceImpl implements DownloadService {
|
||||
byte[] buffer = readFileToByteArray(file);
|
||||
response.reset();
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.addHeader(
|
||||
"Content-Disposition",
|
||||
response.addHeader("Content-Disposition",
|
||||
"attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
|
||||
response.addHeader("Content-Length", "" + file.length());
|
||||
try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {
|
||||
|
||||
@@ -88,10 +88,8 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
|
||||
private final AuthenticationConfig authenticationConfig;
|
||||
private final UserService userService;
|
||||
|
||||
public FlightServiceImpl(
|
||||
SemanticLayerService queryService,
|
||||
AuthenticationConfig authenticationConfig,
|
||||
UserService userService) {
|
||||
public FlightServiceImpl(SemanticLayerService queryService,
|
||||
AuthenticationConfig authenticationConfig, UserService userService) {
|
||||
this.queryService = queryService;
|
||||
this.authenticationConfig = authenticationConfig;
|
||||
|
||||
@@ -104,14 +102,11 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setExecutorService(
|
||||
ExecutorService executorService, Integer queue, Integer expireMinute) {
|
||||
public void setExecutorService(ExecutorService executorService, Integer queue,
|
||||
Integer expireMinute) {
|
||||
this.executorService = executorService;
|
||||
this.preparedStatementCache =
|
||||
CacheBuilder.newBuilder()
|
||||
.maximumSize(queue)
|
||||
.expireAfterWrite(expireMinute, TimeUnit.MINUTES)
|
||||
.build();
|
||||
this.preparedStatementCache = CacheBuilder.newBuilder().maximumSize(queue)
|
||||
.expireAfterWrite(expireMinute, TimeUnit.MINUTES).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -120,26 +115,20 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getStreamStatement(
|
||||
final TicketStatementQuery ticketStatementQuery,
|
||||
final CallContext context,
|
||||
final ServerStreamListener listener) {
|
||||
public void getStreamStatement(final TicketStatementQuery ticketStatementQuery,
|
||||
final CallContext context, final ServerStreamListener listener) {
|
||||
final ByteString handle = ticketStatementQuery.getStatementHandle();
|
||||
log.info("getStreamStatement {} ", handle);
|
||||
executeQuery(handle, listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FlightInfo getFlightInfoStatement(
|
||||
final CommandStatementQuery request,
|
||||
final CallContext context,
|
||||
final FlightDescriptor descriptor) {
|
||||
public FlightInfo getFlightInfoStatement(final CommandStatementQuery request,
|
||||
final CallContext context, final FlightDescriptor descriptor) {
|
||||
try {
|
||||
ByteString preparedStatementHandle = addPrepared(context, request.getQuery());
|
||||
TicketStatementQuery ticket =
|
||||
TicketStatementQuery.newBuilder()
|
||||
.setStatementHandle(preparedStatementHandle)
|
||||
.build();
|
||||
TicketStatementQuery ticket = TicketStatementQuery.newBuilder()
|
||||
.setStatementHandle(preparedStatementHandle).build();
|
||||
return getFlightInfoForSchema(ticket, descriptor, null);
|
||||
} catch (Exception e) {
|
||||
log.error("getFlightInfoStatement error {}", e);
|
||||
@@ -148,10 +137,8 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getStreamPreparedStatement(
|
||||
final CommandPreparedStatementQuery command,
|
||||
final CallContext context,
|
||||
final ServerStreamListener listener) {
|
||||
public void getStreamPreparedStatement(final CommandPreparedStatementQuery command,
|
||||
final CallContext context, final ServerStreamListener listener) {
|
||||
log.info("getStreamPreparedStatement {}", command.getPreparedStatementHandle());
|
||||
executeQuery(command.getPreparedStatementHandle(), listener);
|
||||
}
|
||||
@@ -160,117 +147,88 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
|
||||
SemanticQueryReq semanticQueryReq = preparedStatementCache.getIfPresent(hander);
|
||||
if (Objects.isNull(semanticQueryReq)) {
|
||||
listener.error(
|
||||
CallStatus.INTERNAL
|
||||
.withDescription("Failed to get prepared statement: empty")
|
||||
CallStatus.INTERNAL.withDescription("Failed to get prepared statement: empty")
|
||||
.toRuntimeException());
|
||||
log.error("getStreamPreparedStatement error {}", hander);
|
||||
listener.completed();
|
||||
return;
|
||||
}
|
||||
executorService.submit(
|
||||
() -> {
|
||||
BufferAllocator rootAllocator = new RootAllocator();
|
||||
try {
|
||||
Optional<Param> authOpt =
|
||||
semanticQueryReq.getParams().stream()
|
||||
.filter(
|
||||
p ->
|
||||
p.getName()
|
||||
.equals(
|
||||
authenticationConfig
|
||||
.getTokenHttpHeaderKey()))
|
||||
.findFirst();
|
||||
if (authOpt.isPresent()) {
|
||||
User user =
|
||||
UserHolder.findUser(
|
||||
authOpt.get().getValue(),
|
||||
authenticationConfig.getTokenHttpHeaderAppKey());
|
||||
SemanticQueryResp resp =
|
||||
queryService.queryByReq(semanticQueryReq, user);
|
||||
ResultSet resultSet =
|
||||
semanticQueryRespToResultSet(
|
||||
resp, semanticQueryReq.getDataSetId());
|
||||
final Schema schema =
|
||||
jdbcToArrowSchema(resultSet.getMetaData(), defaultCalendar);
|
||||
try (final VectorSchemaRoot vectorSchemaRoot =
|
||||
VectorSchemaRoot.create(schema, rootAllocator)) {
|
||||
final VectorLoader loader = new VectorLoader(vectorSchemaRoot);
|
||||
listener.start(vectorSchemaRoot);
|
||||
final ArrowVectorIterator iterator =
|
||||
sqlToArrowVectorIterator(resultSet, rootAllocator);
|
||||
while (iterator.hasNext()) {
|
||||
final VectorSchemaRoot batch = iterator.next();
|
||||
if (batch.getRowCount() == 0) {
|
||||
break;
|
||||
}
|
||||
final VectorUnloader unloader = new VectorUnloader(batch);
|
||||
loader.load(unloader.getRecordBatch());
|
||||
listener.putNext();
|
||||
vectorSchemaRoot.clear();
|
||||
}
|
||||
|
||||
listener.putNext();
|
||||
executorService.submit(() -> {
|
||||
BufferAllocator rootAllocator = new RootAllocator();
|
||||
try {
|
||||
Optional<Param> authOpt = semanticQueryReq.getParams().stream().filter(
|
||||
p -> p.getName().equals(authenticationConfig.getTokenHttpHeaderKey()))
|
||||
.findFirst();
|
||||
if (authOpt.isPresent()) {
|
||||
User user = UserHolder.findUser(authOpt.get().getValue(),
|
||||
authenticationConfig.getTokenHttpHeaderAppKey());
|
||||
SemanticQueryResp resp = queryService.queryByReq(semanticQueryReq, user);
|
||||
ResultSet resultSet =
|
||||
semanticQueryRespToResultSet(resp, semanticQueryReq.getDataSetId());
|
||||
final Schema schema =
|
||||
jdbcToArrowSchema(resultSet.getMetaData(), defaultCalendar);
|
||||
try (final VectorSchemaRoot vectorSchemaRoot =
|
||||
VectorSchemaRoot.create(schema, rootAllocator)) {
|
||||
final VectorLoader loader = new VectorLoader(vectorSchemaRoot);
|
||||
listener.start(vectorSchemaRoot);
|
||||
final ArrowVectorIterator iterator =
|
||||
sqlToArrowVectorIterator(resultSet, rootAllocator);
|
||||
while (iterator.hasNext()) {
|
||||
final VectorSchemaRoot batch = iterator.next();
|
||||
if (batch.getRowCount() == 0) {
|
||||
break;
|
||||
}
|
||||
final VectorUnloader unloader = new VectorUnloader(batch);
|
||||
loader.load(unloader.getRecordBatch());
|
||||
listener.putNext();
|
||||
vectorSchemaRoot.clear();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
listener.error(
|
||||
CallStatus.INTERNAL
|
||||
.withDescription(
|
||||
String.format(
|
||||
"Failed to get exec statement %s",
|
||||
e.getMessage()))
|
||||
.toRuntimeException());
|
||||
log.error("getStreamPreparedStatement error {}", hander);
|
||||
} finally {
|
||||
preparedStatementCache.invalidate(hander);
|
||||
listener.completed();
|
||||
rootAllocator.close();
|
||||
|
||||
listener.putNext();
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (Exception e) {
|
||||
listener.error(CallStatus.INTERNAL
|
||||
.withDescription(
|
||||
String.format("Failed to get exec statement %s", e.getMessage()))
|
||||
.toRuntimeException());
|
||||
log.error("getStreamPreparedStatement error {}", hander);
|
||||
} finally {
|
||||
preparedStatementCache.invalidate(hander);
|
||||
listener.completed();
|
||||
rootAllocator.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closePreparedStatement(
|
||||
final ActionClosePreparedStatementRequest request,
|
||||
final CallContext context,
|
||||
final StreamListener<Result> listener) {
|
||||
public void closePreparedStatement(final ActionClosePreparedStatementRequest request,
|
||||
final CallContext context, final StreamListener<Result> listener) {
|
||||
log.info("closePreparedStatement {}", request.getPreparedStatementHandle());
|
||||
listener.onCompleted();
|
||||
}
|
||||
|
||||
@Override
|
||||
public FlightInfo getFlightInfoPreparedStatement(
|
||||
final CommandPreparedStatementQuery command,
|
||||
final CallContext context,
|
||||
final FlightDescriptor descriptor) {
|
||||
public FlightInfo getFlightInfoPreparedStatement(final CommandPreparedStatementQuery command,
|
||||
final CallContext context, final FlightDescriptor descriptor) {
|
||||
return getFlightInfoForSchema(command, descriptor, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createPreparedStatement(
|
||||
final ActionCreatePreparedStatementRequest request,
|
||||
final CallContext context,
|
||||
final StreamListener<Result> listener) {
|
||||
public void createPreparedStatement(final ActionCreatePreparedStatementRequest request,
|
||||
final CallContext context, final StreamListener<Result> listener) {
|
||||
prepared(request, context, listener);
|
||||
}
|
||||
|
||||
private ByteString addPrepared(final CallContext context, String query) throws Exception {
|
||||
if (Arrays.asList(dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey).stream()
|
||||
.anyMatch(
|
||||
h ->
|
||||
!context.getMiddleware(FlightConstants.HEADER_KEY)
|
||||
.headers()
|
||||
.containsKey(h))) {
|
||||
throw new Exception(
|
||||
String.format(
|
||||
"Failed to create prepared statement: HeaderCallOption miss %s %s %s",
|
||||
dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey));
|
||||
if (Arrays.asList(dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey).stream().anyMatch(
|
||||
h -> !context.getMiddleware(FlightConstants.HEADER_KEY).headers().containsKey(h))) {
|
||||
throw new Exception(String.format(
|
||||
"Failed to create prepared statement: HeaderCallOption miss %s %s %s",
|
||||
dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey));
|
||||
}
|
||||
Long dataSetId =
|
||||
Long.valueOf(
|
||||
context.getMiddleware(FlightConstants.HEADER_KEY)
|
||||
.headers()
|
||||
.get(dataSetIdHeaderKey));
|
||||
Long dataSetId = Long.valueOf(context.getMiddleware(FlightConstants.HEADER_KEY).headers()
|
||||
.get(dataSetIdHeaderKey));
|
||||
if (StringUtils.isBlank(query)) {
|
||||
throw new Exception("Failed to create prepared statement: query is empty");
|
||||
}
|
||||
@@ -287,34 +245,28 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
|
||||
querySqlReq.setParams(
|
||||
Arrays.asList(new Param(authenticationConfig.getTokenHttpHeaderKey(), auth)));
|
||||
preparedStatementCache.put(preparedStatementHandle, querySqlReq);
|
||||
log.info(
|
||||
"createPreparedStatement {} {} {} ", preparedStatementHandle, dataSetId, query);
|
||||
log.info("createPreparedStatement {} {} {} ", preparedStatementHandle, dataSetId,
|
||||
query);
|
||||
return preparedStatementHandle;
|
||||
} catch (Exception e) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private void prepared(
|
||||
final ActionCreatePreparedStatementRequest request,
|
||||
final CallContext context,
|
||||
final StreamListener<Result> listener) {
|
||||
private void prepared(final ActionCreatePreparedStatementRequest request,
|
||||
final CallContext context, final StreamListener<Result> listener) {
|
||||
try {
|
||||
ByteString preparedStatementHandle = addPrepared(context, request.getQuery());
|
||||
final ActionCreatePreparedStatementResult result =
|
||||
ActionCreatePreparedStatementResult.newBuilder()
|
||||
.setDatasetSchema(ByteString.EMPTY)
|
||||
.setParameterSchema(ByteString.empty())
|
||||
.setPreparedStatementHandle(preparedStatementHandle)
|
||||
.build();
|
||||
final ActionCreatePreparedStatementResult result = ActionCreatePreparedStatementResult
|
||||
.newBuilder().setDatasetSchema(ByteString.EMPTY)
|
||||
.setParameterSchema(ByteString.empty())
|
||||
.setPreparedStatementHandle(preparedStatementHandle).build();
|
||||
listener.onNext(new Result(pack(result).toByteArray()));
|
||||
} catch (Exception e) {
|
||||
listener.onError(
|
||||
CallStatus.INTERNAL
|
||||
.withDescription(
|
||||
String.format(
|
||||
"Failed to create prepared statement: %s",
|
||||
e.getMessage()))
|
||||
.withDescription(String.format(
|
||||
"Failed to create prepared statement: %s", e.getMessage()))
|
||||
.toRuntimeException());
|
||||
} finally {
|
||||
listener.onCompleted();
|
||||
@@ -322,13 +274,13 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T extends Message> List<FlightEndpoint> determineEndpoints(
|
||||
T t, FlightDescriptor flightDescriptor, Schema schema) {
|
||||
protected <T extends Message> List<FlightEndpoint> determineEndpoints(T t,
|
||||
FlightDescriptor flightDescriptor, Schema schema) {
|
||||
throw CallStatus.UNIMPLEMENTED.withDescription("Not implemented.").toRuntimeException();
|
||||
}
|
||||
|
||||
private <T extends Message> FlightInfo getFlightInfoForSchema(
|
||||
final T request, final FlightDescriptor descriptor, final Schema schema) {
|
||||
private <T extends Message> FlightInfo getFlightInfoForSchema(final T request,
|
||||
final FlightDescriptor descriptor, final Schema schema) {
|
||||
final Ticket ticket = new Ticket(pack(request).toByteArray());
|
||||
Location listenLocation = Location.forGrpcInsecure(host, port);
|
||||
final List<FlightEndpoint> endpoints =
|
||||
@@ -359,13 +311,9 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
|
||||
for (int i = 1; i <= columnNum; i++) {
|
||||
String columnName = resp.getColumns().get(i - 1).getNameEn();
|
||||
rowSetMetaData.setColumnName(i, columnName);
|
||||
Optional<Map<String, Object>> valOpt =
|
||||
resp.getResultList().stream()
|
||||
.filter(
|
||||
r ->
|
||||
r.containsKey(columnName)
|
||||
&& Objects.nonNull(r.get(columnName)))
|
||||
.findFirst();
|
||||
Optional<Map<String, Object>> valOpt = resp.getResultList().stream()
|
||||
.filter(r -> r.containsKey(columnName) && Objects.nonNull(r.get(columnName)))
|
||||
.findFirst();
|
||||
if (valOpt.isPresent()) {
|
||||
int type = FlightUtils.resolveType(valOpt.get());
|
||||
rowSetMetaData.setColumnType(i, type);
|
||||
|
||||
@@ -110,15 +110,10 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
|
||||
private ChatLayerService chatLayerService;
|
||||
|
||||
public MetricServiceImpl(
|
||||
MetricRepository metricRepository,
|
||||
ModelService modelService,
|
||||
AliasGenerateHelper aliasGenerateHelper,
|
||||
CollectService collectService,
|
||||
DataSetService dataSetService,
|
||||
ApplicationEventPublisher eventPublisher,
|
||||
DimensionService dimensionService,
|
||||
TagMetaService tagMetaService,
|
||||
public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService,
|
||||
AliasGenerateHelper aliasGenerateHelper, CollectService collectService,
|
||||
DataSetService dataSetService, ApplicationEventPublisher eventPublisher,
|
||||
DimensionService dimensionService, TagMetaService tagMetaService,
|
||||
@Lazy ChatLayerService chatLayerService) {
|
||||
this.metricRepository = metricRepository;
|
||||
this.modelService = modelService;
|
||||
@@ -149,27 +144,21 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
}
|
||||
Long modelId = metricReqs.get(0).getModelId();
|
||||
List<MetricResp> metricResps = getMetrics(new MetaFilter(Lists.newArrayList(modelId)));
|
||||
Map<String, MetricResp> bizNameMap =
|
||||
metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, MetricResp> nameMap =
|
||||
metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, MetricResp> bizNameMap = metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, MetricResp> nameMap = metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
|
||||
List<MetricReq> metricToInsert =
|
||||
metricReqs.stream()
|
||||
.filter(
|
||||
metric ->
|
||||
!bizNameMap.containsKey(metric.getBizName())
|
||||
&& !nameMap.containsKey(metric.getName()))
|
||||
.filter(metric -> !bizNameMap.containsKey(metric.getBizName())
|
||||
&& !nameMap.containsKey(metric.getName()))
|
||||
.collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(metricToInsert)) {
|
||||
return;
|
||||
}
|
||||
List<MetricDO> metricDOS =
|
||||
metricToInsert.stream()
|
||||
.peek(metric -> metric.createdBy(user.getName()))
|
||||
.map(MetricConverter::convert2MetricDO)
|
||||
.collect(Collectors.toList());
|
||||
metricToInsert.stream().peek(metric -> metric.createdBy(user.getName()))
|
||||
.map(MetricConverter::convert2MetricDO).collect(Collectors.toList());
|
||||
metricRepository.createMetricBatch(metricDOS);
|
||||
sendEventBatch(metricDOS, EventType.ADD);
|
||||
}
|
||||
@@ -201,15 +190,11 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
if (CollectionUtils.isEmpty(metricDOS)) {
|
||||
return;
|
||||
}
|
||||
metricDOS =
|
||||
metricDOS.stream()
|
||||
.peek(
|
||||
metricDO -> {
|
||||
metricDO.setStatus(metaBatchReq.getStatus());
|
||||
metricDO.setUpdatedAt(new Date());
|
||||
metricDO.setUpdatedBy(user.getName());
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
metricDOS = metricDOS.stream().peek(metricDO -> {
|
||||
metricDO.setStatus(metaBatchReq.getStatus());
|
||||
metricDO.setUpdatedAt(new Date());
|
||||
metricDO.setUpdatedBy(user.getName());
|
||||
}).collect(Collectors.toList());
|
||||
metricRepository.batchUpdateStatus(metricDOS);
|
||||
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
||||
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
||||
@@ -314,20 +299,14 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
return metricRespPageInfo;
|
||||
}
|
||||
Map<Long, Double> result =
|
||||
dataSetMapInfoMap.values().stream()
|
||||
.map(DataSetMapInfo::getMapFields)
|
||||
.filter(Objects::nonNull)
|
||||
.flatMap(Collection::stream)
|
||||
.filter(
|
||||
schemaElementMatch ->
|
||||
SchemaElementType.METRIC.equals(
|
||||
schemaElementMatch.getElement().getType()))
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
schemaElementMatch ->
|
||||
schemaElementMatch.getElement().getId(),
|
||||
SchemaElementMatch::getSimilarity,
|
||||
(existingValue, newValue) -> existingValue));
|
||||
dataSetMapInfoMap.values().stream().map(DataSetMapInfo::getMapFields)
|
||||
.filter(Objects::nonNull).flatMap(Collection::stream)
|
||||
.filter(schemaElementMatch -> SchemaElementType.METRIC
|
||||
.equals(schemaElementMatch.getElement().getType()))
|
||||
.collect(Collectors.toMap(
|
||||
schemaElementMatch -> schemaElementMatch.getElement().getId(),
|
||||
SchemaElementMatch::getSimilarity,
|
||||
(existingValue, newValue) -> existingValue));
|
||||
List<Long> metricIds = new ArrayList<>(result.keySet());
|
||||
if (CollectionUtils.isEmpty(result.keySet())) {
|
||||
return metricRespPageInfo;
|
||||
@@ -434,20 +413,13 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
return new ArrayList<>(metricRespFiltered);
|
||||
}
|
||||
|
||||
private boolean filterByField(
|
||||
List<MetricResp> metricResps,
|
||||
MetricResp metricResp,
|
||||
List<String> fields,
|
||||
Set<MetricResp> metricRespFiltered) {
|
||||
private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
|
||||
List<String> fields, Set<MetricResp> metricRespFiltered) {
|
||||
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
|
||||
List<Long> ids =
|
||||
metricResp.getMetricDefineByMetricParams().getMetrics().stream()
|
||||
.map(MetricParam::getId)
|
||||
.collect(Collectors.toList());
|
||||
List<MetricResp> metricById =
|
||||
metricResps.stream()
|
||||
.filter(metric -> ids.contains(metric.getId()))
|
||||
.collect(Collectors.toList());
|
||||
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics().stream()
|
||||
.map(MetricParam::getId).collect(Collectors.toList());
|
||||
List<MetricResp> metricById = metricResps.stream()
|
||||
.filter(metric -> ids.contains(metric.getId())).collect(Collectors.toList());
|
||||
for (MetricResp metric : metricById) {
|
||||
if (filterByField(metricResps, metric, fields, metricRespFiltered)) {
|
||||
metricRespFiltered.add(metricResp);
|
||||
@@ -461,12 +433,10 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
}
|
||||
} else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
|
||||
List<MeasureParam> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
|
||||
List<String> fieldNameDepended =
|
||||
measures.stream()
|
||||
.map(MeasureParam::getBizName)
|
||||
// measure bizName = model bizName_fieldName
|
||||
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
|
||||
.collect(Collectors.toList());
|
||||
List<String> fieldNameDepended = measures.stream().map(MeasureParam::getBizName)
|
||||
// measure bizName = model bizName_fieldName
|
||||
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
|
||||
.collect(Collectors.toList());
|
||||
if (fields.stream().anyMatch(fieldNameDepended::contains)) {
|
||||
metricRespFiltered.add(metricResp);
|
||||
return true;
|
||||
@@ -480,12 +450,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
MetricFilter metricFilter = new MetricFilter();
|
||||
metricFilter.setModelIds(Lists.newArrayList(modelId));
|
||||
List<MetricResp> metricResps = getMetrics(metricFilter);
|
||||
return metricResps.stream()
|
||||
.filter(
|
||||
metricResp ->
|
||||
MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|
||||
|| MetricDefineType.MEASURE.equals(
|
||||
metricResp.getMetricDefineType()))
|
||||
return metricResps.stream().filter(
|
||||
metricResp -> MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|
||||
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@@ -546,19 +513,10 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
@Override
|
||||
public List<String> mockAlias(MetricBaseReq metricReq, String mockType, User user) {
|
||||
|
||||
String mockAlias =
|
||||
aliasGenerateHelper.generateAlias(
|
||||
mockType,
|
||||
metricReq.getName(),
|
||||
metricReq.getBizName(),
|
||||
"",
|
||||
metricReq.getDescription());
|
||||
String ret =
|
||||
mockAlias
|
||||
.replaceAll("`", "")
|
||||
.replace("json", "")
|
||||
.replace("\n", "")
|
||||
.replace(" ", "");
|
||||
String mockAlias = aliasGenerateHelper.generateAlias(mockType, metricReq.getName(),
|
||||
metricReq.getBizName(), "", metricReq.getDescription());
|
||||
String ret = mockAlias.replaceAll("`", "").replace("json", "").replace("\n", "")
|
||||
.replace(" ", "");
|
||||
return JSONObject.parseObject(ret, new TypeReference<List<String>>() {});
|
||||
}
|
||||
|
||||
@@ -568,8 +526,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
if (CollectionUtils.isEmpty(metricResps)) {
|
||||
return new HashSet<>();
|
||||
}
|
||||
return metricResps.stream()
|
||||
.flatMap(metricResp -> metricResp.getClassifications().stream())
|
||||
return metricResps.stream().flatMap(metricResp -> metricResp.getClassifications().stream())
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
@@ -580,11 +537,10 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
if (metricResp == null) {
|
||||
return drillDownDimensions;
|
||||
}
|
||||
if (metricResp.getRelateDimension() != null
|
||||
&& !CollectionUtils.isEmpty(
|
||||
metricResp.getRelateDimension().getDrillDownDimensions())) {
|
||||
for (DrillDownDimension drillDownDimension :
|
||||
metricResp.getRelateDimension().getDrillDownDimensions()) {
|
||||
if (metricResp.getRelateDimension() != null && !CollectionUtils
|
||||
.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) {
|
||||
for (DrillDownDimension drillDownDimension : metricResp.getRelateDimension()
|
||||
.getDrillDownDimensions()) {
|
||||
if (drillDownDimension.isInheritedFromModel()
|
||||
&& !drillDownDimension.isNecessary()) {
|
||||
continue;
|
||||
@@ -597,10 +553,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
return drillDownDimensions;
|
||||
}
|
||||
for (DrillDownDimension drillDownDimension : modelResp.getDrillDownDimensions()) {
|
||||
if (!drillDownDimensions.stream()
|
||||
.map(DrillDownDimension::getDimensionId)
|
||||
.collect(Collectors.toList())
|
||||
.contains(drillDownDimension.getDimensionId())) {
|
||||
if (!drillDownDimensions.stream().map(DrillDownDimension::getDimensionId)
|
||||
.collect(Collectors.toList()).contains(drillDownDimension.getDimensionId())) {
|
||||
drillDownDimension.setInheritedFromModel(true);
|
||||
drillDownDimensions.add(drillDownDimension);
|
||||
}
|
||||
@@ -639,29 +593,23 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
MetaFilter metaFilter = new MetaFilter();
|
||||
metaFilter.setModelIds(Lists.newArrayList(modelId));
|
||||
List<MetricResp> metricResps = getMetrics(metaFilter);
|
||||
Map<String, MetricResp> bizNameMap =
|
||||
metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, MetricResp> nameMap =
|
||||
metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, MetricResp> bizNameMap = metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
|
||||
Map<String, MetricResp> nameMap = metricResps.stream()
|
||||
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
|
||||
for (MetricBaseReq metricReq : metricReqs) {
|
||||
if (bizNameMap.containsKey(metricReq.getBizName())) {
|
||||
MetricResp metricResp = bizNameMap.get(metricReq.getBizName());
|
||||
if (!metricResp.getId().equals(metricReq.getId())) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"该模型下存在相同的指标字段名:%s 创建人:%s",
|
||||
metricReq.getBizName(), metricResp.getCreatedBy()));
|
||||
throw new RuntimeException(String.format("该模型下存在相同的指标字段名:%s 创建人:%s",
|
||||
metricReq.getBizName(), metricResp.getCreatedBy()));
|
||||
}
|
||||
}
|
||||
if (nameMap.containsKey(metricReq.getName())) {
|
||||
MetricResp metricResp = nameMap.get(metricReq.getName());
|
||||
if (!metricResp.getId().equals(metricReq.getId())) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"该模型下存在相同的指标名:%s 创建人:%s",
|
||||
metricReq.getName(), metricResp.getCreatedBy()));
|
||||
throw new RuntimeException(String.format("该模型下存在相同的指标名:%s 创建人:%s",
|
||||
metricReq.getName(), metricResp.getCreatedBy()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -678,13 +626,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
ModelFilter modelFilter = new ModelFilter(false, modelIds);
|
||||
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
|
||||
if (!CollectionUtils.isEmpty(metricDOS)) {
|
||||
metricResps =
|
||||
metricDOS.stream()
|
||||
.map(
|
||||
metricDO ->
|
||||
MetricConverter.convert2MetricResp(
|
||||
metricDO, modelMap, collect))
|
||||
.collect(Collectors.toList());
|
||||
metricResps = metricDOS.stream().map(
|
||||
metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
return metricResps;
|
||||
}
|
||||
@@ -729,19 +673,15 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
MetricResp metricResp =
|
||||
MetricConverter.convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
|
||||
fillDefaultAgg(metricResp);
|
||||
return DataItem.builder()
|
||||
.id(metricDO.getId() + Constants.UNDERLINE)
|
||||
.name(metricDO.getName())
|
||||
.bizName(metricDO.getBizName())
|
||||
.modelId(metricDO.getModelId() + Constants.UNDERLINE)
|
||||
.type(TypeEnums.METRIC)
|
||||
.defaultAgg(metricResp.getDefaultAgg())
|
||||
.build();
|
||||
return DataItem.builder().id(metricDO.getId() + Constants.UNDERLINE)
|
||||
.name(metricDO.getName()).bizName(metricDO.getBizName())
|
||||
.modelId(metricDO.getModelId() + Constants.UNDERLINE).type(TypeEnums.METRIC)
|
||||
.defaultAgg(metricResp.getDefaultAgg()).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchFillMetricDefaultAgg(
|
||||
List<MetricResp> metricResps, List<ModelResp> modelResps) {
|
||||
public void batchFillMetricDefaultAgg(List<MetricResp> metricResps,
|
||||
List<ModelResp> modelResps) {
|
||||
Map<Long, ModelResp> modelRespMap =
|
||||
modelResps.stream().collect(Collectors.toMap(ModelResp::getId, m -> m));
|
||||
for (MetricResp metricResp : metricResps) {
|
||||
@@ -762,9 +702,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
}
|
||||
|
||||
private String getDefaultAgg(MetricResp metricResp, ModelResp modelResp) {
|
||||
if (modelResp == null
|
||||
|| (Objects.nonNull(metricResp.getDefaultAgg())
|
||||
&& !metricResp.getDefaultAgg().isEmpty())) {
|
||||
if (modelResp == null || (Objects.nonNull(metricResp.getDefaultAgg())
|
||||
&& !metricResp.getDefaultAgg().isEmpty())) {
|
||||
return metricResp.getDefaultAgg();
|
||||
}
|
||||
// FIELD define will get from expr
|
||||
@@ -824,20 +763,12 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
queryMetricReq.setDateInfo(null);
|
||||
}
|
||||
// 4. set groups
|
||||
List<String> dimensionBizNames =
|
||||
dimensionResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.filter(
|
||||
entry ->
|
||||
queryMetricReq.getDimensionNames().contains(entry.getName())
|
||||
|| queryMetricReq
|
||||
.getDimensionNames()
|
||||
.contains(entry.getBizName())
|
||||
|| queryMetricReq
|
||||
.getDimensionIds()
|
||||
.contains(entry.getId()))
|
||||
.map(SchemaItem::getBizName)
|
||||
.collect(Collectors.toList());
|
||||
List<String> dimensionBizNames = dimensionResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.filter(entry -> queryMetricReq.getDimensionNames().contains(entry.getName())
|
||||
|| queryMetricReq.getDimensionNames().contains(entry.getBizName())
|
||||
|| queryMetricReq.getDimensionIds().contains(entry.getId()))
|
||||
.map(SchemaItem::getBizName).collect(Collectors.toList());
|
||||
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
DateConf dateInfo = queryMetricReq.getDateInfo();
|
||||
@@ -848,11 +779,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
queryStructReq.getGroups().addAll(dimensionBizNames);
|
||||
}
|
||||
// 5. set aggregators
|
||||
List<String> metricBizNames =
|
||||
metricResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.map(SchemaItem::getBizName)
|
||||
.collect(Collectors.toList());
|
||||
List<String> metricBizNames = metricResps.stream()
|
||||
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
|
||||
.map(SchemaItem::getBizName).collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(metricBizNames)) {
|
||||
throw new IllegalArgumentException(
|
||||
"Invalid input parameters, unable to obtain valid metrics");
|
||||
@@ -898,18 +827,14 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
}
|
||||
}
|
||||
}
|
||||
String keyWithMaxSize =
|
||||
modelClusterToMatchCount.entrySet().stream()
|
||||
.max(Comparator.comparingInt(entry -> entry.getValue().size()))
|
||||
.map(Map.Entry::getKey)
|
||||
.orElse(null);
|
||||
String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream()
|
||||
.max(Comparator.comparingInt(entry -> entry.getValue().size()))
|
||||
.map(Map.Entry::getKey).orElse(null);
|
||||
|
||||
return modelClusterMap.get(keyWithMaxSize);
|
||||
}
|
||||
|
||||
private Set<Long> getModelIds(
|
||||
Set<Long> modelIdsByDomainId,
|
||||
List<MetricResp> metricResps,
|
||||
private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
|
||||
List<DimensionResp> dimensionResps) {
|
||||
Set<Long> result = new HashSet<>();
|
||||
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
|
||||
@@ -920,10 +845,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
metricResps.stream().map(entry -> entry.getModelId()).collect(Collectors.toSet());
|
||||
result.addAll(metricModelIds);
|
||||
|
||||
Set<Long> dimensionModelIds =
|
||||
dimensionResps.stream()
|
||||
.map(entry -> entry.getModelId())
|
||||
.collect(Collectors.toSet());
|
||||
Set<Long> dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId())
|
||||
.collect(Collectors.toSet());
|
||||
result.addAll(dimensionModelIds);
|
||||
return result;
|
||||
}
|
||||
@@ -942,9 +865,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
||||
}
|
||||
|
||||
private Set<Long> getModelIdsByDomainId(QueryMetricReq queryMetricReq) {
|
||||
List<ModelResp> modelResps =
|
||||
modelService.getAllModelByDomainIds(
|
||||
Collections.singletonList(queryMetricReq.getDomainId()));
|
||||
List<ModelResp> modelResps = modelService
|
||||
.getAllModelByDomainIds(Collections.singletonList(queryMetricReq.getDomainId()));
|
||||
return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,9 +48,7 @@ public class ModelRelaServiceImpl extends ServiceImpl<ModelRelaDOMapper, ModelRe
|
||||
if (CollectionUtils.isEmpty(modelIds)) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
wrapper.lambda()
|
||||
.in(ModelRelaDO::getFromModelId, modelIds)
|
||||
.or()
|
||||
wrapper.lambda().in(ModelRelaDO::getFromModelId, modelIds).or()
|
||||
.in(ModelRelaDO::getToModelId, modelIds);
|
||||
return list(wrapper).stream().map(this::convert).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@@ -78,14 +78,9 @@ public class ModelServiceImpl implements ModelService {
|
||||
|
||||
private DateInfoRepository dateInfoRepository;
|
||||
|
||||
public ModelServiceImpl(
|
||||
ModelRepository modelRepository,
|
||||
DatabaseService databaseService,
|
||||
@Lazy DimensionService dimensionService,
|
||||
@Lazy MetricService metricService,
|
||||
DomainService domainService,
|
||||
UserService userService,
|
||||
DataSetService dataSetService,
|
||||
public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
|
||||
@Lazy DimensionService dimensionService, @Lazy MetricService metricService,
|
||||
DomainService domainService, UserService userService, DataSetService dataSetService,
|
||||
DateInfoRepository dateInfoRepository) {
|
||||
this.modelRepository = modelRepository;
|
||||
this.databaseService = databaseService;
|
||||
@@ -185,9 +180,7 @@ public class ModelServiceImpl implements ModelService {
|
||||
metaFilter.setFieldsDepend(fieldRemovedReq.getFields());
|
||||
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
|
||||
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
|
||||
return UnAvailableItemResp.builder()
|
||||
.dimensionResps(dimensionResps)
|
||||
.metricResps(metricResps)
|
||||
return UnAvailableItemResp.builder().dimensionResps(dimensionResps).metricResps(metricResps)
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -204,9 +197,8 @@ public class ModelServiceImpl implements ModelService {
|
||||
private void checkParams(ModelReq modelReq) {
|
||||
String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(modelReq.getName());
|
||||
if (StringUtils.isNotBlank(forbiddenCharacters)) {
|
||||
String message =
|
||||
String.format(
|
||||
"模型名称[%s]包含特殊字符(%s), 请修改", modelReq.getName(), forbiddenCharacters);
|
||||
String message = String.format("模型名称[%s]包含特殊字符(%s), 请修改", modelReq.getName(),
|
||||
forbiddenCharacters);
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
|
||||
@@ -228,10 +220,8 @@ public class ModelServiceImpl implements ModelService {
|
||||
NameCheckUtils.findForbiddenCharacters(measure.getName());
|
||||
if (StringUtils.isNotBlank(measure.getName())
|
||||
&& StringUtils.isNotBlank(measureForbiddenCharacters)) {
|
||||
String message =
|
||||
String.format(
|
||||
"度量[%s]包含特殊字符(%s), 请修改",
|
||||
measure.getName(), measureForbiddenCharacters);
|
||||
String message = String.format("度量[%s]包含特殊字符(%s), 请修改", measure.getName(),
|
||||
measureForbiddenCharacters);
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
}
|
||||
@@ -239,9 +229,8 @@ public class ModelServiceImpl implements ModelService {
|
||||
String dimForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(dim.getName());
|
||||
if (StringUtils.isNotBlank(dim.getName())
|
||||
&& StringUtils.isNotBlank(dimForbiddenCharacters)) {
|
||||
String message =
|
||||
String.format(
|
||||
"维度[%s]包含特殊字符(%s), 请修改", dim.getName(), dimForbiddenCharacters);
|
||||
String message = String.format("维度[%s]包含特殊字符(%s), 请修改", dim.getName(),
|
||||
dimForbiddenCharacters);
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
}
|
||||
@@ -250,10 +239,8 @@ public class ModelServiceImpl implements ModelService {
|
||||
NameCheckUtils.findForbiddenCharacters(identify.getName());
|
||||
if (StringUtils.isNotBlank(identify.getName())
|
||||
&& StringUtils.isNotBlank(identifyForbiddenCharacters)) {
|
||||
String message =
|
||||
String.format(
|
||||
"主键/外键[%s]包含特殊字符(%s), 请修改",
|
||||
identify.getName(), identifyForbiddenCharacters);
|
||||
String message = String.format("主键/外键[%s]包含特殊字符(%s), 请修改", identify.getName(),
|
||||
identifyForbiddenCharacters);
|
||||
throw new InvalidArgumentException(message);
|
||||
}
|
||||
}
|
||||
@@ -304,26 +291,21 @@ public class ModelServiceImpl implements ModelService {
|
||||
List<ModelResp> modelRespsAuthInheritDomain =
|
||||
getModelRespAuthInheritDomain(user, domainId, authType);
|
||||
modelRespSet.addAll(modelRespsAuthInheritDomain);
|
||||
return modelRespSet.stream()
|
||||
.sorted(Comparator.comparingLong(ModelResp::getId))
|
||||
return modelRespSet.stream().sorted(Comparator.comparingLong(ModelResp::getId))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<ModelResp> getModelRespAuthInheritDomain(
|
||||
User user, Long domainId, AuthType authType) {
|
||||
public List<ModelResp> getModelRespAuthInheritDomain(User user, Long domainId,
|
||||
AuthType authType) {
|
||||
List<Long> domainIds =
|
||||
domainService.getDomainAuthSet(user, authType).stream()
|
||||
.filter(
|
||||
domainResp -> {
|
||||
if (domainId == null) {
|
||||
return true;
|
||||
} else {
|
||||
return domainId.equals(domainResp.getId())
|
||||
|| domainId.equals(domainResp.getParentId());
|
||||
}
|
||||
})
|
||||
.map(DomainResp::getId)
|
||||
.collect(Collectors.toList());
|
||||
domainService.getDomainAuthSet(user, authType).stream().filter(domainResp -> {
|
||||
if (domainId == null) {
|
||||
return true;
|
||||
} else {
|
||||
return domainId.equals(domainResp.getId())
|
||||
|| domainId.equals(domainResp.getParentId());
|
||||
}
|
||||
}).map(DomainResp::getId).collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(domainIds)) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
@@ -342,16 +324,14 @@ public class ModelServiceImpl implements ModelService {
|
||||
Set<String> orgIds = userService.getUserAllOrgId(user.getName());
|
||||
List<ModelResp> modelWithAuth = Lists.newArrayList();
|
||||
if (authTypeEnum.equals(AuthType.ADMIN)) {
|
||||
modelWithAuth =
|
||||
modelResps.stream()
|
||||
.filter(modelResp -> checkAdminPermission(orgIds, user, modelResp))
|
||||
.collect(Collectors.toList());
|
||||
modelWithAuth = modelResps.stream()
|
||||
.filter(modelResp -> checkAdminPermission(orgIds, user, modelResp))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
if (authTypeEnum.equals(AuthType.VISIBLE)) {
|
||||
modelWithAuth =
|
||||
modelResps.stream()
|
||||
.filter(domainResp -> checkDataSetPermission(orgIds, user, domainResp))
|
||||
.collect(Collectors.toList());
|
||||
modelWithAuth = modelResps.stream()
|
||||
.filter(domainResp -> checkDataSetPermission(orgIds, user, domainResp))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
return modelWithAuth;
|
||||
}
|
||||
@@ -368,8 +348,7 @@ public class ModelServiceImpl implements ModelService {
|
||||
if (CollectionUtils.isEmpty(modelResps)) {
|
||||
return modelResps;
|
||||
}
|
||||
return modelResps.stream()
|
||||
.filter(modelResp -> domainIds.contains(modelResp.getDomainId()))
|
||||
return modelResps.stream().filter(modelResp -> domainIds.contains(modelResp.getDomainId()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@@ -434,35 +413,23 @@ public class ModelServiceImpl implements ModelService {
|
||||
if (CollectionUtils.isEmpty(modelDOS)) {
|
||||
return;
|
||||
}
|
||||
modelDOS =
|
||||
modelDOS.stream()
|
||||
.peek(
|
||||
modelDO -> {
|
||||
modelDO.setStatus(metaBatchReq.getStatus());
|
||||
modelDO.setUpdatedAt(new Date());
|
||||
modelDO.setUpdatedBy(user.getName());
|
||||
if (StatusEnum.OFFLINE
|
||||
.getCode()
|
||||
.equals(metaBatchReq.getStatus())
|
||||
|| StatusEnum.DELETED
|
||||
.getCode()
|
||||
.equals(metaBatchReq.getStatus())) {
|
||||
metricService.sendMetricEventBatch(
|
||||
Lists.newArrayList(modelDO.getId()),
|
||||
EventType.DELETE);
|
||||
dimensionService.sendDimensionEventBatch(
|
||||
Lists.newArrayList(modelDO.getId()),
|
||||
EventType.DELETE);
|
||||
} else if (StatusEnum.ONLINE
|
||||
.getCode()
|
||||
.equals(metaBatchReq.getStatus())) {
|
||||
metricService.sendMetricEventBatch(
|
||||
Lists.newArrayList(modelDO.getId()), EventType.ADD);
|
||||
dimensionService.sendDimensionEventBatch(
|
||||
Lists.newArrayList(modelDO.getId()), EventType.ADD);
|
||||
}
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
modelDOS = modelDOS.stream().peek(modelDO -> {
|
||||
modelDO.setStatus(metaBatchReq.getStatus());
|
||||
modelDO.setUpdatedAt(new Date());
|
||||
modelDO.setUpdatedBy(user.getName());
|
||||
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|
||||
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
|
||||
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()),
|
||||
EventType.DELETE);
|
||||
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()),
|
||||
EventType.DELETE);
|
||||
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
|
||||
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()),
|
||||
EventType.ADD);
|
||||
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()),
|
||||
EventType.ADD);
|
||||
}
|
||||
}).collect(Collectors.toList());
|
||||
modelRepository.batchUpdate(modelDOS);
|
||||
}
|
||||
|
||||
@@ -472,14 +439,13 @@ public class ModelServiceImpl implements ModelService {
|
||||
|
||||
private List<DateInfoReq> convert(List<DateInfoDO> dateInfoDOList) {
|
||||
List<DateInfoReq> dateInfoCommendList = new ArrayList<>();
|
||||
dateInfoDOList.forEach(
|
||||
dateInfoDO -> {
|
||||
DateInfoReq dateInfoCommend = new DateInfoReq();
|
||||
BeanUtils.copyProperties(dateInfoDO, dateInfoCommend);
|
||||
dateInfoCommend.setUnavailableDateList(
|
||||
JsonUtil.toList(dateInfoDO.getUnavailableDateList(), String.class));
|
||||
dateInfoCommendList.add(dateInfoCommend);
|
||||
});
|
||||
dateInfoDOList.forEach(dateInfoDO -> {
|
||||
DateInfoReq dateInfoCommend = new DateInfoReq();
|
||||
BeanUtils.copyProperties(dateInfoDO, dateInfoCommend);
|
||||
dateInfoCommend.setUnavailableDateList(
|
||||
JsonUtil.toList(dateInfoDO.getUnavailableDateList(), String.class));
|
||||
dateInfoCommendList.add(dateInfoCommend);
|
||||
});
|
||||
return dateInfoCommendList;
|
||||
}
|
||||
|
||||
@@ -504,8 +470,8 @@ public class ModelServiceImpl implements ModelService {
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean checkDataSetPermission(
|
||||
Set<String> orgIds, User user, ModelResp modelResp) {
|
||||
public static boolean checkDataSetPermission(Set<String> orgIds, User user,
|
||||
ModelResp modelResp) {
|
||||
if (checkAdminPermission(orgIds, user, modelResp)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -26,8 +26,8 @@ public class QueryRuleServiceImpl implements QueryRuleService {
|
||||
private final QueryRuleRepository queryRuleRepository;
|
||||
private final DataSetService dataSetService;
|
||||
|
||||
public QueryRuleServiceImpl(
|
||||
QueryRuleRepository queryRuleRepository, DataSetService dataSetService) {
|
||||
public QueryRuleServiceImpl(QueryRuleRepository queryRuleRepository,
|
||||
DataSetService dataSetService) {
|
||||
this.queryRuleRepository = queryRuleRepository;
|
||||
this.dataSetService = dataSetService;
|
||||
}
|
||||
|
||||
@@ -48,13 +48,17 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
|
||||
private static final int RESULT_SIZE = 10;
|
||||
|
||||
@Autowired private DataSetService dataSetService;
|
||||
@Autowired
|
||||
private DataSetService dataSetService;
|
||||
|
||||
@Autowired private SchemaService schemaService;
|
||||
@Autowired
|
||||
private SchemaService schemaService;
|
||||
|
||||
@Autowired private KnowledgeBaseService knowledgeBaseService;
|
||||
@Autowired
|
||||
private KnowledgeBaseService knowledgeBaseService;
|
||||
|
||||
@Autowired private SearchMatchStrategy searchMatchStrategy;
|
||||
@Autowired
|
||||
private SearchMatchStrategy searchMatchStrategy;
|
||||
|
||||
@Override
|
||||
public List<SearchResult> retrieve(QueryNLReq queryNLReq) {
|
||||
@@ -65,9 +69,8 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
schemaService.getSemanticSchema(queryNLReq.getDataSetIds());
|
||||
List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
|
||||
final Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
|
||||
Map<Long, List<Long>> modelIdToDataSetIds =
|
||||
dataSetService.getModelIdToDataSetIds(
|
||||
new ArrayList<>(dataSetIdToName.keySet()), User.getFakeUser());
|
||||
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(
|
||||
new ArrayList<>(dataSetIdToName.keySet()), User.getFakeUser());
|
||||
// 2.detect by segment
|
||||
List<S2Term> originals = knowledgeBaseService.getTerms(queryText, modelIdToDataSetIds);
|
||||
log.debug("hanlp parse result: {}", originals);
|
||||
@@ -86,14 +89,9 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
Optional<Map.Entry<MatchText, List<HanlpMapResult>>> mostSimilarSearchResult =
|
||||
regTextMap.entrySet().stream()
|
||||
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
|
||||
.reduce(
|
||||
(entry1, entry2) ->
|
||||
entry1.getKey().getDetectSegment().length()
|
||||
>= entry2.getKey()
|
||||
.getDetectSegment()
|
||||
.length()
|
||||
? entry1
|
||||
: entry2);
|
||||
.reduce((entry1, entry2) -> entry1.getKey().getDetectSegment()
|
||||
.length() >= entry2.getKey().getDetectSegment().length() ? entry1
|
||||
: entry2);
|
||||
|
||||
// 4.optimize the results after the query
|
||||
if (!mostSimilarSearchResult.isPresent()) {
|
||||
@@ -108,12 +106,8 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
List<Long> possibleDataSets = getPossibleDataSets(queryNLReq, originals, dataSetIds);
|
||||
|
||||
// 5.1 priority dimension metric
|
||||
boolean existMetricAndDimension =
|
||||
searchMetricAndDimension(
|
||||
new HashSet<>(possibleDataSets),
|
||||
dataSetIdToName,
|
||||
searchTextEntry,
|
||||
searchResults);
|
||||
boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleDataSets),
|
||||
dataSetIdToName, searchTextEntry, searchResults);
|
||||
|
||||
// 5.2 process based on dimension values
|
||||
MatchText matchText = searchTextEntry.getKey();
|
||||
@@ -123,24 +117,17 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
|
||||
for (Map.Entry<String, String> natureToNameEntry : natureToNameMap.entrySet()) {
|
||||
|
||||
Set<SearchResult> searchResultSet =
|
||||
searchDimensionValue(
|
||||
metricsDb,
|
||||
dataSetIdToName,
|
||||
dataSetInfoStat.getMetricDataSetCount(),
|
||||
existMetricAndDimension,
|
||||
matchText,
|
||||
natureToNameMap,
|
||||
natureToNameEntry,
|
||||
queryNLReq.getQueryFilters());
|
||||
Set<SearchResult> searchResultSet = searchDimensionValue(metricsDb, dataSetIdToName,
|
||||
dataSetInfoStat.getMetricDataSetCount(), existMetricAndDimension, matchText,
|
||||
natureToNameMap, natureToNameEntry, queryNLReq.getQueryFilters());
|
||||
|
||||
searchResults.addAll(searchResultSet);
|
||||
}
|
||||
return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<Long> getPossibleDataSets(
|
||||
QueryNLReq queryCtx, List<S2Term> originals, Set<Long> dataSetIds) {
|
||||
private List<Long> getPossibleDataSets(QueryNLReq queryCtx, List<S2Term> originals,
|
||||
Set<Long> dataSetIds) {
|
||||
if (CollectionUtils.isNotEmpty(dataSetIds)) {
|
||||
return new ArrayList<>(dataSetIds);
|
||||
}
|
||||
@@ -155,15 +142,10 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
return possibleDataSets;
|
||||
}
|
||||
|
||||
private Set<SearchResult> searchDimensionValue(
|
||||
List<SchemaElement> metricsDb,
|
||||
Map<Long, String> modelToName,
|
||||
long metricModelCount,
|
||||
boolean existMetricAndDimension,
|
||||
MatchText matchText,
|
||||
Map<String, String> natureToNameMap,
|
||||
Map.Entry<String, String> natureToNameEntry,
|
||||
QueryFilters queryFilters) {
|
||||
private Set<SearchResult> searchDimensionValue(List<SchemaElement> metricsDb,
|
||||
Map<Long, String> modelToName, long metricModelCount, boolean existMetricAndDimension,
|
||||
MatchText matchText, Map<String, String> natureToNameMap,
|
||||
Map.Entry<String, String> natureToNameEntry, QueryFilters queryFilters) {
|
||||
|
||||
Set<SearchResult> searchResults = new LinkedHashSet();
|
||||
String nature = natureToNameEntry.getKey();
|
||||
@@ -175,15 +157,10 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
if (SchemaElementType.ENTITY.equals(schemaElementType)) {
|
||||
return searchResults;
|
||||
}
|
||||
// If there are no metric/dimension, complete the metric information
|
||||
SearchResult searchResult =
|
||||
SearchResult.builder()
|
||||
.modelId(modelId)
|
||||
.modelName(modelToName.get(modelId))
|
||||
.recommend(matchText.getRegText() + wordName)
|
||||
.schemaElementType(schemaElementType)
|
||||
.subRecommend(wordName)
|
||||
.build();
|
||||
// If there are no metric/dimension, complete the metric information
|
||||
SearchResult searchResult = SearchResult.builder().modelId(modelId)
|
||||
.modelName(modelToName.get(modelId)).recommend(matchText.getRegText() + wordName)
|
||||
.schemaElementType(schemaElementType).subRecommend(wordName).build();
|
||||
|
||||
if (metricModelCount <= 0 && !existMetricAndDimension) {
|
||||
if (filterByQueryFilter(wordName, queryFilters)) {
|
||||
@@ -191,24 +168,15 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
}
|
||||
searchResults.add(searchResult);
|
||||
int metricSize = getMetricSize(natureToNameMap);
|
||||
List<String> metrics =
|
||||
filerMetricsByModel(metricsDb, modelId, metricSize * 3).stream()
|
||||
.limit(metricSize)
|
||||
.collect(Collectors.toList());
|
||||
List<String> metrics = filerMetricsByModel(metricsDb, modelId, metricSize * 3).stream()
|
||||
.limit(metricSize).collect(Collectors.toList());
|
||||
|
||||
for (String metric : metrics) {
|
||||
SearchResult result =
|
||||
SearchResult.builder()
|
||||
.modelId(modelId)
|
||||
.modelName(modelToName.get(modelId))
|
||||
.recommend(
|
||||
matchText.getRegText()
|
||||
+ wordName
|
||||
+ DictWordType.SPACE
|
||||
+ metric)
|
||||
.subRecommend(wordName + DictWordType.SPACE + metric)
|
||||
.isComplete(false)
|
||||
.build();
|
||||
SearchResult result = SearchResult.builder().modelId(modelId)
|
||||
.modelName(modelToName.get(modelId))
|
||||
.recommend(matchText.getRegText() + wordName + DictWordType.SPACE + metric)
|
||||
.subRecommend(wordName + DictWordType.SPACE + metric).isComplete(false)
|
||||
.build();
|
||||
searchResults.add(result);
|
||||
}
|
||||
} else {
|
||||
@@ -238,22 +206,19 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
return true;
|
||||
}
|
||||
|
||||
protected List<String> filerMetricsByModel(
|
||||
List<SchemaElement> metricsDb, Long model, int metricSize) {
|
||||
protected List<String> filerMetricsByModel(List<SchemaElement> metricsDb, Long model,
|
||||
int metricSize) {
|
||||
if (CollectionUtils.isEmpty(metricsDb)) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
return metricsDb.stream()
|
||||
.filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getDataSetId()))
|
||||
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
|
||||
.flatMap(
|
||||
entry -> {
|
||||
List<String> result = new ArrayList<>();
|
||||
result.add(entry.getName());
|
||||
return result.stream();
|
||||
})
|
||||
.limit(metricSize)
|
||||
.collect(Collectors.toList());
|
||||
.flatMap(entry -> {
|
||||
List<String> result = new ArrayList<>();
|
||||
result.add(entry.getName());
|
||||
return result.stream();
|
||||
}).limit(metricSize).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -267,35 +232,23 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
Set<Long> possibleModels) {
|
||||
List<HanlpMapResult> recommendValues = recommendTextListEntry.getValue();
|
||||
return recommendValues.stream()
|
||||
.flatMap(
|
||||
entry ->
|
||||
entry.getNatures().stream()
|
||||
.filter(
|
||||
nature -> {
|
||||
if (CollectionUtils.isEmpty(possibleModels)) {
|
||||
return true;
|
||||
}
|
||||
Long model = NatureHelper.getDataSetId(nature);
|
||||
return possibleModels.contains(model);
|
||||
})
|
||||
.map(
|
||||
nature -> {
|
||||
DictWord posDO = new DictWord();
|
||||
posDO.setWord(entry.getName());
|
||||
posDO.setNature(nature);
|
||||
return posDO;
|
||||
}))
|
||||
.sorted(Comparator.comparingInt(a -> a.getWord().length()))
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
DictWord::getNature,
|
||||
DictWord::getWord,
|
||||
(value1, value2) -> value1,
|
||||
LinkedHashMap::new));
|
||||
.flatMap(entry -> entry.getNatures().stream().filter(nature -> {
|
||||
if (CollectionUtils.isEmpty(possibleModels)) {
|
||||
return true;
|
||||
}
|
||||
Long model = NatureHelper.getDataSetId(nature);
|
||||
return possibleModels.contains(model);
|
||||
}).map(nature -> {
|
||||
DictWord posDO = new DictWord();
|
||||
posDO.setWord(entry.getName());
|
||||
posDO.setNature(nature);
|
||||
return posDO;
|
||||
})).sorted(Comparator.comparingInt(a -> a.getWord().length()))
|
||||
.collect(Collectors.toMap(DictWord::getNature, DictWord::getWord,
|
||||
(value1, value2) -> value1, LinkedHashMap::new));
|
||||
}
|
||||
|
||||
private boolean searchMetricAndDimension(
|
||||
Set<Long> possibleDataSets,
|
||||
private boolean searchMetricAndDimension(Set<Long> possibleDataSets,
|
||||
Map<Long, String> modelToName,
|
||||
Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry,
|
||||
Set<SearchResult> searchResults) {
|
||||
@@ -306,15 +259,12 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
|
||||
for (HanlpMapResult hanlpMapResult : hanlpMapResults) {
|
||||
|
||||
List<ModelWithSemanticType> dimensionMetricClassIds =
|
||||
hanlpMapResult.getNatures().stream()
|
||||
.map(
|
||||
nature ->
|
||||
new ModelWithSemanticType(
|
||||
NatureHelper.getDataSetId(nature),
|
||||
NatureHelper.convertToElementType(nature)))
|
||||
.filter(entry -> matchCondition(entry, possibleDataSets))
|
||||
.collect(Collectors.toList());
|
||||
List<ModelWithSemanticType> dimensionMetricClassIds = hanlpMapResult.getNatures()
|
||||
.stream()
|
||||
.map(nature -> new ModelWithSemanticType(NatureHelper.getDataSetId(nature),
|
||||
NatureHelper.convertToElementType(nature)))
|
||||
.filter(entry -> matchCondition(entry, possibleDataSets))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (CollectionUtils.isEmpty(dimensionMetricClassIds)) {
|
||||
continue;
|
||||
@@ -324,21 +274,15 @@ public class RetrieveServiceImpl implements RetrieveService {
|
||||
Long modelId = modelWithSemanticType.getModel();
|
||||
SchemaElementType schemaElementType = modelWithSemanticType.getSchemaElementType();
|
||||
SearchResult searchResult =
|
||||
SearchResult.builder()
|
||||
.modelId(modelId)
|
||||
.modelName(modelToName.get(modelId))
|
||||
SearchResult.builder().modelId(modelId).modelName(modelToName.get(modelId))
|
||||
.recommend(matchText.getRegText() + hanlpMapResult.getName())
|
||||
.subRecommend(hanlpMapResult.getName())
|
||||
.schemaElementType(schemaElementType)
|
||||
.build();
|
||||
// visibility to filter metrics
|
||||
.schemaElementType(schemaElementType).build();
|
||||
// visibility to filter metrics
|
||||
searchResults.add(searchResult);
|
||||
}
|
||||
log.debug(
|
||||
"parseResult:{},dimensionMetricClassIds:{},possibleDataSets:{}",
|
||||
hanlpMapResult,
|
||||
dimensionMetricClassIds,
|
||||
possibleDataSets);
|
||||
log.debug("parseResult:{},dimensionMetricClassIds:{},possibleDataSets:{}",
|
||||
hanlpMapResult, dimensionMetricClassIds, possibleDataSets);
|
||||
}
|
||||
log.info("searchMetricAndDimension searchResults:{}", searchResults);
|
||||
return existMetric;
|
||||
|
||||
@@ -103,17 +103,10 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
@Value("${s2.schema.cache.enable:true}")
|
||||
private boolean schemaCacheEnable;
|
||||
|
||||
public SchemaServiceImpl(
|
||||
ModelService modelService,
|
||||
DimensionService dimensionService,
|
||||
MetricService metricService,
|
||||
DomainService domainService,
|
||||
DataSetService dataSetService,
|
||||
ModelRelaService modelRelaService,
|
||||
StatUtils statUtils,
|
||||
TagMetaService tagService,
|
||||
TermService termService,
|
||||
DatabaseService databaseService) {
|
||||
public SchemaServiceImpl(ModelService modelService, DimensionService dimensionService,
|
||||
MetricService metricService, DomainService domainService, DataSetService dataSetService,
|
||||
ModelRelaService modelRelaService, StatUtils statUtils, TagMetaService tagService,
|
||||
TermService termService, DatabaseService databaseService) {
|
||||
this.modelService = modelService;
|
||||
this.dimensionService = dimensionService;
|
||||
this.metricService = metricService;
|
||||
@@ -142,8 +135,7 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
if (dataSetId == null) {
|
||||
return null;
|
||||
}
|
||||
return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream()
|
||||
.findFirst()
|
||||
return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream().findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
@@ -159,10 +151,8 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
ids.add(dataSetId);
|
||||
List<DataSetSchemaResp> dataSetSchemaResps = fetchDataSetSchema(ids);
|
||||
if (!CollectionUtils.isEmpty(dataSetSchemaResps)) {
|
||||
Optional<DataSetSchemaResp> dataSetSchemaResp =
|
||||
dataSetSchemaResps.stream()
|
||||
.filter(d -> d.getId().equals(dataSetId))
|
||||
.findFirst();
|
||||
Optional<DataSetSchemaResp> dataSetSchemaResp = dataSetSchemaResps.stream()
|
||||
.filter(d -> d.getId().equals(dataSetId)).findFirst();
|
||||
if (dataSetSchemaResp.isPresent()) {
|
||||
DataSetSchemaResp dataSetSchema = dataSetSchemaResp.get();
|
||||
return DataSetSchemaBuilder.build(dataSetSchema);
|
||||
@@ -199,11 +189,8 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
Map<Long, DataSetResp> dataSetRespMap = getDataSetMap(dataSetResps);
|
||||
Set<Long> domainIds =
|
||||
dataSetResps.stream().map(DataSetResp::getDomainId).collect(Collectors.toSet());
|
||||
List<Long> modelIds =
|
||||
dataSetRespMap.values().stream()
|
||||
.map(DataSetResp::getAllModels)
|
||||
.flatMap(Collection::stream)
|
||||
.collect(Collectors.toList());
|
||||
List<Long> modelIds = dataSetRespMap.values().stream().map(DataSetResp::getAllModels)
|
||||
.flatMap(Collection::stream).collect(Collectors.toList());
|
||||
Map<Long, List<TermResp>> termMaps = termService.getTermSets(domainIds);
|
||||
|
||||
metaFilter.setModelIds(modelIds);
|
||||
@@ -228,28 +215,22 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
}
|
||||
List<MetricSchemaResp> metricSchemaResps =
|
||||
MetricConverter.filterByDataSet(metricResps, dataSetResp).stream()
|
||||
.map(this::convert)
|
||||
.collect(Collectors.toList());
|
||||
.map(this::convert).collect(Collectors.toList());
|
||||
List<DimSchemaResp> dimSchemaResps =
|
||||
DimensionConverter.filterByDataSet(dimensionResps, dataSetResp).stream()
|
||||
.map(this::convert)
|
||||
.collect(Collectors.toList());
|
||||
.map(this::convert).collect(Collectors.toList());
|
||||
DataSetSchemaResp dataSetSchemaResp = new DataSetSchemaResp();
|
||||
BeanUtils.copyProperties(dataSetResp, dataSetSchemaResp);
|
||||
dataSetSchemaResp.setDimensions(dimSchemaResps);
|
||||
dataSetSchemaResp.setMetrics(metricSchemaResps);
|
||||
dataSetSchemaResp.setModelResps(
|
||||
modelResps.stream()
|
||||
.filter(
|
||||
modelResp ->
|
||||
dataSetResp.getAllModels().contains(modelResp.getId()))
|
||||
.collect(Collectors.toList()));
|
||||
dataSetSchemaResp.setModelResps(modelResps.stream()
|
||||
.filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId()))
|
||||
.collect(Collectors.toList()));
|
||||
dataSetSchemaResp.setTermResps(
|
||||
termMaps.getOrDefault(dataSetResp.getDomainId(), Lists.newArrayList()));
|
||||
if (!CollectionUtils.isEmpty(dataSetSchemaResp.getModelResps())) {
|
||||
DatabaseResp databaseResp =
|
||||
databaseService.getDatabase(
|
||||
dataSetSchemaResp.getModelResps().get(0).getDatabaseId());
|
||||
DatabaseResp databaseResp = databaseService
|
||||
.getDatabase(dataSetSchemaResp.getModelResps().get(0).getDatabaseId());
|
||||
dataSetSchemaResp.setDatabaseType(databaseResp.getType());
|
||||
}
|
||||
dataSetSchemaResps.add(dataSetSchemaResp);
|
||||
@@ -265,9 +246,8 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
}
|
||||
MetaFilter metaFilter = new MetaFilter(modelIds);
|
||||
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
|
||||
Map<Long, List<MetricResp>> metricRespMap =
|
||||
metricService.getMetrics(metaFilter).stream()
|
||||
.collect(Collectors.groupingBy(MetricResp::getModelId));
|
||||
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetrics(metaFilter).stream()
|
||||
.collect(Collectors.groupingBy(MetricResp::getModelId));
|
||||
Map<Long, List<DimensionResp>> dimensionRespsMap =
|
||||
dimensionService.getDimensions(metaFilter).stream()
|
||||
.collect(Collectors.groupingBy(DimensionResp::getModelId));
|
||||
@@ -285,19 +265,15 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
metricResps.stream().map(this::convert).collect(Collectors.toList());
|
||||
List<DimSchemaResp> dimensionResps =
|
||||
dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList()).stream()
|
||||
.map(this::convert)
|
||||
.collect(Collectors.toList());
|
||||
.map(this::convert).collect(Collectors.toList());
|
||||
ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
|
||||
BeanUtils.copyProperties(modelResp, modelSchemaResp);
|
||||
modelSchemaResp.setDimensions(dimensionResps);
|
||||
modelSchemaResp.setMetrics(metricSchemaResps);
|
||||
modelSchemaResp.setModelRelas(
|
||||
modelRelas.stream()
|
||||
.filter(
|
||||
modelRela ->
|
||||
modelRela.getFromModelId().equals(modelId)
|
||||
|| modelRela.getToModelId().equals(modelId))
|
||||
.collect(Collectors.toList()));
|
||||
modelSchemaResp.setModelRelas(modelRelas.stream()
|
||||
.filter(modelRela -> modelRela.getFromModelId().equals(modelId)
|
||||
|| modelRela.getToModelId().equals(modelId))
|
||||
.collect(Collectors.toList()));
|
||||
modelSchemaResps.add(modelSchemaResp);
|
||||
}
|
||||
return modelSchemaResps;
|
||||
@@ -305,17 +281,11 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
|
||||
private void fillCnt(List<DataSetSchemaResp> dataSetSchemaResps, List<ItemUseResp> statInfos) {
|
||||
|
||||
Map<String, ItemUseResp> typeIdAndStatPair =
|
||||
statInfos.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
itemUseInfo ->
|
||||
itemUseInfo.getType()
|
||||
+ AT_SYMBOL
|
||||
+ AT_SYMBOL
|
||||
+ itemUseInfo.getBizName(),
|
||||
itemUseInfo -> itemUseInfo,
|
||||
(item1, item2) -> item1));
|
||||
Map<String, ItemUseResp> typeIdAndStatPair = statInfos.stream()
|
||||
.collect(Collectors.toMap(
|
||||
itemUseInfo -> itemUseInfo.getType() + AT_SYMBOL + AT_SYMBOL
|
||||
+ itemUseInfo.getBizName(),
|
||||
itemUseInfo -> itemUseInfo, (item1, item2) -> item1));
|
||||
log.debug("typeIdAndStatPair:{}", typeIdAndStatPair);
|
||||
for (DataSetSchemaResp dataSetSchemaResp : dataSetSchemaResps) {
|
||||
fillDimCnt(dataSetSchemaResp, typeIdAndStatPair);
|
||||
@@ -323,49 +293,39 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
}
|
||||
}
|
||||
|
||||
private void fillMetricCnt(
|
||||
DataSetSchemaResp dataSetSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
|
||||
private void fillMetricCnt(DataSetSchemaResp dataSetSchemaResp,
|
||||
Map<String, ItemUseResp> typeIdAndStatPair) {
|
||||
List<MetricSchemaResp> metrics = dataSetSchemaResp.getMetrics();
|
||||
if (CollectionUtils.isEmpty(dataSetSchemaResp.getMetrics())) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!CollectionUtils.isEmpty(metrics)) {
|
||||
metrics.stream()
|
||||
.forEach(
|
||||
metric -> {
|
||||
String key =
|
||||
TypeEnums.METRIC.name().toLowerCase()
|
||||
+ AT_SYMBOL
|
||||
+ AT_SYMBOL
|
||||
+ metric.getBizName();
|
||||
if (typeIdAndStatPair.containsKey(key)) {
|
||||
metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
|
||||
}
|
||||
});
|
||||
metrics.stream().forEach(metric -> {
|
||||
String key = TypeEnums.METRIC.name().toLowerCase() + AT_SYMBOL + AT_SYMBOL
|
||||
+ metric.getBizName();
|
||||
if (typeIdAndStatPair.containsKey(key)) {
|
||||
metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
|
||||
}
|
||||
});
|
||||
}
|
||||
dataSetSchemaResp.setMetrics(metrics);
|
||||
}
|
||||
|
||||
private void fillDimCnt(
|
||||
DataSetSchemaResp dataSetSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
|
||||
private void fillDimCnt(DataSetSchemaResp dataSetSchemaResp,
|
||||
Map<String, ItemUseResp> typeIdAndStatPair) {
|
||||
List<DimSchemaResp> dimensions = dataSetSchemaResp.getDimensions();
|
||||
if (CollectionUtils.isEmpty(dataSetSchemaResp.getDimensions())) {
|
||||
return;
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(dimensions)) {
|
||||
dimensions.stream()
|
||||
.forEach(
|
||||
dim -> {
|
||||
String key =
|
||||
TypeEnums.DIMENSION.name().toLowerCase()
|
||||
+ AT_SYMBOL
|
||||
+ AT_SYMBOL
|
||||
+ dim.getBizName();
|
||||
if (typeIdAndStatPair.containsKey(key)) {
|
||||
dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
|
||||
}
|
||||
});
|
||||
dimensions.stream().forEach(dim -> {
|
||||
String key = TypeEnums.DIMENSION.name().toLowerCase() + AT_SYMBOL + AT_SYMBOL
|
||||
+ dim.getBizName();
|
||||
if (typeIdAndStatPair.containsKey(key)) {
|
||||
dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
|
||||
}
|
||||
});
|
||||
}
|
||||
dataSetSchemaResp.setDimensions(dimensions);
|
||||
}
|
||||
@@ -404,11 +364,9 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
public List<ModelResp> getModelList(List<Long> modelIds) {
|
||||
List<ModelResp> modelRespList = new ArrayList<>();
|
||||
if (!org.apache.commons.collections.CollectionUtils.isEmpty(modelIds)) {
|
||||
modelIds.stream()
|
||||
.forEach(
|
||||
m -> {
|
||||
modelRespList.add(modelService.getModel(m));
|
||||
});
|
||||
modelIds.stream().forEach(m -> {
|
||||
modelRespList.add(modelService.getModel(m));
|
||||
});
|
||||
}
|
||||
return modelRespList;
|
||||
}
|
||||
@@ -434,21 +392,14 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
} else if (!CollectionUtils.isEmpty(schemaFilterReq.getModelIds())) {
|
||||
List<ModelSchemaResp> modelSchemaResps =
|
||||
fetchModelSchemaResps(schemaFilterReq.getModelIds());
|
||||
semanticSchemaResp.setMetrics(
|
||||
modelSchemaResps.stream()
|
||||
.map(ModelSchemaResp::getMetrics)
|
||||
.flatMap(Collection::stream)
|
||||
.collect(Collectors.toList()));
|
||||
semanticSchemaResp.setDimensions(
|
||||
modelSchemaResps.stream()
|
||||
.map(ModelSchemaResp::getDimensions)
|
||||
.flatMap(Collection::stream)
|
||||
.collect(Collectors.toList()));
|
||||
semanticSchemaResp.setModelRelas(
|
||||
modelSchemaResps.stream()
|
||||
.map(ModelSchemaResp::getModelRelas)
|
||||
.flatMap(Collection::stream)
|
||||
.collect(Collectors.toList()));
|
||||
semanticSchemaResp.setMetrics(modelSchemaResps.stream().map(ModelSchemaResp::getMetrics)
|
||||
.flatMap(Collection::stream).collect(Collectors.toList()));
|
||||
semanticSchemaResp
|
||||
.setDimensions(modelSchemaResps.stream().map(ModelSchemaResp::getDimensions)
|
||||
.flatMap(Collection::stream).collect(Collectors.toList()));
|
||||
semanticSchemaResp
|
||||
.setModelRelas(modelSchemaResps.stream().map(ModelSchemaResp::getModelRelas)
|
||||
.flatMap(Collection::stream).collect(Collectors.toList()));
|
||||
semanticSchemaResp.setModelResps(
|
||||
modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
|
||||
semanticSchemaResp.setSchemaType(SchemaType.MODEL);
|
||||
@@ -485,13 +436,11 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
@Override
|
||||
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
|
||||
if (itemUseReq.getCacheEnable()) {
|
||||
return itemUseCache.get(
|
||||
JsonUtil.toString(itemUseReq),
|
||||
() -> {
|
||||
List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
|
||||
itemUseCache.put(JsonUtil.toString(itemUseReq), data);
|
||||
return data;
|
||||
});
|
||||
return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> {
|
||||
List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
|
||||
itemUseCache.put(JsonUtil.toString(itemUseReq), data);
|
||||
return data;
|
||||
});
|
||||
}
|
||||
return statUtils.getStatInfo(itemUseReq);
|
||||
}
|
||||
@@ -499,28 +448,17 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
@Override
|
||||
public List<ItemResp> getDomainDataSetTree() {
|
||||
List<DomainResp> domainResps = domainService.getDomainList();
|
||||
List<ItemResp> itemResps =
|
||||
domainResps.stream()
|
||||
.map(
|
||||
domain ->
|
||||
new ItemResp(
|
||||
domain.getId(),
|
||||
domain.getParentId(),
|
||||
domain.getName(),
|
||||
TypeEnums.DOMAIN))
|
||||
.collect(Collectors.toList());
|
||||
List<ItemResp> itemResps = domainResps.stream().map(domain -> new ItemResp(domain.getId(),
|
||||
domain.getParentId(), domain.getName(), TypeEnums.DOMAIN))
|
||||
.collect(Collectors.toList());
|
||||
Map<Long, ItemResp> itemRespMap =
|
||||
itemResps.stream().collect(Collectors.toMap(ItemResp::getId, item -> item));
|
||||
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(new MetaFilter());
|
||||
for (DataSetResp dataSetResp : dataSetResps) {
|
||||
ItemResp itemResp = itemRespMap.get(dataSetResp.getDomainId());
|
||||
if (itemResp != null) {
|
||||
ItemResp dataSet =
|
||||
new ItemResp(
|
||||
dataSetResp.getId(),
|
||||
dataSetResp.getDomainId(),
|
||||
dataSetResp.getName(),
|
||||
TypeEnums.DATASET);
|
||||
ItemResp dataSet = new ItemResp(dataSetResp.getId(), dataSetResp.getDomainId(),
|
||||
dataSetResp.getName(), TypeEnums.DATASET);
|
||||
itemResp.getChildren().add(dataSet);
|
||||
}
|
||||
}
|
||||
@@ -528,10 +466,8 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
}
|
||||
|
||||
private void fillStaticInfo(List<DataSetSchemaResp> dataSetSchemaResps) {
|
||||
List<Long> dataSetIds =
|
||||
dataSetSchemaResps.stream()
|
||||
.map(DataSetSchemaResp::getId)
|
||||
.collect(Collectors.toList());
|
||||
List<Long> dataSetIds = dataSetSchemaResps.stream().map(DataSetSchemaResp::getId)
|
||||
.collect(Collectors.toList());
|
||||
ItemUseReq itemUseReq = new ItemUseReq();
|
||||
itemUseReq.setModelIds(dataSetIds);
|
||||
|
||||
@@ -567,11 +503,9 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getSchemaYamlTpl(
|
||||
SemanticSchemaResp semanticSchemaResp,
|
||||
public void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
|
||||
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
|
||||
List<DataModelYamlTpl> dataModelYamlTplList,
|
||||
List<MetricYamlTpl> metricYamlTplList,
|
||||
List<DataModelYamlTpl> dataModelYamlTplList, List<MetricYamlTpl> metricYamlTplList,
|
||||
Map<Long, String> modelIdName) {
|
||||
|
||||
List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
|
||||
@@ -587,15 +521,10 @@ public class SchemaServiceImpl implements SchemaService {
|
||||
if (!dimensionYamlMap.containsKey(modelResp.getBizName())) {
|
||||
dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>());
|
||||
}
|
||||
List<DimensionResp> dimensionRespList =
|
||||
dimensionResps.stream()
|
||||
.filter(
|
||||
d ->
|
||||
d.getModelBizName()
|
||||
.equalsIgnoreCase(modelResp.getBizName()))
|
||||
.collect(Collectors.toList());
|
||||
dimensionYamlMap
|
||||
.get(modelResp.getBizName())
|
||||
List<DimensionResp> dimensionRespList = dimensionResps.stream()
|
||||
.filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName()))
|
||||
.collect(Collectors.toList());
|
||||
dimensionYamlMap.get(modelResp.getBizName())
|
||||
.addAll(DimensionYamlManager.convert2DimensionYaml(dimensionRespList));
|
||||
}
|
||||
List<MetricResp> metricResps = new ArrayList<>(semanticSchemaResp.getMetrics());
|
||||
|
||||
@@ -58,13 +58,9 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
private final TagObjectService tagObjectService;
|
||||
private final DomainService domainService;
|
||||
|
||||
public TagMetaServiceImpl(
|
||||
TagRepository tagRepository,
|
||||
ModelService modelService,
|
||||
CollectService collectService,
|
||||
@Lazy DimensionService dimensionService,
|
||||
@Lazy MetricService metricService,
|
||||
TagObjectService tagObjectService,
|
||||
public TagMetaServiceImpl(TagRepository tagRepository, ModelService modelService,
|
||||
CollectService collectService, @Lazy DimensionService dimensionService,
|
||||
@Lazy MetricService metricService, TagObjectService tagObjectService,
|
||||
DomainService domainService) {
|
||||
this.tagRepository = tagRepository;
|
||||
this.modelService = modelService;
|
||||
@@ -162,11 +158,8 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
if (Objects.nonNull(tagMarketPageReq.getTagObjectId())) {
|
||||
modelRespList =
|
||||
modelRespList.stream()
|
||||
.filter(
|
||||
modelResp ->
|
||||
tagMarketPageReq
|
||||
.getTagObjectId()
|
||||
.equals(modelResp.getTagObjectId()))
|
||||
.filter(modelResp -> tagMarketPageReq.getTagObjectId()
|
||||
.equals(modelResp.getTagObjectId()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
if (CollectionUtils.isEmpty(modelRespList)) {
|
||||
@@ -179,15 +172,9 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
BeanUtils.copyProperties(tagMarketPageReq, tagFilter);
|
||||
List<CollectDO> collectList = collectService.getCollectionList(user.getName());
|
||||
if (tagMarketPageReq.isHasCollect()) {
|
||||
List<Long> collectIds =
|
||||
collectList.stream()
|
||||
.filter(
|
||||
collectDO ->
|
||||
SchemaElementType.TAG
|
||||
.name()
|
||||
.equalsIgnoreCase(collectDO.getType()))
|
||||
.map(CollectDO::getCollectId)
|
||||
.collect(Collectors.toList());
|
||||
List<Long> collectIds = collectList.stream().filter(
|
||||
collectDO -> SchemaElementType.TAG.name().equalsIgnoreCase(collectDO.getType()))
|
||||
.map(CollectDO::getCollectId).collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(collectIds)) {
|
||||
tagFilter.setIds(Lists.newArrayList(-1L));
|
||||
} else {
|
||||
@@ -217,22 +204,14 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
if (CollectionUtils.isEmpty(tagObjects)) {
|
||||
return;
|
||||
}
|
||||
Map<Long, TagObjectResp> tagObjectMap =
|
||||
tagObjects.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
TagObjectResp::getId,
|
||||
tagObject -> tagObject,
|
||||
(v1, v2) -> v2));
|
||||
Map<Long, TagObjectResp> tagObjectMap = tagObjects.stream().collect(
|
||||
Collectors.toMap(TagObjectResp::getId, tagObject -> tagObject, (v1, v2) -> v2));
|
||||
if (CollectionUtils.isNotEmpty(tagRespList)) {
|
||||
tagRespList.stream()
|
||||
.forEach(
|
||||
tagResp -> {
|
||||
if (tagObjectMap.containsKey(tagResp.getTagObjectId())) {
|
||||
tagResp.setTagObjectName(
|
||||
tagObjectMap.get(tagResp.getTagObjectId()).getName());
|
||||
}
|
||||
});
|
||||
tagRespList.stream().forEach(tagResp -> {
|
||||
if (tagObjectMap.containsKey(tagResp.getTagObjectId())) {
|
||||
tagResp.setTagObjectName(tagObjectMap.get(tagResp.getTagObjectId()).getName());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,20 +225,14 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
}
|
||||
|
||||
private void fillDomainInfo(List<TagResp> tagRespList) {
|
||||
Map<Long, DomainResp> domainMap =
|
||||
domainService.getDomainList().stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
DomainResp::getId, domain -> domain, (v1, v2) -> v2));
|
||||
Map<Long, DomainResp> domainMap = domainService.getDomainList().stream()
|
||||
.collect(Collectors.toMap(DomainResp::getId, domain -> domain, (v1, v2) -> v2));
|
||||
if (CollectionUtils.isNotEmpty(tagRespList) && Objects.nonNull(domainMap)) {
|
||||
tagRespList.stream()
|
||||
.forEach(
|
||||
tagResp -> {
|
||||
if (domainMap.containsKey(tagResp.getDomainId())) {
|
||||
tagResp.setDomainName(
|
||||
domainMap.get(tagResp.getDomainId()).getName());
|
||||
}
|
||||
});
|
||||
tagRespList.stream().forEach(tagResp -> {
|
||||
if (domainMap.containsKey(tagResp.getDomainId())) {
|
||||
tagResp.setDomainName(domainMap.get(tagResp.getDomainId()).getName());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,31 +294,21 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
tagRespList.stream().map(TagResp::getModelId).collect(Collectors.toList());
|
||||
ModelFilter modelFilter = new ModelFilter(false, modelIds);
|
||||
Map<Long, ModelResp> modelIdAndRespMap = modelService.getModelMap(modelFilter);
|
||||
tagRespList.stream()
|
||||
.forEach(
|
||||
tagResp -> {
|
||||
if (Objects.nonNull(modelIdAndRespMap)
|
||||
&& modelIdAndRespMap.containsKey(tagResp.getModelId())) {
|
||||
tagResp.setModelName(
|
||||
modelIdAndRespMap.get(tagResp.getModelId()).getName());
|
||||
tagResp.setDomainId(
|
||||
modelIdAndRespMap.get(tagResp.getModelId()).getDomainId());
|
||||
tagResp.setTagObjectId(
|
||||
modelIdAndRespMap
|
||||
.get(tagResp.getModelId())
|
||||
.getTagObjectId());
|
||||
}
|
||||
});
|
||||
tagRespList.stream().forEach(tagResp -> {
|
||||
if (Objects.nonNull(modelIdAndRespMap)
|
||||
&& modelIdAndRespMap.containsKey(tagResp.getModelId())) {
|
||||
tagResp.setModelName(modelIdAndRespMap.get(tagResp.getModelId()).getName());
|
||||
tagResp.setDomainId(modelIdAndRespMap.get(tagResp.getModelId()).getDomainId());
|
||||
tagResp.setTagObjectId(
|
||||
modelIdAndRespMap.get(tagResp.getModelId()).getTagObjectId());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private TagResp fillCollectAndAdminInfo(TagResp tagResp, User user) {
|
||||
List<Long> collectIds =
|
||||
collectService.getCollectionList(user.getName()).stream()
|
||||
.filter(
|
||||
collectDO ->
|
||||
TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
|
||||
.map(CollectDO::getCollectId)
|
||||
.collect(Collectors.toList());
|
||||
List<Long> collectIds = collectService.getCollectionList(user.getName()).stream()
|
||||
.filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
|
||||
.map(CollectDO::getCollectId).collect(Collectors.toList());
|
||||
if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagResp.getId())) {
|
||||
tagResp.setIsCollect(true);
|
||||
} else {
|
||||
@@ -357,24 +320,17 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
}
|
||||
|
||||
private TagResp fillCollectAndAdminInfo(List<TagResp> tagRespList, User user) {
|
||||
List<Long> collectIds =
|
||||
collectService.getCollectionList(user.getName()).stream()
|
||||
.filter(
|
||||
collectDO ->
|
||||
TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
|
||||
.map(CollectDO::getCollectId)
|
||||
.collect(Collectors.toList());
|
||||
List<Long> collectIds = collectService.getCollectionList(user.getName()).stream()
|
||||
.filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
|
||||
.map(CollectDO::getCollectId).collect(Collectors.toList());
|
||||
|
||||
tagRespList.stream()
|
||||
.forEach(
|
||||
tagResp -> {
|
||||
if (CollectionUtils.isNotEmpty(collectIds)
|
||||
&& collectIds.contains(tagResp.getId())) {
|
||||
tagResp.setIsCollect(true);
|
||||
} else {
|
||||
tagResp.setIsCollect(false);
|
||||
}
|
||||
});
|
||||
tagRespList.stream().forEach(tagResp -> {
|
||||
if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagResp.getId())) {
|
||||
tagResp.setIsCollect(true);
|
||||
} else {
|
||||
tagResp.setIsCollect(false);
|
||||
}
|
||||
});
|
||||
|
||||
fillAdminRes(tagRespList, user);
|
||||
return tagRespList.get(0);
|
||||
@@ -418,21 +374,17 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
ModelResp model = modelService.getModel(dimension.getModelId());
|
||||
if (Objects.isNull(model.getTagObjectId())) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"this dimension:%s is not supported to create tag,"
|
||||
+ " no related tag object",
|
||||
tagReq.getItemId()));
|
||||
String.format("this dimension:%s is not supported to create tag,"
|
||||
+ " no related tag object", tagReq.getItemId()));
|
||||
}
|
||||
}
|
||||
if (TagDefineType.METRIC.equals(tagReq.getTagDefineType())) {
|
||||
MetricResp metric = metricService.getMetric(tagReq.getItemId());
|
||||
ModelResp model = modelService.getModel(metric.getModelId());
|
||||
if (Objects.isNull(model.getTagObjectId())) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
"this metric:%s is not supported to create tag,"
|
||||
+ " no related tag object",
|
||||
tagReq.getItemId()));
|
||||
throw new RuntimeException(String.format(
|
||||
"this metric:%s is not supported to create tag," + " no related tag object",
|
||||
tagReq.getItemId()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -451,15 +403,11 @@ public class TagMetaServiceImpl implements TagMetaService {
|
||||
tagFilter.setItemIds(itemIds);
|
||||
Set<Long> dimensionItemSet =
|
||||
getTagDOList(tagFilter).stream().map(TagDO::getItemId).collect(Collectors.toSet());
|
||||
return itemIds.stream()
|
||||
.map(
|
||||
entry -> {
|
||||
TagItem tagItem = new TagItem();
|
||||
tagItem.setIsTag(
|
||||
Boolean.compare(dimensionItemSet.contains(entry), false));
|
||||
tagItem.setItemId(entry);
|
||||
return tagItem;
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
return itemIds.stream().map(entry -> {
|
||||
TagItem tagItem = new TagItem();
|
||||
tagItem.setIsTag(Boolean.compare(dimensionItemSet.contains(entry), false));
|
||||
tagItem.setItemId(entry);
|
||||
return tagItem;
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,9 +34,7 @@ public class TagObjectServiceImpl implements TagObjectService {
|
||||
private final ModelService modelService;
|
||||
private final TagMetaService tagMetaService;
|
||||
|
||||
public TagObjectServiceImpl(
|
||||
TagObjectRepository tagObjectRepository,
|
||||
ModelService modelService,
|
||||
public TagObjectServiceImpl(TagObjectRepository tagObjectRepository, ModelService modelService,
|
||||
@Lazy TagMetaService tagMetaService) {
|
||||
this.tagObjectRepository = tagObjectRepository;
|
||||
this.modelService = modelService;
|
||||
@@ -66,14 +64,9 @@ public class TagObjectServiceImpl implements TagObjectService {
|
||||
if (CollectionUtils.isEmpty(tagObjectRespList)) {
|
||||
return;
|
||||
}
|
||||
tagObjectRespList =
|
||||
tagObjectRespList.stream()
|
||||
.filter(
|
||||
tagObjectResp ->
|
||||
StatusEnum.ONLINE
|
||||
.getCode()
|
||||
.equals(tagObjectResp.getStatus()))
|
||||
.collect(Collectors.toList());
|
||||
tagObjectRespList = tagObjectRespList.stream().filter(
|
||||
tagObjectResp -> StatusEnum.ONLINE.getCode().equals(tagObjectResp.getStatus()))
|
||||
.collect(Collectors.toList());
|
||||
for (TagObjectResp tagObject : tagObjectRespList) {
|
||||
if (tagObject.getBizName().equalsIgnoreCase(tagObjectReq.getBizName())) {
|
||||
throw new Exception(
|
||||
@@ -128,9 +121,8 @@ public class TagObjectServiceImpl implements TagObjectService {
|
||||
if (!CollectionUtils.isEmpty(allModelByDomainIds)) {
|
||||
List<Long> modelIds =
|
||||
allModelByDomainIds.stream().map(ModelResp::getId).collect(Collectors.toList());
|
||||
throw new Exception(
|
||||
"delete operation is not supported at the moment. related modelIds:"
|
||||
+ modelIds);
|
||||
throw new Exception("delete operation is not supported at the moment. related modelIds:"
|
||||
+ modelIds);
|
||||
}
|
||||
TagFilterPageReq tagMarketPageReq = new TagFilterPageReq();
|
||||
tagMarketPageReq.setTagObjectId(tagObjectDO.getId());
|
||||
@@ -173,9 +165,8 @@ public class TagObjectServiceImpl implements TagObjectService {
|
||||
List<TagObjectDO> tagObjectDOList = tagObjectRepository.query(filter);
|
||||
List<TagObjectResp> tagObjectRespList =
|
||||
TagObjectConverter.convert2RespList(tagObjectDOList);
|
||||
Map<Long, TagObjectResp> map =
|
||||
tagObjectRespList.stream()
|
||||
.collect(Collectors.toMap(TagObjectResp::getId, a -> a, (k1, k2) -> k1));
|
||||
Map<Long, TagObjectResp> map = tagObjectRespList.stream()
|
||||
.collect(Collectors.toMap(TagObjectResp::getId, a -> a, (k1, k2) -> k1));
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,11 +52,8 @@ public class TagQueryServiceImpl implements TagQueryService {
|
||||
private final ModelService modelService;
|
||||
private final SqlGenerateUtils sqlGenerateUtils;
|
||||
|
||||
public TagQueryServiceImpl(
|
||||
TagMetaService tagMetaService,
|
||||
SemanticLayerService queryService,
|
||||
ModelService modelService,
|
||||
SqlGenerateUtils sqlGenerateUtils) {
|
||||
public TagQueryServiceImpl(TagMetaService tagMetaService, SemanticLayerService queryService,
|
||||
ModelService modelService, SqlGenerateUtils sqlGenerateUtils) {
|
||||
this.tagMetaService = tagMetaService;
|
||||
this.queryService = queryService;
|
||||
this.modelService = modelService;
|
||||
@@ -88,9 +85,8 @@ public class TagQueryServiceImpl implements TagQueryService {
|
||||
private void checkTag(TagResp tag) throws Exception {
|
||||
if (Objects.nonNull(tag)
|
||||
&& TagDefineType.METRIC.name().equalsIgnoreCase(tag.getTagDefineType())) {
|
||||
throw new Exception(
|
||||
"do not support value distribution query for tag (from metric): "
|
||||
+ tag.getBizName());
|
||||
throw new Exception("do not support value distribution query for tag (from metric): "
|
||||
+ tag.getBizName());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,16 +117,12 @@ public class TagQueryServiceImpl implements TagQueryService {
|
||||
return LocalDate.now().plusDays(-dayBefore).format(formatter);
|
||||
}
|
||||
|
||||
private String queryTagDateFromDbBySql(
|
||||
Dim dim, TagResp tag, ItemValueReq itemValueReq, User user) {
|
||||
private String queryTagDateFromDbBySql(Dim dim, TagResp tag, ItemValueReq itemValueReq,
|
||||
User user) {
|
||||
|
||||
String sqlPattern = "select max(%s) as %s from tbl where %s is not null";
|
||||
String sql =
|
||||
String.format(
|
||||
sqlPattern,
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
maxDateAlias,
|
||||
tag.getBizName());
|
||||
String sql = String.format(sqlPattern, TimeDimensionEnum.DAY.getName(), maxDateAlias,
|
||||
tag.getBizName());
|
||||
|
||||
// 添加时间过滤信息
|
||||
log.info("[queryTagDateFromDbBySql] calculate the maximum time start");
|
||||
@@ -143,22 +135,13 @@ public class TagQueryServiceImpl implements TagQueryService {
|
||||
if (StringUtils.isEmpty(dateFormat)) {
|
||||
dateFormat = itemValueDateFormat;
|
||||
}
|
||||
String start =
|
||||
LocalDate.now()
|
||||
.minusDays(itemValueReq.getDateConf().getUnit())
|
||||
.format(DateTimeFormatter.ofPattern(dateFormat));
|
||||
String end =
|
||||
LocalDate.now()
|
||||
.minusDays(0)
|
||||
.format(DateTimeFormatter.ofPattern(dateFormat));
|
||||
sql =
|
||||
sql
|
||||
+ String.format(
|
||||
" and ( %s > '%s' and %s <= '%s' )",
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
start,
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
end);
|
||||
String start = LocalDate.now().minusDays(itemValueReq.getDateConf().getUnit())
|
||||
.format(DateTimeFormatter.ofPattern(dateFormat));
|
||||
String end = LocalDate.now().minusDays(0)
|
||||
.format(DateTimeFormatter.ofPattern(dateFormat));
|
||||
sql = sql + String.format(" and ( %s > '%s' and %s <= '%s' )",
|
||||
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(),
|
||||
end);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -219,42 +202,27 @@ public class TagQueryServiceImpl implements TagQueryService {
|
||||
return " and " + dateWhereClause;
|
||||
}
|
||||
|
||||
private void fillTagValueInfo(
|
||||
ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp, Long totalCount) {
|
||||
private void fillTagValueInfo(ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp,
|
||||
Long totalCount) {
|
||||
List<ValueDistribution> valueDistributionList = new ArrayList<>();
|
||||
List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
|
||||
if (!CollectionUtils.isEmpty(resultList)) {
|
||||
resultList.stream()
|
||||
.forEach(
|
||||
line -> {
|
||||
Object tagValue = line.get(itemValueResp.getBizName());
|
||||
Long tagValueCount =
|
||||
Long.parseLong(line.get(tagValueAlias).toString());
|
||||
valueDistributionList.add(
|
||||
ValueDistribution.builder()
|
||||
.totalCount(totalCount)
|
||||
.valueMap(tagValue)
|
||||
.valueCount(tagValueCount)
|
||||
.ratio(1.0 * tagValueCount / totalCount)
|
||||
.build());
|
||||
});
|
||||
resultList.stream().forEach(line -> {
|
||||
Object tagValue = line.get(itemValueResp.getBizName());
|
||||
Long tagValueCount = Long.parseLong(line.get(tagValueAlias).toString());
|
||||
valueDistributionList.add(ValueDistribution.builder().totalCount(totalCount)
|
||||
.valueMap(tagValue).valueCount(tagValueCount)
|
||||
.ratio(1.0 * tagValueCount / totalCount).build());
|
||||
});
|
||||
}
|
||||
itemValueResp.setValueDistributionList(valueDistributionList);
|
||||
}
|
||||
|
||||
private QuerySqlReq generateReq(TagResp tag, ItemValueReq itemValueReq) {
|
||||
String sqlPattern =
|
||||
"select %s, count(1) as %s from tbl where %s is not null %s "
|
||||
+ "group by %s order by %s desc";
|
||||
String sql =
|
||||
String.format(
|
||||
sqlPattern,
|
||||
tag.getBizName(),
|
||||
tagValueAlias,
|
||||
tag.getBizName(),
|
||||
getDateFilter(itemValueReq),
|
||||
tag.getBizName(),
|
||||
tag.getBizName());
|
||||
String sqlPattern = "select %s, count(1) as %s from tbl where %s is not null %s "
|
||||
+ "group by %s order by %s desc";
|
||||
String sql = String.format(sqlPattern, tag.getBizName(), tagValueAlias, tag.getBizName(),
|
||||
getDateFilter(itemValueReq), tag.getBizName(), tag.getBizName());
|
||||
|
||||
Set<Long> modelIds = new HashSet<>();
|
||||
modelIds.add(tag.getModelId());
|
||||
|
||||
@@ -56,15 +56,8 @@ public class TermServiceImpl extends ServiceImpl<TermMapper, TermDO> implements
|
||||
QueryWrapper<TermDO> queryWrapper = new QueryWrapper<>();
|
||||
queryWrapper.lambda().eq(TermDO::getDomainId, domainId);
|
||||
if (StringUtils.isNotBlank(queryKey)) {
|
||||
queryWrapper
|
||||
.lambda()
|
||||
.and(
|
||||
i ->
|
||||
i.like(TermDO::getName, queryKey)
|
||||
.or()
|
||||
.like(TermDO::getDescription, queryKey)
|
||||
.or()
|
||||
.like(TermDO::getAlias, queryKey));
|
||||
queryWrapper.lambda().and(i -> i.like(TermDO::getName, queryKey).or()
|
||||
.like(TermDO::getDescription, queryKey).or().like(TermDO::getAlias, queryKey));
|
||||
}
|
||||
List<TermDO> termDOS = list(queryWrapper);
|
||||
return termDOS.stream().map(this::convert).collect(Collectors.toList());
|
||||
@@ -78,8 +71,7 @@ public class TermServiceImpl extends ServiceImpl<TermMapper, TermDO> implements
|
||||
QueryWrapper<TermDO> queryWrapper = new QueryWrapper<>();
|
||||
queryWrapper.lambda().in(TermDO::getDomainId, domainIds);
|
||||
List<TermDO> list = list(queryWrapper);
|
||||
return list.stream()
|
||||
.map(this::convert)
|
||||
return list.stream().map(this::convert)
|
||||
.collect(Collectors.groupingBy(TermResp::getDomainId));
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,8 @@ import org.springframework.stereotype.Component;
|
||||
@Order(2)
|
||||
public class DictionaryReloadTask implements CommandLineRunner {
|
||||
|
||||
@Autowired private DictWordService dictWordService;
|
||||
@Autowired
|
||||
private DictWordService dictWordService;
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
|
||||
@@ -25,13 +25,17 @@ import java.util.List;
|
||||
@Order(2)
|
||||
public class MetaEmbeddingTask implements CommandLineRunner {
|
||||
|
||||
@Autowired private EmbeddingService embeddingService;
|
||||
@Autowired
|
||||
private EmbeddingService embeddingService;
|
||||
|
||||
@Autowired private EmbeddingConfig embeddingConfig;
|
||||
@Autowired
|
||||
private EmbeddingConfig embeddingConfig;
|
||||
|
||||
@Autowired private MetricService metricService;
|
||||
@Autowired
|
||||
private MetricService metricService;
|
||||
|
||||
@Autowired private DimensionService dimensionService;
|
||||
@Autowired
|
||||
private DimensionService dimensionService;
|
||||
|
||||
@PreDestroy
|
||||
public void onShutdown() {
|
||||
@@ -62,13 +66,11 @@ public class MetaEmbeddingTask implements CommandLineRunner {
|
||||
try {
|
||||
List<DataItem> metricDataItems = metricService.getDataEvent().getDataItems();
|
||||
|
||||
embeddingService.addQuery(
|
||||
embeddingConfig.getMetaCollectionName(),
|
||||
embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
|
||||
TextSegmentConvert.convertToEmbedding(metricDataItems));
|
||||
|
||||
List<DataItem> dimensionDataItems = dimensionService.getDataEvent().getDataItems();
|
||||
embeddingService.addQuery(
|
||||
embeddingConfig.getMetaCollectionName(),
|
||||
embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
|
||||
TextSegmentConvert.convertToEmbedding(dimensionDataItems));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to reload meta embedding.", e);
|
||||
|
||||
@@ -23,38 +23,32 @@ public class AliasGenerateHelper {
|
||||
|
||||
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
|
||||
|
||||
private static final String NAME_ALIAS_INSTRUCTION =
|
||||
""
|
||||
+ "\n#Role: You are a professional data analyst specializing in metrics and dimensions."
|
||||
+ "\n#Task: You will be provided with metadata about a metric or dimension, please help "
|
||||
+ "generate a few aliases in the same language as its `fieldName`."
|
||||
+ "\n#Rules:"
|
||||
+ "1. Please do not generate aliases like xxx1, xxx2, xxx3."
|
||||
+ "2. Please do not generate aliases that are the same as the original names of metrics/dimensions."
|
||||
+ "3. Please pay attention to the quality of the generated aliases and "
|
||||
+ "avoid creating aliases that look like test data."
|
||||
+ "4. Please output as a json string array."
|
||||
+ "\n#Metadata: {'table':'{{table}}', 'name':'{{name}}', 'type':'{{type}}', "
|
||||
+ "'field':'field', 'description':'{{desc}}'}"
|
||||
+ "\n#Output:";
|
||||
private static final String NAME_ALIAS_INSTRUCTION = ""
|
||||
+ "\n#Role: You are a professional data analyst specializing in metrics and dimensions."
|
||||
+ "\n#Task: You will be provided with metadata about a metric or dimension, please help "
|
||||
+ "generate a few aliases in the same language as its `fieldName`." + "\n#Rules:"
|
||||
+ "1. Please do not generate aliases like xxx1, xxx2, xxx3."
|
||||
+ "2. Please do not generate aliases that are the same as the original names of metrics/dimensions."
|
||||
+ "3. Please pay attention to the quality of the generated aliases and "
|
||||
+ "avoid creating aliases that look like test data."
|
||||
+ "4. Please output as a json string array."
|
||||
+ "\n#Metadata: {'table':'{{table}}', 'name':'{{name}}', 'type':'{{type}}', "
|
||||
+ "'field':'field', 'description':'{{desc}}'}" + "\n#Output:";
|
||||
|
||||
private static final String VALUE_ALIAS_INSTRUCTION =
|
||||
""
|
||||
+ "\n#Role: You are a professional data analyst."
|
||||
"" + "\n#Role: You are a professional data analyst."
|
||||
+ "\n#Task: You will be provided with a json array of dimension values,"
|
||||
+ "please help generate a few aliases for each value."
|
||||
+ "\n#Rule:"
|
||||
+ "please help generate a few aliases for each value." + "\n#Rule:"
|
||||
+ "1. ALWAYS output json array for each value."
|
||||
+ "2. The aliases should be in the same language as its original value."
|
||||
+ "\n#Exemplar:"
|
||||
+ "Values: [\\\"qq_music\\\",\\\"kugou_music\\\"], "
|
||||
+ "\n#Exemplar:" + "Values: [\\\"qq_music\\\",\\\"kugou_music\\\"], "
|
||||
+ "Output: {\\\"tran\\\":[\\\"qq音乐\\\",\\\"酷狗音乐\\\"],"
|
||||
+ " \\\"alias\\\":{\\\"qq_music\\\":[\\\"q音\\\",\\\"qq音乐\\\"],"
|
||||
+ " \\\"kugou_music\\\":[\\\"kugou\\\",\\\"酷狗\\\"]}}"
|
||||
+ "\nValues: {{values}}, Output:";
|
||||
|
||||
public String generateAlias(
|
||||
String mockType, String name, String bizName, String table, String desc) {
|
||||
public String generateAlias(String mockType, String name, String bizName, String table,
|
||||
String desc) {
|
||||
Map<String, Object> variable = new HashMap<>();
|
||||
variable.put("table", table);
|
||||
variable.put("name", name);
|
||||
@@ -88,8 +82,8 @@ public class AliasGenerateHelper {
|
||||
return response.content().text();
|
||||
}
|
||||
|
||||
private static String extractString(
|
||||
String targetString, String left, String right, Boolean exclusionFlag) {
|
||||
private static String extractString(String targetString, String left, String right,
|
||||
Boolean exclusionFlag) {
|
||||
if (targetString == null || left == null || right == null || exclusionFlag == null) {
|
||||
return targetString;
|
||||
}
|
||||
@@ -139,19 +133,18 @@ public class AliasGenerateHelper {
|
||||
}
|
||||
}
|
||||
BoundaryPattern[] patterns = {
|
||||
// 不做任何匹配
|
||||
new BoundaryPattern(null, null, null),
|
||||
// ```{"name":"Alice","age":25,"city":"NewYork"}```
|
||||
new BoundaryPattern("```", "```", true),
|
||||
// ```json {"name":"Alice","age":25,"city":"NewYork"}```
|
||||
new BoundaryPattern("```json", "```", true),
|
||||
// ```JSON {"name":"Alice","age":25,"city":"NewYork"}```
|
||||
new BoundaryPattern("```JSON", "```", true),
|
||||
// {"name":"Alice","age":25,"city":"NewYork"}
|
||||
new BoundaryPattern("{", "}", false),
|
||||
// ["Alice", "Bob"]
|
||||
new BoundaryPattern("[", "]", false)
|
||||
};
|
||||
// 不做任何匹配
|
||||
new BoundaryPattern(null, null, null),
|
||||
// ```{"name":"Alice","age":25,"city":"NewYork"}```
|
||||
new BoundaryPattern("```", "```", true),
|
||||
// ```json {"name":"Alice","age":25,"city":"NewYork"}```
|
||||
new BoundaryPattern("```json", "```", true),
|
||||
// ```JSON {"name":"Alice","age":25,"city":"NewYork"}```
|
||||
new BoundaryPattern("```JSON", "```", true),
|
||||
// {"name":"Alice","age":25,"city":"NewYork"}
|
||||
new BoundaryPattern("{", "}", false),
|
||||
// ["Alice", "Bob"]
|
||||
new BoundaryPattern("[", "]", false)};
|
||||
for (BoundaryPattern pattern : patterns) {
|
||||
String extracted =
|
||||
extractString(aiMessage, pattern.left, pattern.right, pattern.exclusionFlag);
|
||||
|
||||
@@ -62,10 +62,8 @@ public class ChatWorkflowEngine {
|
||||
parseResult.setErrorMsg("No semantic queries can be parsed out.");
|
||||
queryCtx.setChatWorkflowState(ChatWorkflowState.FINISHED);
|
||||
} else {
|
||||
List<SemanticParseInfo> parseInfos =
|
||||
queryCtx.getCandidateQueries().stream()
|
||||
.map(SemanticQuery::getParseInfo)
|
||||
.collect(Collectors.toList());
|
||||
List<SemanticParseInfo> parseInfos = queryCtx.getCandidateQueries().stream()
|
||||
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
|
||||
parseResult.setSelectedParses(parseInfos);
|
||||
queryCtx.setChatWorkflowState(ChatWorkflowState.CORRECTING);
|
||||
}
|
||||
@@ -101,14 +99,11 @@ public class ChatWorkflowEngine {
|
||||
}
|
||||
|
||||
private void performParsing(ChatQueryContext queryCtx) {
|
||||
semanticParsers.forEach(
|
||||
parser -> {
|
||||
parser.parse(queryCtx);
|
||||
log.debug(
|
||||
"{} result:{}",
|
||||
parser.getClass().getSimpleName(),
|
||||
JsonUtil.toString(queryCtx));
|
||||
});
|
||||
semanticParsers.forEach(parser -> {
|
||||
parser.parse(queryCtx);
|
||||
log.debug("{} result:{}", parser.getClass().getSimpleName(),
|
||||
JsonUtil.toString(queryCtx));
|
||||
});
|
||||
}
|
||||
|
||||
private void performCorrecting(ChatQueryContext queryCtx) {
|
||||
@@ -126,45 +121,38 @@ public class ChatWorkflowEngine {
|
||||
}
|
||||
|
||||
private void performProcessing(ChatQueryContext queryCtx, ParseResp parseResult) {
|
||||
resultProcessors.forEach(
|
||||
processor -> {
|
||||
processor.process(parseResult, queryCtx);
|
||||
});
|
||||
resultProcessors.forEach(processor -> {
|
||||
processor.process(parseResult, queryCtx);
|
||||
});
|
||||
}
|
||||
|
||||
private void performTranslating(ChatQueryContext chatQueryContext) {
|
||||
List<SemanticParseInfo> semanticParseInfos =
|
||||
chatQueryContext.getCandidateQueries().stream()
|
||||
.map(SemanticQuery::getParseInfo)
|
||||
.collect(Collectors.toList());
|
||||
List<SemanticParseInfo> semanticParseInfos = chatQueryContext.getCandidateQueries().stream()
|
||||
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
|
||||
|
||||
semanticParseInfos.forEach(
|
||||
parseInfo -> {
|
||||
try {
|
||||
SemanticQuery semanticQuery =
|
||||
QueryManager.createQuery(parseInfo.getQueryMode());
|
||||
if (Objects.isNull(semanticQuery)) {
|
||||
return;
|
||||
}
|
||||
semanticQuery.setParseInfo(parseInfo);
|
||||
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
|
||||
SemanticLayerService queryService =
|
||||
ContextUtils.getBean(SemanticLayerService.class);
|
||||
SemanticTranslateResp explain =
|
||||
queryService.translate(
|
||||
semanticQueryReq, chatQueryContext.getUser());
|
||||
parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL());
|
||||
semanticParseInfos.forEach(parseInfo -> {
|
||||
try {
|
||||
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
|
||||
if (Objects.isNull(semanticQuery)) {
|
||||
return;
|
||||
}
|
||||
semanticQuery.setParseInfo(parseInfo);
|
||||
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
|
||||
SemanticLayerService queryService =
|
||||
ContextUtils.getBean(SemanticLayerService.class);
|
||||
SemanticTranslateResp explain =
|
||||
queryService.translate(semanticQueryReq, chatQueryContext.getUser());
|
||||
parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL());
|
||||
|
||||
keyPipelineLog.info(
|
||||
"SqlInfoProcessor results:\n"
|
||||
+ "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
|
||||
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getParsedS2SQL()),
|
||||
StringUtils.normalizeSpace(
|
||||
parseInfo.getSqlInfo().getCorrectedS2SQL()),
|
||||
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getQuerySQL()));
|
||||
} catch (Exception e) {
|
||||
log.warn("get sql info failed:{}", parseInfo, e);
|
||||
}
|
||||
});
|
||||
keyPipelineLog.info(
|
||||
"SqlInfoProcessor results:\n"
|
||||
+ "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
|
||||
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getParsedS2SQL()),
|
||||
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getCorrectedS2SQL()),
|
||||
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getQuerySQL()));
|
||||
} catch (Exception e) {
|
||||
log.warn("get sql info failed:{}", parseInfo, e);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,9 +29,7 @@ public class ClassConverter {
|
||||
private final DomainService domainService;
|
||||
private final TagObjectService tagObjectService;
|
||||
|
||||
public ClassConverter(
|
||||
ClassRepository classRepository,
|
||||
DomainService domainService,
|
||||
public ClassConverter(ClassRepository classRepository, DomainService domainService,
|
||||
TagObjectService tagObjectService) {
|
||||
this.classRepository = classRepository;
|
||||
this.domainService = domainService;
|
||||
@@ -57,16 +55,13 @@ public class ClassConverter {
|
||||
return convert2RespInternal(classDO, idAndDomain, classFullPathMap);
|
||||
}
|
||||
|
||||
private ClassResp convert2RespInternal(
|
||||
ClassDO classDO,
|
||||
Map<Long, DomainResp> idAndDomain,
|
||||
private ClassResp convert2RespInternal(ClassDO classDO, Map<Long, DomainResp> idAndDomain,
|
||||
Map<Long, String> classFullPathMap) {
|
||||
ClassResp classResp = new ClassResp();
|
||||
BeanUtils.copyProperties(classDO, classResp);
|
||||
|
||||
Long domainId = classResp.getDomainId();
|
||||
if (Objects.nonNull(idAndDomain)
|
||||
&& idAndDomain.containsKey(domainId)
|
||||
if (Objects.nonNull(idAndDomain) && idAndDomain.containsKey(domainId)
|
||||
&& Objects.nonNull(idAndDomain.get(domainId))) {
|
||||
classResp.setDomainName(idAndDomain.get(domainId).getName());
|
||||
}
|
||||
@@ -94,9 +89,8 @@ public class ClassConverter {
|
||||
public Map<Long, String> getClassFullPathMap() {
|
||||
Map<Long, String> classFullPathMap = new HashMap<>();
|
||||
List<ClassDO> classDOList = classRepository.getAllClassDOList();
|
||||
Map<Long, ClassDO> classDOMap =
|
||||
classDOList.stream()
|
||||
.collect(Collectors.toMap(ClassDO::getId, a -> a, (k1, k2) -> k1));
|
||||
Map<Long, ClassDO> classDOMap = classDOList.stream()
|
||||
.collect(Collectors.toMap(ClassDO::getId, a -> a, (k1, k2) -> k1));
|
||||
for (ClassDO classDO : classDOList) {
|
||||
final Long domainId = classDO.getId();
|
||||
StringBuilder fullPath = new StringBuilder(classDO.getBizName() + "/");
|
||||
|
||||
@@ -27,8 +27,7 @@ public class ComponentFactory {
|
||||
}
|
||||
|
||||
public static List<SchemaMapper> getSchemaMappers() {
|
||||
return CollectionUtils.isEmpty(schemaMappers)
|
||||
? init(SchemaMapper.class, schemaMappers)
|
||||
return CollectionUtils.isEmpty(schemaMappers) ? init(SchemaMapper.class, schemaMappers)
|
||||
: schemaMappers;
|
||||
}
|
||||
|
||||
@@ -49,15 +48,13 @@ public class ComponentFactory {
|
||||
}
|
||||
|
||||
private static <T> List<T> init(Class<T> factoryType, List list) {
|
||||
list.addAll(
|
||||
SpringFactoriesLoader.loadFactories(
|
||||
factoryType, Thread.currentThread().getContextClassLoader()));
|
||||
list.addAll(SpringFactoriesLoader.loadFactories(factoryType,
|
||||
Thread.currentThread().getContextClassLoader()));
|
||||
return list;
|
||||
}
|
||||
|
||||
private static <T> T init(Class<T> factoryType) {
|
||||
return SpringFactoriesLoader.loadFactories(
|
||||
factoryType, Thread.currentThread().getContextClassLoader())
|
||||
.get(0);
|
||||
return SpringFactoriesLoader
|
||||
.loadFactories(factoryType, Thread.currentThread().getContextClassLoader()).get(0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,15 +26,9 @@ public class DataSetSchemaBuilder {
|
||||
public static DataSetSchema build(DataSetSchemaResp resp) {
|
||||
DataSetSchema dataSetSchema = new DataSetSchema();
|
||||
dataSetSchema.setQueryConfig(resp.getQueryConfig());
|
||||
SchemaElement dataSet =
|
||||
SchemaElement.builder()
|
||||
.dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName())
|
||||
.id(resp.getId())
|
||||
.name(resp.getName())
|
||||
.bizName(resp.getBizName())
|
||||
.type(SchemaElementType.DATASET)
|
||||
.build();
|
||||
SchemaElement dataSet = SchemaElement.builder().dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName()).id(resp.getId()).name(resp.getName())
|
||||
.bizName(resp.getBizName()).type(SchemaElementType.DATASET).build();
|
||||
dataSetSchema.setDataSet(dataSet);
|
||||
dataSetSchema.setDatabaseType(resp.getDatabaseType());
|
||||
|
||||
@@ -68,21 +62,12 @@ public class DataSetSchemaBuilder {
|
||||
for (MetricSchemaResp metric : resp.getMetrics()) {
|
||||
List<String> alias = SchemaItem.getAliasList(metric.getAlias());
|
||||
if (metric.getIsTag() == 1) {
|
||||
SchemaElement tagToAdd =
|
||||
SchemaElement.builder()
|
||||
.dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName())
|
||||
.model(metric.getModelId())
|
||||
.id(metric.getId())
|
||||
.name(metric.getName())
|
||||
.bizName(metric.getBizName())
|
||||
.type(SchemaElementType.TAG)
|
||||
.useCnt(metric.getUseCnt())
|
||||
.alias(alias)
|
||||
.defaultAgg(metric.getDefaultAgg())
|
||||
.isTag(metric.getIsTag())
|
||||
.description(metric.getDescription())
|
||||
.build();
|
||||
SchemaElement tagToAdd = SchemaElement.builder().dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName()).model(metric.getModelId()).id(metric.getId())
|
||||
.name(metric.getName()).bizName(metric.getBizName())
|
||||
.type(SchemaElementType.TAG).useCnt(metric.getUseCnt()).alias(alias)
|
||||
.defaultAgg(metric.getDefaultAgg()).isTag(metric.getIsTag())
|
||||
.description(metric.getDescription()).build();
|
||||
tags.add(tagToAdd);
|
||||
}
|
||||
}
|
||||
@@ -103,21 +88,11 @@ public class DataSetSchemaBuilder {
|
||||
}
|
||||
}
|
||||
if (dim.getIsTag() == 1) {
|
||||
SchemaElement tagToAdd =
|
||||
SchemaElement.builder()
|
||||
.dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName())
|
||||
.model(dim.getModelId())
|
||||
.id(dim.getId())
|
||||
.name(dim.getName())
|
||||
.bizName(dim.getBizName())
|
||||
.type(SchemaElementType.TAG)
|
||||
.useCnt(dim.getUseCnt())
|
||||
.alias(alias)
|
||||
.schemaValueMaps(schemaValueMaps)
|
||||
.isTag(dim.getIsTag())
|
||||
.description(dim.getDescription())
|
||||
.build();
|
||||
SchemaElement tagToAdd = SchemaElement.builder().dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName()).model(dim.getModelId()).id(dim.getId())
|
||||
.name(dim.getName()).bizName(dim.getBizName()).type(SchemaElementType.TAG)
|
||||
.useCnt(dim.getUseCnt()).alias(alias).schemaValueMaps(schemaValueMaps)
|
||||
.isTag(dim.getIsTag()).description(dim.getDescription()).build();
|
||||
tags.add(tagToAdd);
|
||||
}
|
||||
}
|
||||
@@ -129,15 +104,9 @@ public class DataSetSchemaBuilder {
|
||||
if (Objects.isNull(dim)) {
|
||||
return null;
|
||||
}
|
||||
return SchemaElement.builder()
|
||||
.dataSetId(resp.getId())
|
||||
.model(dim.getModelId())
|
||||
.id(dim.getId())
|
||||
.name(dim.getName())
|
||||
.bizName(dim.getBizName())
|
||||
.type(SchemaElementType.ENTITY)
|
||||
.useCnt(dim.getUseCnt())
|
||||
.alias(dim.getEntityAlias())
|
||||
return SchemaElement.builder().dataSetId(resp.getId()).model(dim.getModelId())
|
||||
.id(dim.getId()).name(dim.getName()).bizName(dim.getBizName())
|
||||
.type(SchemaElementType.ENTITY).useCnt(dim.getUseCnt()).alias(dim.getEntityAlias())
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -154,21 +123,11 @@ public class DataSetSchemaBuilder {
|
||||
schemaValueMaps.add(schemaValueMap);
|
||||
}
|
||||
}
|
||||
SchemaElement dimToAdd =
|
||||
SchemaElement.builder()
|
||||
.dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName())
|
||||
.model(dim.getModelId())
|
||||
.id(dim.getId())
|
||||
.name(dim.getName())
|
||||
.bizName(dim.getBizName())
|
||||
.useCnt(dim.getUseCnt())
|
||||
.alias(alias)
|
||||
.schemaValueMaps(schemaValueMaps)
|
||||
.isTag(dim.getIsTag())
|
||||
.description(dim.getDescription())
|
||||
.type(SchemaElementType.DIMENSION)
|
||||
.build();
|
||||
SchemaElement dimToAdd = SchemaElement.builder().dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName()).model(dim.getModelId()).id(dim.getId())
|
||||
.name(dim.getName()).bizName(dim.getBizName()).useCnt(dim.getUseCnt())
|
||||
.alias(alias).schemaValueMaps(schemaValueMaps).isTag(dim.getIsTag())
|
||||
.description(dim.getDescription()).type(SchemaElementType.DIMENSION).build();
|
||||
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TYPE, dim.getType());
|
||||
|
||||
if (dim.isTimeDimension()) {
|
||||
@@ -196,22 +155,12 @@ public class DataSetSchemaBuilder {
|
||||
}
|
||||
}
|
||||
}
|
||||
SchemaElement dimValueToAdd =
|
||||
SchemaElement.builder()
|
||||
.dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName())
|
||||
.model(dim.getModelId())
|
||||
.id(dim.getId())
|
||||
.name(dim.getName())
|
||||
.bizName(dim.getBizName())
|
||||
.type(SchemaElementType.VALUE)
|
||||
.useCnt(dim.getUseCnt())
|
||||
.alias(
|
||||
new ArrayList<>(
|
||||
Arrays.asList(dimValueAlias.toArray(new String[0]))))
|
||||
.isTag(dim.getIsTag())
|
||||
.description(dim.getDescription())
|
||||
.build();
|
||||
SchemaElement dimValueToAdd = SchemaElement.builder().dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName()).model(dim.getModelId()).id(dim.getId())
|
||||
.name(dim.getName()).bizName(dim.getBizName()).type(SchemaElementType.VALUE)
|
||||
.useCnt(dim.getUseCnt())
|
||||
.alias(new ArrayList<>(Arrays.asList(dimValueAlias.toArray(new String[0]))))
|
||||
.isTag(dim.getIsTag()).description(dim.getDescription()).build();
|
||||
dimensionValues.add(dimValueToAdd);
|
||||
}
|
||||
return dimensionValues;
|
||||
@@ -223,23 +172,13 @@ public class DataSetSchemaBuilder {
|
||||
|
||||
List<String> alias = SchemaItem.getAliasList(metric.getAlias());
|
||||
|
||||
SchemaElement metricToAdd =
|
||||
SchemaElement.builder()
|
||||
.dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName())
|
||||
.model(metric.getModelId())
|
||||
.id(metric.getId())
|
||||
.name(metric.getName())
|
||||
.bizName(metric.getBizName())
|
||||
.type(SchemaElementType.METRIC)
|
||||
.useCnt(metric.getUseCnt())
|
||||
.alias(alias)
|
||||
.relatedSchemaElements(getRelateSchemaElement(metric))
|
||||
.defaultAgg(metric.getDefaultAgg())
|
||||
.dataFormatType(metric.getDataFormatType())
|
||||
.isTag(metric.getIsTag())
|
||||
.description(metric.getDescription())
|
||||
.build();
|
||||
SchemaElement metricToAdd = SchemaElement.builder().dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName()).model(metric.getModelId()).id(metric.getId())
|
||||
.name(metric.getName()).bizName(metric.getBizName())
|
||||
.type(SchemaElementType.METRIC).useCnt(metric.getUseCnt()).alias(alias)
|
||||
.relatedSchemaElements(getRelateSchemaElement(metric))
|
||||
.defaultAgg(metric.getDefaultAgg()).dataFormatType(metric.getDataFormatType())
|
||||
.isTag(metric.getIsTag()).description(metric.getDescription()).build();
|
||||
metrics.add(metricToAdd);
|
||||
}
|
||||
return metrics;
|
||||
@@ -250,18 +189,10 @@ public class DataSetSchemaBuilder {
|
||||
for (TermResp termResp : resp.getTermResps()) {
|
||||
List<String> alias = termResp.getAlias();
|
||||
SchemaElement metricToAdd =
|
||||
SchemaElement.builder()
|
||||
.dataSetId(resp.getId())
|
||||
.dataSetName(resp.getName())
|
||||
.model(-1L)
|
||||
.id(termResp.getId())
|
||||
.name(termResp.getName())
|
||||
.bizName(termResp.getName())
|
||||
.type(SchemaElementType.TERM)
|
||||
.useCnt(0L)
|
||||
.alias(alias)
|
||||
.description(termResp.getDescription())
|
||||
.build();
|
||||
SchemaElement.builder().dataSetId(resp.getId()).dataSetName(resp.getName())
|
||||
.model(-1L).id(termResp.getId()).name(termResp.getName())
|
||||
.bizName(termResp.getName()).type(SchemaElementType.TERM).useCnt(0L)
|
||||
.alias(alias).description(termResp.getDescription()).build();
|
||||
terms.add(metricToAdd);
|
||||
}
|
||||
return terms;
|
||||
@@ -274,26 +205,19 @@ public class DataSetSchemaBuilder {
|
||||
|| CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
return relateDimension.getDrillDownDimensions().stream()
|
||||
.map(
|
||||
dimension -> {
|
||||
RelatedSchemaElement relateSchemaElement = new RelatedSchemaElement();
|
||||
BeanUtils.copyProperties(dimension, relateSchemaElement);
|
||||
return relateSchemaElement;
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
return relateDimension.getDrillDownDimensions().stream().map(dimension -> {
|
||||
RelatedSchemaElement relateSchemaElement = new RelatedSchemaElement();
|
||||
BeanUtils.copyProperties(dimension, relateSchemaElement);
|
||||
return relateSchemaElement;
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private static void setDefaultTimeFormat(
|
||||
SchemaElement dimToAdd,
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams,
|
||||
String timeFormat) {
|
||||
if (null != dimensionTimeTypeParams
|
||||
&& TimeDimensionEnum.DAY
|
||||
.name()
|
||||
.equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
|
||||
dimToAdd.getExtInfo()
|
||||
.put(DimensionConstants.DIMENSION_TIME_FORMAT, DateUtils.DEFAULT_DATE_FORMAT);
|
||||
private static void setDefaultTimeFormat(SchemaElement dimToAdd,
|
||||
DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) {
|
||||
if (null != dimensionTimeTypeParams && TimeDimensionEnum.DAY.name()
|
||||
.equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
|
||||
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT,
|
||||
DateUtils.DEFAULT_DATE_FORMAT);
|
||||
} else {
|
||||
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
|
||||
}
|
||||
|
||||
@@ -90,11 +90,8 @@ public class DictUtils {
|
||||
private final ModelService modelService;
|
||||
private final TagMetaService tagMetaService;
|
||||
|
||||
public DictUtils(
|
||||
DimensionService dimensionService,
|
||||
MetricService metricService,
|
||||
SemanticLayerService queryService,
|
||||
ModelService modelService,
|
||||
public DictUtils(DimensionService dimensionService, MetricService metricService,
|
||||
SemanticLayerService queryService, ModelService modelService,
|
||||
@Lazy TagMetaService tagMetaService) {
|
||||
this.dimensionService = dimensionService;
|
||||
this.metricService = metricService;
|
||||
@@ -104,13 +101,12 @@ public class DictUtils {
|
||||
}
|
||||
|
||||
public String fetchDictFileName(DictItemResp dictItemResp) {
|
||||
return String.format(
|
||||
"dic_value_%d_%s_%s",
|
||||
dictItemResp.getModelId(), dictItemResp.getType().name(), dictItemResp.getItemId());
|
||||
return String.format("dic_value_%d_%s_%s", dictItemResp.getModelId(),
|
||||
dictItemResp.getType().name(), dictItemResp.getItemId());
|
||||
}
|
||||
|
||||
public DictTaskDO generateDictTaskDO(
|
||||
DictItemResp dictItemResp, User user, TaskStatusEnum status) {
|
||||
public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user,
|
||||
TaskStatusEnum status) {
|
||||
DictTaskDO taskDO = new DictTaskDO();
|
||||
Date createAt = new Date();
|
||||
String name = dictItemResp.fetchDictFileName();
|
||||
@@ -141,14 +137,12 @@ public class DictUtils {
|
||||
|
||||
public List<DictItemResp> dictDOList2Req(List<DictConfDO> dictConfDOList) {
|
||||
List<DictItemResp> dictItemReqList = new ArrayList<>();
|
||||
dictConfDOList.stream()
|
||||
.forEach(
|
||||
conf -> {
|
||||
DictItemResp dictItemResp = dictDO2Req(conf);
|
||||
if (Objects.nonNull(dictItemResp)) {
|
||||
dictItemReqList.add(dictDO2Req(conf));
|
||||
}
|
||||
});
|
||||
dictConfDOList.stream().forEach(conf -> {
|
||||
DictItemResp dictItemResp = dictDO2Req(conf);
|
||||
if (Objects.nonNull(dictItemResp)) {
|
||||
dictItemReqList.add(dictDO2Req(conf));
|
||||
}
|
||||
});
|
||||
return dictItemReqList;
|
||||
}
|
||||
|
||||
@@ -190,10 +184,8 @@ public class DictUtils {
|
||||
Map<String, Long> valueAndFrequencyPair = new HashMap<>(2000);
|
||||
for (Map<String, Object> line : semanticQueryResp.getResultList()) {
|
||||
|
||||
if (CollectionUtils.isEmpty(line)
|
||||
|| !line.containsKey(bizName)
|
||||
|| line.get(bizName) == null
|
||||
|| line.size() != 2) {
|
||||
if (CollectionUtils.isEmpty(line) || !line.containsKey(bizName)
|
||||
|| line.get(bizName) == null || line.size() != 2) {
|
||||
continue;
|
||||
}
|
||||
String dimValue = line.get(bizName).toString();
|
||||
@@ -218,38 +210,35 @@ public class DictUtils {
|
||||
}
|
||||
|
||||
private void addWhiteValueLines(DictItemResp dictItemResp, List<String> lines, String nature) {
|
||||
if (Objects.isNull(dictItemResp)
|
||||
|| Objects.isNull(dictItemResp.getConfig())
|
||||
if (Objects.isNull(dictItemResp) || Objects.isNull(dictItemResp.getConfig())
|
||||
|| CollectionUtils.isEmpty(dictItemResp.getConfig().getWhiteList())) {
|
||||
return;
|
||||
}
|
||||
List<String> whiteList = dictItemResp.getConfig().getWhiteList();
|
||||
whiteList.forEach(
|
||||
white -> {
|
||||
if (!StringUtils.isEmpty(white)) {
|
||||
white = white.replace(SPACE, POUND);
|
||||
}
|
||||
lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency));
|
||||
});
|
||||
whiteList.forEach(white -> {
|
||||
if (!StringUtils.isEmpty(white)) {
|
||||
white = white.replace(SPACE, POUND);
|
||||
}
|
||||
lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency));
|
||||
});
|
||||
}
|
||||
|
||||
private void constructDictLines(
|
||||
Map<String, Long> valueAndFrequencyPair, List<String> lines, String nature) {
|
||||
private void constructDictLines(Map<String, Long> valueAndFrequencyPair, List<String> lines,
|
||||
String nature) {
|
||||
if (CollectionUtils.isEmpty(valueAndFrequencyPair)) {
|
||||
return;
|
||||
}
|
||||
|
||||
valueAndFrequencyPair.forEach(
|
||||
(value, frequency) -> {
|
||||
if (!StringUtils.isEmpty(value)) {
|
||||
value = value.replace(SPACE, POUND);
|
||||
}
|
||||
lines.add(String.format("%s %s %s", value, nature, frequency));
|
||||
});
|
||||
valueAndFrequencyPair.forEach((value, frequency) -> {
|
||||
if (!StringUtils.isEmpty(value)) {
|
||||
value = value.replace(SPACE, POUND);
|
||||
}
|
||||
lines.add(String.format("%s %s %s", value, nature, frequency));
|
||||
});
|
||||
}
|
||||
|
||||
private void mergeMultivaluedValue(
|
||||
Map<String, Long> valueAndFrequencyPair, String dimValue, Long metric) {
|
||||
private void mergeMultivaluedValue(Map<String, Long> valueAndFrequencyPair, String dimValue,
|
||||
Long metric) {
|
||||
if (StringUtils.isEmpty(dimValue)) {
|
||||
return;
|
||||
}
|
||||
@@ -263,8 +252,7 @@ public class DictUtils {
|
||||
|
||||
for (String value : tmp.keySet()) {
|
||||
long metricOld =
|
||||
valueAndFrequencyPair.containsKey(value)
|
||||
? valueAndFrequencyPair.get(value)
|
||||
valueAndFrequencyPair.containsKey(value) ? valueAndFrequencyPair.get(value)
|
||||
: 0L;
|
||||
valueAndFrequencyPair.put(value, metric + metricOld);
|
||||
}
|
||||
@@ -286,8 +274,7 @@ public class DictUtils {
|
||||
String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr;
|
||||
ItemValueConfig config = dictItemResp.getConfig();
|
||||
int limit =
|
||||
(Objects.isNull(config) || Objects.isNull(config.getLimit()))
|
||||
? itemValueMaxCount
|
||||
(Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount
|
||||
: dictItemResp.getConfig().getLimit();
|
||||
|
||||
// todo 自定义指标
|
||||
@@ -312,8 +299,7 @@ public class DictUtils {
|
||||
}
|
||||
|
||||
private QuerySqlReq constructDimQueryReq(DictItemResp dictItemResp) {
|
||||
if (Objects.nonNull(dictItemResp)
|
||||
&& Objects.nonNull(dictItemResp.getConfig())
|
||||
if (Objects.nonNull(dictItemResp) && Objects.nonNull(dictItemResp.getConfig())
|
||||
&& Objects.nonNull(dictItemResp.getConfig().getMetricId())) {
|
||||
// 查询默认指标
|
||||
QueryStructReq queryStructReq = generateQueryStruct(dictItemResp);
|
||||
@@ -332,8 +318,7 @@ public class DictUtils {
|
||||
String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr;
|
||||
ItemValueConfig config = dictItemResp.getConfig();
|
||||
long limit =
|
||||
(Objects.isNull(config) || Objects.isNull(config.getLimit()))
|
||||
? itemValueMaxCount
|
||||
(Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount
|
||||
: dictItemResp.getConfig().getLimit();
|
||||
String sql = String.format(sqlPattern, bizName, where, bizName, limit);
|
||||
Set<Long> modelIds = new HashSet<>();
|
||||
@@ -371,10 +356,8 @@ public class DictUtils {
|
||||
|
||||
fillStructDateInfo(queryStructReq, dictItemResp);
|
||||
|
||||
int limit =
|
||||
Objects.isNull(dictItemResp.getConfig().getLimit())
|
||||
? itemValueMaxCount
|
||||
: dictItemResp.getConfig().getLimit();
|
||||
int limit = Objects.isNull(dictItemResp.getConfig().getLimit()) ? itemValueMaxCount
|
||||
: dictItemResp.getConfig().getLimit();
|
||||
queryStructReq.setLimit(limit);
|
||||
queryStructReq.setNeedAuth(false);
|
||||
return queryStructReq;
|
||||
@@ -410,25 +393,18 @@ public class DictUtils {
|
||||
}
|
||||
}
|
||||
|
||||
private void fillStructDateBetween(
|
||||
QueryStructReq queryStructReq,
|
||||
ModelResp model,
|
||||
Integer itemValueDateStart,
|
||||
Integer itemValueDateEnd) {
|
||||
private void fillStructDateBetween(QueryStructReq queryStructReq, ModelResp model,
|
||||
Integer itemValueDateStart, Integer itemValueDateEnd) {
|
||||
if (Objects.nonNull(model)) {
|
||||
List<Dim> timeDims = model.getTimeDimension();
|
||||
if (!CollectionUtils.isEmpty(timeDims)) {
|
||||
DateConf dateConf = new DateConf();
|
||||
dateConf.setDateMode(DateConf.DateMode.BETWEEN);
|
||||
String format = timeDims.get(0).getDateFormat();
|
||||
String start =
|
||||
LocalDate.now()
|
||||
.minusDays(itemValueDateStart)
|
||||
.format(DateTimeFormatter.ofPattern(format));
|
||||
String end =
|
||||
LocalDate.now()
|
||||
.minusDays(itemValueDateEnd)
|
||||
.format(DateTimeFormatter.ofPattern(format));
|
||||
String start = LocalDate.now().minusDays(itemValueDateStart)
|
||||
.format(DateTimeFormatter.ofPattern(format));
|
||||
String end = LocalDate.now().minusDays(itemValueDateEnd)
|
||||
.format(DateTimeFormatter.ofPattern(format));
|
||||
dateConf.setStartDate(start);
|
||||
dateConf.setEndDate(end);
|
||||
queryStructReq.setDateInfo(dateConf);
|
||||
@@ -477,17 +453,12 @@ public class DictUtils {
|
||||
|
||||
public String defaultDateFilter() {
|
||||
String format = itemValueDateFormat;
|
||||
String start =
|
||||
LocalDate.now()
|
||||
.minusDays(itemValueDateStart)
|
||||
.format(DateTimeFormatter.ofPattern(format));
|
||||
String end =
|
||||
LocalDate.now()
|
||||
.minusDays(itemValueDateEnd)
|
||||
.format(DateTimeFormatter.ofPattern(format));
|
||||
return String.format(
|
||||
"( %s >= '%s' and %s <= '%s' )",
|
||||
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(), end);
|
||||
String start = LocalDate.now().minusDays(itemValueDateStart)
|
||||
.format(DateTimeFormatter.ofPattern(format));
|
||||
String end = LocalDate.now().minusDays(itemValueDateEnd)
|
||||
.format(DateTimeFormatter.ofPattern(format));
|
||||
return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(),
|
||||
start, TimeDimensionEnum.DAY.getName(), end);
|
||||
}
|
||||
|
||||
private String generateDictDateFilter(DictItemResp dictItemResp) {
|
||||
@@ -502,11 +473,8 @@ public class DictUtils {
|
||||
}
|
||||
// 静态日期
|
||||
if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) {
|
||||
return String.format(
|
||||
"( %s >= '%s' and %s <= '%s' )",
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
config.getDateConf().getStartDate(),
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(),
|
||||
config.getDateConf().getStartDate(), TimeDimensionEnum.DAY.getName(),
|
||||
config.getDateConf().getEndDate());
|
||||
}
|
||||
// 动态日期
|
||||
@@ -527,18 +495,12 @@ public class DictUtils {
|
||||
dateFormat = itemValueDateFormat;
|
||||
}
|
||||
String start =
|
||||
LocalDate.now()
|
||||
.minusDays(dictItemResp.getConfig().getDateConf().getUnit())
|
||||
LocalDate.now().minusDays(dictItemResp.getConfig().getDateConf().getUnit())
|
||||
.format(DateTimeFormatter.ofPattern(dateFormat));
|
||||
String end =
|
||||
LocalDate.now()
|
||||
.minusDays(0)
|
||||
.format(DateTimeFormatter.ofPattern(dateFormat));
|
||||
return String.format(
|
||||
"( %s > '%s' and %s <= '%s' )",
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
start,
|
||||
TimeDimensionEnum.DAY.getName(),
|
||||
String end = LocalDate.now().minusDays(0)
|
||||
.format(DateTimeFormatter.ofPattern(dateFormat));
|
||||
return String.format("( %s > '%s' and %s <= '%s' )",
|
||||
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(),
|
||||
end);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,24 +69,20 @@ public class DimensionConverter {
|
||||
return dimensionDO;
|
||||
}
|
||||
|
||||
public static DimensionResp convert2DimensionResp(
|
||||
DimensionDO dimensionDO, Map<Long, ModelResp> modelRespMap) {
|
||||
public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO,
|
||||
Map<Long, ModelResp> modelRespMap) {
|
||||
DimensionResp dimensionResp = new DimensionResp();
|
||||
BeanUtils.copyProperties(dimensionDO, dimensionResp);
|
||||
dimensionResp.setModelName(
|
||||
modelRespMap.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getName());
|
||||
dimensionResp.setModelBizName(
|
||||
modelRespMap
|
||||
.getOrDefault(dimensionResp.getModelId(), new ModelResp())
|
||||
.getBizName());
|
||||
dimensionResp.setModelBizName(modelRespMap
|
||||
.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getBizName());
|
||||
if (dimensionDO.getDefaultValues() != null) {
|
||||
dimensionResp.setDefaultValues(
|
||||
JSONObject.parseObject(dimensionDO.getDefaultValues(), List.class));
|
||||
}
|
||||
dimensionResp.setModelFilterSql(
|
||||
modelRespMap
|
||||
.getOrDefault(dimensionResp.getModelId(), new ModelResp())
|
||||
.getFilterSql());
|
||||
dimensionResp.setModelFilterSql(modelRespMap
|
||||
.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getFilterSql());
|
||||
if (StringUtils.isNotEmpty(dimensionDO.getDimValueMaps())) {
|
||||
dimensionResp.setDimValueMaps(
|
||||
JsonUtil.toList(dimensionDO.getDimValueMaps(), DimValueMap.class));
|
||||
@@ -98,9 +94,8 @@ public class DimensionConverter {
|
||||
dimensionResp.setExt(JSONObject.parseObject(dimensionDO.getExt(), Map.class));
|
||||
}
|
||||
if (StringUtils.isNoneBlank(dimensionDO.getTypeParams())) {
|
||||
dimensionResp.setTypeParams(
|
||||
JSONObject.parseObject(
|
||||
dimensionDO.getTypeParams(), DimensionTimeTypeParams.class));
|
||||
dimensionResp.setTypeParams(JSONObject.parseObject(dimensionDO.getTypeParams(),
|
||||
DimensionTimeTypeParams.class));
|
||||
}
|
||||
dimensionResp.setType(getType(dimensionDO.getType()));
|
||||
dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
|
||||
@@ -122,15 +117,12 @@ public class DimensionConverter {
|
||||
}
|
||||
}
|
||||
|
||||
public static List<DimensionResp> filterByDataSet(
|
||||
List<DimensionResp> dimensionResps, DataSetResp dataSetResp) {
|
||||
public static List<DimensionResp> filterByDataSet(List<DimensionResp> dimensionResps,
|
||||
DataSetResp dataSetResp) {
|
||||
return dimensionResps.stream()
|
||||
.filter(
|
||||
dimensionResp ->
|
||||
dataSetResp.dimensionIds().contains(dimensionResp.getId())
|
||||
|| dataSetResp
|
||||
.getAllIncludeAllModels()
|
||||
.contains(dimensionResp.getModelId()))
|
||||
.filter(dimensionResp -> dataSetResp.dimensionIds().contains(dimensionResp.getId())
|
||||
|| dataSetResp.getAllIncludeAllModels()
|
||||
.contains(dimensionResp.getModelId()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,22 +29,14 @@ public class DomainConvert {
|
||||
DomainResp domainResp = new DomainResp();
|
||||
BeanUtils.copyProperties(domainDO, domainResp);
|
||||
domainResp.setFullPath(domainFullPathMap.get(domainDO.getId()));
|
||||
domainResp.setAdmins(
|
||||
StringUtils.isBlank(domainDO.getAdmin())
|
||||
? Lists.newArrayList()
|
||||
: Arrays.asList(domainDO.getAdmin().split(",")));
|
||||
domainResp.setAdminOrgs(
|
||||
StringUtils.isBlank(domainDO.getAdminOrg())
|
||||
? Lists.newArrayList()
|
||||
: Arrays.asList(domainDO.getAdminOrg().split(",")));
|
||||
domainResp.setViewers(
|
||||
StringUtils.isBlank(domainDO.getViewer())
|
||||
? Lists.newArrayList()
|
||||
: Arrays.asList(domainDO.getViewer().split(",")));
|
||||
domainResp.setViewOrgs(
|
||||
StringUtils.isBlank(domainDO.getViewOrg())
|
||||
? Lists.newArrayList()
|
||||
: Arrays.asList(domainDO.getViewOrg().split(",")));
|
||||
domainResp.setAdmins(StringUtils.isBlank(domainDO.getAdmin()) ? Lists.newArrayList()
|
||||
: Arrays.asList(domainDO.getAdmin().split(",")));
|
||||
domainResp.setAdminOrgs(StringUtils.isBlank(domainDO.getAdminOrg()) ? Lists.newArrayList()
|
||||
: Arrays.asList(domainDO.getAdminOrg().split(",")));
|
||||
domainResp.setViewers(StringUtils.isBlank(domainDO.getViewer()) ? Lists.newArrayList()
|
||||
: Arrays.asList(domainDO.getViewer().split(",")));
|
||||
domainResp.setViewOrgs(StringUtils.isBlank(domainDO.getViewOrg()) ? Lists.newArrayList()
|
||||
: Arrays.asList(domainDO.getViewOrg().split(",")));
|
||||
return domainResp;
|
||||
}
|
||||
|
||||
|
||||
@@ -70,13 +70,13 @@ public class MetricConverter {
|
||||
return convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
|
||||
}
|
||||
|
||||
public static MetricResp convert2MetricResp(
|
||||
MetricDO metricDO, Map<Long, ModelResp> modelMap, List<Long> collect) {
|
||||
public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap,
|
||||
List<Long> collect) {
|
||||
MetricResp metricResp = new MetricResp();
|
||||
BeanUtils.copyProperties(metricDO, metricResp);
|
||||
|
||||
metricResp.setDataFormat(
|
||||
JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class));
|
||||
metricResp
|
||||
.setDataFormat(JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class));
|
||||
ModelResp modelResp = modelMap.get(metricDO.getModelId());
|
||||
if (modelResp != null) {
|
||||
metricResp.setModelName(modelResp.getName());
|
||||
@@ -97,17 +97,14 @@ public class MetricConverter {
|
||||
}
|
||||
metricResp.setTypeEnum(TypeEnums.METRIC);
|
||||
if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {
|
||||
metricResp.setMetricDefineByMeasureParams(
|
||||
JSONObject.parseObject(
|
||||
metricDO.getTypeParams(), MetricDefineByMeasureParams.class));
|
||||
metricResp.setMetricDefineByMeasureParams(JSONObject
|
||||
.parseObject(metricDO.getTypeParams(), MetricDefineByMeasureParams.class));
|
||||
} else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) {
|
||||
metricResp.setMetricDefineByMetricParams(
|
||||
JSONObject.parseObject(
|
||||
metricDO.getTypeParams(), MetricDefineByMetricParams.class));
|
||||
metricResp.setMetricDefineByMetricParams(JSONObject
|
||||
.parseObject(metricDO.getTypeParams(), MetricDefineByMetricParams.class));
|
||||
} else if (MetricDefineType.FIELD.name().equalsIgnoreCase(metricDO.getDefineType())) {
|
||||
metricResp.setMetricDefineByFieldParams(
|
||||
JSONObject.parseObject(
|
||||
metricDO.getTypeParams(), MetricDefineByFieldParams.class));
|
||||
metricResp.setMetricDefineByFieldParams(JSONObject.parseObject(metricDO.getTypeParams(),
|
||||
MetricDefineByFieldParams.class));
|
||||
}
|
||||
if (metricDO.getDefineType() != null) {
|
||||
metricResp.setMetricDefineType(MetricDefineType.valueOf(metricDO.getDefineType()));
|
||||
@@ -116,15 +113,11 @@ public class MetricConverter {
|
||||
return metricResp;
|
||||
}
|
||||
|
||||
public static List<MetricResp> filterByDataSet(
|
||||
List<MetricResp> metricResps, DataSetResp dataSetResp) {
|
||||
public static List<MetricResp> filterByDataSet(List<MetricResp> metricResps,
|
||||
DataSetResp dataSetResp) {
|
||||
return metricResps.stream()
|
||||
.filter(
|
||||
metricResp ->
|
||||
dataSetResp.metricIds().contains(metricResp.getId())
|
||||
|| dataSetResp
|
||||
.getAllIncludeAllModels()
|
||||
.contains(metricResp.getModelId()))
|
||||
.filter(metricResp -> dataSetResp.metricIds().contains(metricResp.getId())
|
||||
|| dataSetResp.getAllIncludeAllModels().contains(metricResp.getModelId()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,8 @@ import java.util.stream.Collectors;
|
||||
@Slf4j
|
||||
public class MetricDrillDownChecker {
|
||||
|
||||
@Autowired private MetricService metricService;
|
||||
@Autowired
|
||||
private MetricService metricService;
|
||||
|
||||
public void checkQuery(QueryStatement queryStatement) {
|
||||
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
|
||||
@@ -54,12 +55,8 @@ public class MetricDrillDownChecker {
|
||||
getNecessaryDimensionMissing(necessaryDimensions, dimensionFields);
|
||||
if (!CollectionUtils.isEmpty(dimensionsMissing)) {
|
||||
String errMsg =
|
||||
String.format(
|
||||
"指标:%s 缺失必要下钻维度:%s",
|
||||
metric.getName(),
|
||||
dimensionsMissing.stream()
|
||||
.map(DimensionResp::getName)
|
||||
.collect(Collectors.toList()));
|
||||
String.format("指标:%s 缺失必要下钻维度:%s", metric.getName(), dimensionsMissing
|
||||
.stream().map(DimensionResp::getName).collect(Collectors.toList()));
|
||||
throw new InvalidArgumentException(errMsg);
|
||||
}
|
||||
}
|
||||
@@ -95,28 +92,18 @@ public class MetricDrillDownChecker {
|
||||
* To check whether the dimension can drill down the metric, eg: some descriptive dimensions are
|
||||
* not suitable as drill-down dimensions
|
||||
*/
|
||||
private boolean checkDrillDownDimension(
|
||||
String dimensionName,
|
||||
List<MetricResp> metricResps,
|
||||
private boolean checkDrillDownDimension(String dimensionName, List<MetricResp> metricResps,
|
||||
SemanticSchemaResp semanticSchemaResp) {
|
||||
if (CollectionUtils.isEmpty(metricResps)) {
|
||||
return true;
|
||||
}
|
||||
List<String> relateDimensions =
|
||||
metricResps.stream()
|
||||
.map(this::getDrillDownDimensions)
|
||||
.filter(
|
||||
drillDownDimensions ->
|
||||
!CollectionUtils.isEmpty(drillDownDimensions))
|
||||
.map(
|
||||
drillDownDimensions ->
|
||||
drillDownDimensions.stream()
|
||||
.map(DrillDownDimension::getDimensionId)
|
||||
.collect(Collectors.toList()))
|
||||
.flatMap(Collection::stream)
|
||||
.map(id -> convertDimensionIdToBizName(id, semanticSchemaResp))
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toList());
|
||||
List<String> relateDimensions = metricResps.stream().map(this::getDrillDownDimensions)
|
||||
.filter(drillDownDimensions -> !CollectionUtils.isEmpty(drillDownDimensions))
|
||||
.map(drillDownDimensions -> drillDownDimensions.stream()
|
||||
.map(DrillDownDimension::getDimensionId).collect(Collectors.toList()))
|
||||
.flatMap(Collection::stream)
|
||||
.map(id -> convertDimensionIdToBizName(id, semanticSchemaResp))
|
||||
.filter(Objects::nonNull).collect(Collectors.toList());
|
||||
// if no metric has drill down dimension, return true
|
||||
if (CollectionUtils.isEmpty(relateDimensions)) {
|
||||
return true;
|
||||
@@ -125,8 +112,8 @@ public class MetricDrillDownChecker {
|
||||
return relateDimensions.contains(dimensionName);
|
||||
}
|
||||
|
||||
private List<DimensionResp> getNecessaryDimensions(
|
||||
MetricSchemaResp metric, SemanticSchemaResp semanticSchemaResp) {
|
||||
private List<DimensionResp> getNecessaryDimensions(MetricSchemaResp metric,
|
||||
SemanticSchemaResp semanticSchemaResp) {
|
||||
if (metric == null) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
@@ -134,12 +121,9 @@ public class MetricDrillDownChecker {
|
||||
if (CollectionUtils.isEmpty(drillDownDimensions)) {
|
||||
return Lists.newArrayList();
|
||||
}
|
||||
return drillDownDimensions.stream()
|
||||
.filter(DrillDownDimension::isNecessary)
|
||||
.map(DrillDownDimension::getDimensionId)
|
||||
.map(semanticSchemaResp::getDimension)
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toList());
|
||||
return drillDownDimensions.stream().filter(DrillDownDimension::isNecessary)
|
||||
.map(DrillDownDimension::getDimensionId).map(semanticSchemaResp::getDimension)
|
||||
.filter(Objects::nonNull).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<String> getDimensionFields(List<String> groupByFields, List<String> whereFields) {
|
||||
@@ -153,8 +137,8 @@ public class MetricDrillDownChecker {
|
||||
return dimensionFields;
|
||||
}
|
||||
|
||||
private List<MetricResp> getMetrics(
|
||||
List<String> metricFields, SemanticSchemaResp semanticSchemaResp) {
|
||||
private List<MetricResp> getMetrics(List<String> metricFields,
|
||||
SemanticSchemaResp semanticSchemaResp) {
|
||||
return semanticSchemaResp.getMetrics().stream()
|
||||
.filter(metricSchemaResp -> metricFields.contains(metricSchemaResp.getBizName()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
@@ -19,11 +19,8 @@ public class ModelClusterBuilder {
|
||||
public static Map<String, ModelCluster> buildModelClusters(List<Long> modelIds) {
|
||||
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
|
||||
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchemaResps(modelIds);
|
||||
Map<Long, ModelSchemaResp> modelIdToModelSchema =
|
||||
modelSchemaResps.stream()
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
ModelSchemaResp::getId, value -> value, (k1, k2) -> k1));
|
||||
Map<Long, ModelSchemaResp> modelIdToModelSchema = modelSchemaResps.stream()
|
||||
.collect(Collectors.toMap(ModelSchemaResp::getId, value -> value, (k1, k2) -> k1));
|
||||
|
||||
Set<Long> visited = new HashSet<>();
|
||||
List<Set<Long>> modelClusters = new ArrayList<>();
|
||||
@@ -40,25 +37,17 @@ public class ModelClusterBuilder {
|
||||
.collect(Collectors.toMap(ModelCluster::getKey, value -> value, (k1, k2) -> k1));
|
||||
}
|
||||
|
||||
private static ModelCluster getModelCluster(
|
||||
Map<Long, ModelSchemaResp> modelIdToModelSchema, Set<Long> modelIds) {
|
||||
boolean containsPartitionDimensions =
|
||||
modelIds.stream()
|
||||
.map(modelIdToModelSchema::get)
|
||||
.filter(Objects::nonNull)
|
||||
.anyMatch(
|
||||
modelSchemaResp ->
|
||||
CollectionUtils.isNotEmpty(
|
||||
modelSchemaResp.getTimeDimension()));
|
||||
private static ModelCluster getModelCluster(Map<Long, ModelSchemaResp> modelIdToModelSchema,
|
||||
Set<Long> modelIds) {
|
||||
boolean containsPartitionDimensions = modelIds.stream().map(modelIdToModelSchema::get)
|
||||
.filter(Objects::nonNull).anyMatch(modelSchemaResp -> CollectionUtils
|
||||
.isNotEmpty(modelSchemaResp.getTimeDimension()));
|
||||
|
||||
return ModelCluster.build(modelIds, containsPartitionDimensions);
|
||||
}
|
||||
|
||||
private static void dfs(
|
||||
ModelSchemaResp model,
|
||||
Map<Long, ModelSchemaResp> modelMap,
|
||||
Set<Long> visited,
|
||||
Set<Long> modelCluster) {
|
||||
private static void dfs(ModelSchemaResp model, Map<Long, ModelSchemaResp> modelMap,
|
||||
Set<Long> visited, Set<Long> modelCluster) {
|
||||
visited.add(model.getId());
|
||||
modelCluster.add(model.getId());
|
||||
for (Long neighborId : model.getModelClusterSet()) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user