(improvement)[build] Use Spotless to customize the code formatting (#1750)

This commit is contained in:
lexluo09
2024-10-04 00:05:04 +08:00
committed by GitHub
parent 44d1cde34f
commit 71a9954be5
521 changed files with 7811 additions and 13046 deletions

View File

@@ -7,4 +7,5 @@ import java.lang.annotation.Target;
@Target({ElementType.PARAMETER, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface ApiHeaderCheck {}
public @interface ApiHeaderCheck {
}

View File

@@ -9,4 +9,5 @@ import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface S2DataPermission {}
public @interface S2DataPermission {
}

View File

@@ -30,7 +30,8 @@ public class ApiHeaderCheckAspect {
private static final String SIGNATURE = "signature";
@Autowired private AppService appService;
@Autowired
private AppService appService;
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck)")
private void apiPermissionCheck() {}
@@ -63,12 +64,8 @@ public class ApiHeaderCheckAspect {
if (!AppStatus.ONLINE.equals(appDetailResp.getAppStatus())) {
throw new InvalidArgumentException("该应用暂时为非在线状态");
}
Pair<Boolean, String> checkResult =
SignatureUtils.isValidSignature(
appId,
appDetailResp.getAppSecret(),
Long.parseLong(timestampStr),
signature);
Pair<Boolean, String> checkResult = SignatureUtils.isValidSignature(appId,
appDetailResp.getAppSecret(), Long.parseLong(timestampStr), signature);
if (!checkResult.first) {
throw new InvalidArgumentException(checkResult.second);
}

View File

@@ -44,10 +44,10 @@ public class DimValueAspect {
@Value("${s2.dimension.value.map.enable:true}")
private Boolean dimensionValueMapEnable;
@Autowired private DimensionService dimensionService;
@Autowired
private DimensionService dimensionService;
@Around(
"execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))")
@Around("execution(* com.tencent.supersonic.headless.server.facade.service.SemanticLayerService.queryByReq(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
@@ -108,22 +108,14 @@ public class DimValueAspect {
}
// consider '=' filter
if (expression.getOperator().equals(FilterOperatorEnum.EQUALS.getValue())) {
dimension.getDimValueMaps().stream()
.forEach(
dimValue -> {
if (!CollectionUtils.isEmpty(dimValue.getAlias())
&& dimValue.getAlias()
.contains(
expression
.getFieldValue()
.toString())) {
getFiledNameToValueMap(
filedNameToValueMap,
expression.getFieldValue().toString(),
dimValue.getTechName(),
expression.getFieldName());
}
});
dimension.getDimValueMaps().stream().forEach(dimValue -> {
if (!CollectionUtils.isEmpty(dimValue.getAlias()) && dimValue.getAlias()
.contains(expression.getFieldValue().toString())) {
getFiledNameToValueMap(filedNameToValueMap,
expression.getFieldValue().toString(), dimValue.getTechName(),
expression.getFieldName());
}
});
}
// consider 'in' filter,each element needs to judge.
replaceInCondition(expression, dimension, filedNameToValueMap);
@@ -141,9 +133,7 @@ public class DimValueAspect {
return queryResultWithColumns;
}
public void replaceInCondition(
FieldExpression expression,
DimensionResp dimension,
public void replaceInCondition(FieldExpression expression, DimensionResp dimension,
Map<String, Map<String, String>> filedNameToValueMap) {
if (expression.getOperator().equals(FilterOperatorEnum.IN.getValue())) {
String fieldValue = JsonUtil.toString(expression.getFieldValue());
@@ -165,27 +155,20 @@ public class DimValueAspect {
}
}
if (!revisedValues.equals(values)) {
getFiledNameToValueMap(
filedNameToValueMap,
JsonUtil.toString(values),
JsonUtil.toString(revisedValues),
expression.getFieldName());
getFiledNameToValueMap(filedNameToValueMap, JsonUtil.toString(values),
JsonUtil.toString(revisedValues), expression.getFieldName());
}
}
}
public void getFiledNameToValueMap(
Map<String, Map<String, String>> filedNameToValueMap,
String oldValue,
String newValue,
String fieldName) {
public void getFiledNameToValueMap(Map<String, Map<String, String>> filedNameToValueMap,
String oldValue, String newValue, String fieldName) {
Map<String, String> map = new HashMap<>();
map.put(oldValue, newValue);
filedNameToValueMap.put(fieldName, map);
}
private void rewriteDimValue(
SemanticQueryResp semanticQueryResp,
private void rewriteDimValue(SemanticQueryResp semanticQueryResp,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (!selectDimValueMap(semanticQueryResp.getColumns(), dimAndTechNameAndBizNamePair)) {
return;
@@ -209,8 +192,7 @@ public class DimValueAspect {
}
}
private boolean selectDimValueMap(
List<QueryColumn> columns,
private boolean selectDimValueMap(List<QueryColumn> columns,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)
|| CollectionUtils.isEmpty(dimAndTechNameAndBizNamePair)) {
@@ -225,8 +207,8 @@ public class DimValueAspect {
return false;
}
private void rewriteFilter(
List<Filter> dimensionFilters, Map<String, Map<String, String>> aliasAndTechNamePair) {
private void rewriteFilter(List<Filter> dimensionFilters,
Map<String, Map<String, String>> aliasAndTechNamePair) {
for (Filter filter : dimensionFilters) {
if (Objects.isNull(filter)) {
continue;
@@ -283,18 +265,15 @@ public class DimValueAspect {
continue;
}
if (StringUtils.isNotEmpty(dimValueMap.getBizName())) {
aliasAndBizNameToTechName.put(
dimValueMap.getBizName(), dimValueMap.getTechName());
aliasAndBizNameToTechName.put(dimValueMap.getBizName(),
dimValueMap.getTechName());
}
if (!CollectionUtils.isEmpty(dimValueMap.getAlias())) {
dimValueMap.getAlias().stream()
.forEach(
alias -> {
if (StringUtils.isNotEmpty(alias)) {
aliasAndBizNameToTechName.put(
alias, dimValueMap.getTechName());
}
});
dimValueMap.getAlias().stream().forEach(alias -> {
if (StringUtils.isNotEmpty(alias)) {
aliasAndBizNameToTechName.put(alias, dimValueMap.getTechName());
}
});
}
}
@@ -339,8 +318,7 @@ public class DimValueAspect {
}
private boolean needSkipDimension(DimensionResp dimension) {
return Objects.isNull(dimension)
|| StringUtils.isEmpty(dimension.getBizName())
return Objects.isNull(dimension) || StringUtils.isEmpty(dimension.getBizName())
|| CollectionUtils.isEmpty(dimension.getDimValueMaps());
}
}

View File

@@ -52,10 +52,14 @@ import java.util.stream.Collectors;
@Slf4j
public class S2DataPermissionAspect {
@Autowired private QueryStructUtils queryStructUtils;
@Autowired private ModelService modelService;
@Autowired private SchemaService schemaService;
@Autowired private AuthService authService;
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private ModelService modelService;
@Autowired
private SchemaService schemaService;
@Autowired
private AuthService authService;
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)")
private void s2PermissionCheck() {}
@@ -109,26 +113,19 @@ public class S2DataPermissionAspect {
return result;
}
private void checkColPermission(
SemanticQueryReq semanticQueryReq,
AuthorizedResourceResp authorizedResource,
Set<Long> modelIds,
private void checkColPermission(SemanticQueryReq semanticQueryReq,
AuthorizedResourceResp authorizedResource, Set<Long> modelIds,
SemanticSchemaResp semanticSchemaResp) {
// get high sensitive fields in query
Set<String> bizNamesInQueryReq = getBizNameInQueryReq(semanticQueryReq, semanticSchemaResp);
Set<String> sensitiveBizNamesByModel =
getHighSensitiveBizNamesByModelId(semanticSchemaResp);
Set<String> sensitiveBizNameInQuery =
bizNamesInQueryReq
.parallelStream()
.filter(sensitiveBizNamesByModel::contains)
.collect(Collectors.toSet());
Set<String> sensitiveBizNameInQuery = bizNamesInQueryReq.parallelStream()
.filter(sensitiveBizNamesByModel::contains).collect(Collectors.toSet());
// get high sensitive field cur user has been authed
Set<String> sensitiveBizNameUserAuthed =
authorizedResource.getAuthResList().stream()
.map(AuthRes::getName)
.collect(Collectors.toSet());
Set<String> sensitiveBizNameUserAuthed = authorizedResource.getAuthResList().stream()
.map(AuthRes::getName).collect(Collectors.toSet());
sensitiveBizNameInQuery.removeAll(sensitiveBizNameUserAuthed);
if (!CollectionUtils.isEmpty(sensitiveBizNameInQuery)) {
Set<String> sensitiveResNames =
@@ -140,8 +137,8 @@ public class S2DataPermissionAspect {
}
}
private Set<Long> getModelIdInQuery(
SemanticQueryReq semanticQueryReq, SemanticSchemaResp semanticSchemaResp) {
private Set<Long> getModelIdInQuery(SemanticQueryReq semanticQueryReq,
SemanticSchemaResp semanticSchemaResp) {
if (semanticQueryReq instanceof QuerySqlReq) {
QuerySqlReq querySqlReq = (QuerySqlReq) semanticQueryReq;
return queryStructUtils.getModelIdFromSql(querySqlReq, semanticSchemaResp);
@@ -153,8 +150,8 @@ public class S2DataPermissionAspect {
return Sets.newHashSet();
}
private void checkRowPermission(
SemanticQueryReq queryReq, AuthorizedResourceResp authorizedResource) {
private void checkRowPermission(SemanticQueryReq queryReq,
AuthorizedResourceResp authorizedResource) {
if (queryReq instanceof QuerySqlReq) {
doRowPermission((QuerySqlReq) queryReq, authorizedResource);
}
@@ -163,8 +160,8 @@ public class S2DataPermissionAspect {
}
}
private Set<String> getBizNameInQueryReq(
SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) {
private Set<String> getBizNameInQueryReq(SemanticQueryReq queryReq,
SemanticSchemaResp semanticSchemaResp) {
if (queryReq instanceof QuerySqlReq) {
return queryStructUtils.getBizNameFromSql((QuerySqlReq) queryReq, semanticSchemaResp);
}
@@ -181,8 +178,8 @@ public class S2DataPermissionAspect {
return schemaService.fetchSemanticSchema(filter);
}
private void doRowPermission(
QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(QuerySqlReq querySqlReq,
AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -196,14 +193,11 @@ public class S2DataPermissionAspect {
return;
}
dimensionFilters.stream()
.forEach(
filter -> {
if (StringUtils.isNotEmpty(filter)
&& StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
dimensionFilters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
@@ -217,8 +211,8 @@ public class S2DataPermissionAspect {
}
}
private void doRowPermission(
QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(QueryStructReq queryStructReq,
AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -232,21 +226,17 @@ public class S2DataPermissionAspect {
return;
}
dimensionFilters.stream()
.forEach(
filter -> {
if (StringUtils.isNotEmpty(filter)
&& StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
dimensionFilters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
if (StringUtils.isNotEmpty(joiner.toString())) {
log.info("before doRowPermission, queryStructReq:{}", queryStructReq);
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString());
List<Filter> filters =
Objects.isNull(queryStructReq.getOriginalFilter())
? new ArrayList<>()
Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>()
: queryStructReq.getOriginalFilter();
filters.add(filter);
queryStructReq.setDimensionFilters(filters);
@@ -269,8 +259,7 @@ public class S2DataPermissionAspect {
public void checkModelVisible(User user, Set<Long> modelIds) {
List<Long> modelListVisible =
modelService.getModelListWithAuth(user, null, AuthType.VISIBLE).stream()
.map(ModelResp::getId)
.collect(Collectors.toList());
.map(ModelResp::getId).collect(Collectors.toList());
List<Long> modelIdCopied = new ArrayList<>(modelIds);
modelIdCopied.removeAll(modelListVisible);
if (!CollectionUtils.isEmpty(modelIdCopied)) {
@@ -281,9 +270,8 @@ public class S2DataPermissionAspect {
if (modelResp == null) {
throw new InvalidArgumentException("查询的模型不存在");
}
String message =
String.format(
"您没有模型[%s]权限,请联系管理员%s开通", modelResp.getName(), modelResp.getAdmins());
String message = String.format("您没有模型[%s]权限,请联系管理员%s开通", modelResp.getName(),
modelResp.getAdmins());
throw new InvalidPermissionException(message);
}
}
@@ -292,20 +280,14 @@ public class S2DataPermissionAspect {
Set<String> highSensitiveCols = new HashSet<>();
if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) {
semanticSchemaResp.getDimensions().stream()
.filter(
dimSchemaResp ->
SensitiveLevelEnum.HIGH
.getCode()
.equals(dimSchemaResp.getSensitiveLevel()))
.filter(dimSchemaResp -> SensitiveLevelEnum.HIGH.getCode()
.equals(dimSchemaResp.getSensitiveLevel()))
.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
if (!CollectionUtils.isEmpty(semanticSchemaResp.getMetrics())) {
semanticSchemaResp.getMetrics().stream()
.filter(
metricSchemaResp ->
SensitiveLevelEnum.HIGH
.getCode()
.equals(metricSchemaResp.getSensitiveLevel()))
.filter(metricSchemaResp -> SensitiveLevelEnum.HIGH.getCode()
.equals(metricSchemaResp.getSensitiveLevel()))
.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
}
return highSensitiveCols;
@@ -315,11 +297,8 @@ public class S2DataPermissionAspect {
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setModelIds(new ArrayList<>(modelIds));
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info(
"user:{}, domainId:{}, after queryAuthorizedResources:{}",
user.getName(),
modelIds,
authorizedResource);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(),
modelIds, authorizedResource);
return authorizedResource;
}
@@ -328,9 +307,7 @@ public class S2DataPermissionAspect {
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
public void addHint(
Set<Long> modelIds,
SemanticQueryResp queryResultWithColumns,
public void addHint(Set<Long> modelIds, SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(filters)) {
@@ -342,20 +319,15 @@ public class S2DataPermissionAspect {
ModelResp modelResp = modelService.getModel(modelIds.iterator().next());
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
filters.stream()
.forEach(
filter -> {
if (StringUtils.isNotEmpty(filter.getDescription())) {
descList.add(filter.getDescription());
}
exprList.add(filter.getExpressions().toString());
});
filters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter.getDescription())) {
descList.add(filter.getDescription());
}
exprList.add(filter.getExpressions().toString());
});
String promptInfo = "当前结果已经过行权限过滤,详细过滤条件如下:%s, 申请权限请联系管理员%s";
String message =
String.format(
promptInfo,
CollectionUtils.isEmpty(descList) ? exprList : descList,
admins);
String message = String.format(promptInfo,
CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
}

View File

@@ -23,45 +23,36 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
public class ChatQueryApiController {
@Autowired private ChatLayerService chatLayerService;
@Autowired
private ChatLayerService chatLayerService;
@Autowired private SemanticLayerService semanticLayerService;
@Autowired
private SemanticLayerService semanticLayerService;
@PostMapping("/chat/search")
public Object search(
@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object search(@RequestBody QueryNLReq queryNLReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
queryNLReq.setUser(UserHolder.findUser(request, response));
return chatLayerService.retrieve(queryNLReq);
}
@PostMapping("/chat/map")
public Object map(
@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
public Object map(@RequestBody QueryNLReq queryNLReq, HttpServletRequest request,
HttpServletResponse response) {
queryNLReq.setUser(UserHolder.findUser(request, response));
return chatLayerService.performMapping(queryNLReq);
}
@PostMapping("/chat/parse")
public Object parse(
@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object parse(@RequestBody QueryNLReq queryNLReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
queryNLReq.setUser(UserHolder.findUser(request, response));
return chatLayerService.performParsing(queryNLReq);
}
@PostMapping("/chat")
public Object queryByNL(
@RequestBody QueryNLReq queryNLReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object queryByNL(@RequestBody QueryNLReq queryNLReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
ParseResp parseResp = chatLayerService.performParsing(queryNLReq);
if (parseResp.getState().equals(ParseResp.ParseState.COMPLETED)) {

View File

@@ -21,15 +21,14 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
public class DataSetQueryApiController {
@Autowired private DataSetService dataSetService;
@Autowired private SemanticLayerService semanticLayerService;
@Autowired
private DataSetService dataSetService;
@Autowired
private SemanticLayerService semanticLayerService;
@PostMapping("/dataSet")
public Object queryByDataSet(
@RequestBody QueryDataSetReq queryDataSetReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object queryByDataSet(@RequestBody QueryDataSetReq queryDataSetReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
SemanticQueryReq queryReq = dataSetService.convert(queryDataSetReq);
return semanticLayerService.queryByReq(queryReq, user);

View File

@@ -19,14 +19,12 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
public class MetaDiscoveryApiController {
@Autowired private ChatLayerService chatLayerService;
@Autowired
private ChatLayerService chatLayerService;
@PostMapping("map")
public Object map(
@RequestBody QueryMapReq queryMapReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object map(@RequestBody QueryMapReq queryMapReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
queryMapReq.setUser(user);
return chatLayerService.map(queryMapReq);

View File

@@ -24,39 +24,33 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
public class MetricQueryApiController {
@Autowired private SemanticLayerService semanticLayerService;
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired private MetricService metricService;
@Autowired
private MetricService metricService;
@Autowired private DownloadService downloadService;
@Autowired
private DownloadService downloadService;
@PostMapping("/metric")
public Object queryByMetric(
@RequestBody QueryMetricReq queryMetricReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object queryByMetric(@RequestBody QueryMetricReq queryMetricReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
QueryStructReq queryStructReq = metricService.convert(queryMetricReq);
return semanticLayerService.queryByReq(queryStructReq.convert(true), user);
}
@PostMapping("/download/metric")
public void downloadMetric(
@RequestBody DownloadMetricReq downloadMetricReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public void downloadMetric(@RequestBody DownloadMetricReq downloadMetricReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
downloadService.downloadByStruct(downloadMetricReq, user, response);
}
@PostMapping("/downloadBatch/metric")
public void downloadBatch(
@RequestBody BatchDownloadReq batchDownloadReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public void downloadBatch(@RequestBody BatchDownloadReq batchDownloadReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
downloadService.batchDownload(batchDownloadReq, user, response);
}

View File

@@ -30,16 +30,15 @@ import java.util.stream.Collectors;
@Slf4j
public class SqlQueryApiController {
@Autowired private SemanticLayerService semanticLayerService;
@Autowired
private SemanticLayerService semanticLayerService;
@Autowired private ChatLayerService chatLayerService;
@Autowired
private ChatLayerService chatLayerService;
@PostMapping("/sql")
public Object queryBySql(
@RequestBody QuerySqlReq querySqlReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object queryBySql(@RequestBody QuerySqlReq querySqlReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
String sql = querySqlReq.getSql();
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
@@ -48,63 +47,40 @@ public class SqlQueryApiController {
}
@PostMapping("/sqls")
public Object queryBySqls(
@RequestBody QuerySqlsReq querySqlsReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object queryBySqls(@RequestBody QuerySqlsReq querySqlsReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
List<SemanticQueryReq> semanticQueryReqs =
querySqlsReq.getSqls().stream()
.map(
sql -> {
QuerySqlReq querySqlReq = new QuerySqlReq();
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
chatLayerService.correct(querySqlReq, user);
return querySqlReq;
})
.collect(Collectors.toList());
List<SemanticQueryReq> semanticQueryReqs = querySqlsReq.getSqls().stream().map(sql -> {
QuerySqlReq querySqlReq = new QuerySqlReq();
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
chatLayerService.correct(querySqlReq, user);
return querySqlReq;
}).collect(Collectors.toList());
List<CompletableFuture<SemanticQueryResp>> futures =
semanticQueryReqs.stream()
.map(
querySqlReq ->
CompletableFuture.supplyAsync(
() -> {
try {
return semanticLayerService.queryByReq(
querySqlReq, user);
} catch (Exception e) {
log.error(
"querySqlReq:{},queryByReq error:",
querySqlReq,
e);
return new SemanticQueryResp();
}
}))
.collect(Collectors.toList());
semanticQueryReqs.stream().map(querySqlReq -> CompletableFuture.supplyAsync(() -> {
try {
return semanticLayerService.queryByReq(querySqlReq, user);
} catch (Exception e) {
log.error("querySqlReq:{},queryByReq error:", querySqlReq, e);
return new SemanticQueryResp();
}
})).collect(Collectors.toList());
return futures.stream().map(CompletableFuture::join).collect(Collectors.toList());
}
@PostMapping("/sqlsWithException")
public Object queryBySqlsWithException(
@RequestBody QuerySqlsReq querySqlsReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object queryBySqlsWithException(@RequestBody QuerySqlsReq querySqlsReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
List<SemanticQueryReq> semanticQueryReqs =
querySqlsReq.getSqls().stream()
.map(
sql -> {
QuerySqlReq querySqlReq = new QuerySqlReq();
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
chatLayerService.correct(querySqlReq, user);
return querySqlReq;
})
.collect(Collectors.toList());
List<SemanticQueryReq> semanticQueryReqs = querySqlsReq.getSqls().stream().map(sql -> {
QuerySqlReq querySqlReq = new QuerySqlReq();
BeanUtils.copyProperties(querySqlsReq, querySqlReq);
querySqlReq.setSql(StringUtil.replaceBackticks(sql));
chatLayerService.correct(querySqlReq, user);
return querySqlReq;
}).collect(Collectors.toList());
List<SemanticQueryResp> semanticQueryRespList = new ArrayList<>();
try {
for (SemanticQueryReq semanticQueryReq : semanticQueryReqs) {
@@ -119,11 +95,8 @@ public class SqlQueryApiController {
}
@PostMapping("/validate")
public Object validate(
@RequestBody QuerySqlReq querySqlReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object validate(@RequestBody QuerySqlReq querySqlReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
String sql = querySqlReq.getSql();
querySqlReq.setSql(StringUtil.replaceBackticks(sql));

View File

@@ -19,14 +19,12 @@ import org.springframework.web.bind.annotation.RestController;
@Slf4j
public class TagQueryApiController {
@Autowired private SemanticLayerService semanticLayerService;
@Autowired
private SemanticLayerService semanticLayerService;
@PostMapping("/tag")
public Object queryByTag(
@RequestBody QueryStructReq queryStructReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Object queryByTag(@RequestBody QueryStructReq queryStructReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return semanticLayerService.queryByReq(queryStructReq.convert(), user);
}

View File

@@ -53,20 +53,22 @@ import java.util.stream.Collectors;
@Service
@Slf4j
public class S2ChatLayerService implements ChatLayerService {
@Autowired private SchemaService schemaService;
@Autowired private DataSetService dataSetService;
@Autowired private RetrieveService retrieveService;
@Autowired private ChatWorkflowEngine chatWorkflowEngine;
@Autowired
private SchemaService schemaService;
@Autowired
private DataSetService dataSetService;
@Autowired
private RetrieveService retrieveService;
@Autowired
private ChatWorkflowEngine chatWorkflowEngine;
@Override
public MapResp performMapping(QueryNLReq queryNLReq) {
MapResp mapResp = new MapResp(queryNLReq.getQueryText());
ChatQueryContext queryCtx = buildChatQueryContext(queryNLReq);
ComponentFactory.getSchemaMappers()
.forEach(
mapper -> {
mapper.map(queryCtx);
});
ComponentFactory.getSchemaMappers().forEach(mapper -> {
mapper.map(queryCtx);
});
mapResp.setMapInfo(queryCtx.getMapInfo());
return mapResp;
}
@@ -96,17 +98,12 @@ public class S2ChatLayerService implements ChatLayerService {
private ChatQueryContext buildChatQueryContext(QueryNLReq queryNLReq) {
SemanticSchema semanticSchema = schemaService.getSemanticSchema(queryNLReq.getDataSetIds());
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds();
ChatQueryContext queryCtx =
ChatQueryContext.builder()
.queryFilters(queryNLReq.getQueryFilters())
.semanticSchema(semanticSchema)
.candidateQueries(new ArrayList<>())
.mapInfo(new SchemaMapInfo())
.modelIdToDataSetIds(modelIdToDataSetIds)
.text2SQLType(queryNLReq.getText2SQLType())
.mapModeEnum(queryNLReq.getMapModeEnum())
.dataSetIds(queryNLReq.getDataSetIds())
.build();
ChatQueryContext queryCtx = ChatQueryContext.builder()
.queryFilters(queryNLReq.getQueryFilters()).semanticSchema(semanticSchema)
.candidateQueries(new ArrayList<>()).mapInfo(new SchemaMapInfo())
.modelIdToDataSetIds(modelIdToDataSetIds).text2SQLType(queryNLReq.getText2SQLType())
.mapModeEnum(queryNLReq.getMapModeEnum()).dataSetIds(queryNLReq.getDataSetIds())
.build();
BeanUtils.copyProperties(queryNLReq, queryCtx);
return queryCtx;
}
@@ -146,14 +143,12 @@ public class S2ChatLayerService implements ChatLayerService {
SchemaElement dataSet = semanticSchema.getDataSet(dataSetId);
semanticParseInfo.setDataSet(dataSet);
semanticParseInfo.setQueryConfig(semanticSchema.getQueryConfig(dataSetId));
ComponentFactory.getSemanticCorrectors()
.forEach(
corrector -> {
if (!(corrector instanceof GrammarCorrector
|| (corrector instanceof SchemaCorrector))) {
corrector.correct(queryCtx, semanticParseInfo);
}
});
ComponentFactory.getSemanticCorrectors().forEach(corrector -> {
if (!(corrector instanceof GrammarCorrector
|| (corrector instanceof SchemaCorrector))) {
corrector.correct(queryCtx, semanticParseInfo);
}
});
log.info("chatQueryServiceImpl correct:{}", sqlInfo.getCorrectedS2SQL());
return semanticParseInfo;
}
@@ -174,8 +169,8 @@ public class S2ChatLayerService implements ChatLayerService {
return mapInfoResp;
}
private Map<String, DataSetMapInfo> getDataSetInfo(
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap, Integer topN) {
private Map<String, DataSetMapInfo> getDataSetInfo(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap, Integer topN) {
Map<String, DataSetMapInfo> map = new HashMap<>();
Map<Long, List<SchemaElementMatch>> mapFields = getMapFields(mapInfo, dataSetMap);
Map<Long, List<SchemaElementMatch>> topFields = getTopFields(topN, mapInfo, dataSetMap);
@@ -197,18 +192,15 @@ public class S2ChatLayerService implements ChatLayerService {
return map;
}
private Map<Long, List<SchemaElementMatch>> getMapFields(
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap) {
private Map<Long, List<SchemaElementMatch>> getMapFields(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
for (Map.Entry<Long, List<SchemaElementMatch>> entry :
mapInfo.getDataSetElementMatches().entrySet()) {
List<SchemaElementMatch> values =
entry.getValue().stream()
.filter(
schemaElementMatch ->
!SchemaElementType.TERM.equals(
schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches()
.entrySet()) {
List<SchemaElementMatch> values = entry.getValue().stream()
.filter(schemaElementMatch -> !SchemaElementType.TERM
.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(values) && dataSetMap.containsKey(entry.getKey())) {
result.put(entry.getKey(), values);
}
@@ -216,15 +208,15 @@ public class S2ChatLayerService implements ChatLayerService {
return result;
}
private Map<Long, List<SchemaElementMatch>> getTopFields(
Integer topN, SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetMap) {
private Map<Long, List<SchemaElementMatch>> getTopFields(Integer topN, SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
if (0 == topN) {
return result;
}
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
for (Map.Entry<Long, List<SchemaElementMatch>> entry :
mapInfo.getDataSetElementMatches().entrySet()) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches()
.entrySet()) {
Long dataSetId = entry.getKey();
List<SchemaElementMatch> values = entry.getValue();
DataSetResp dataSetResp = dataSetMap.get(dataSetId);
@@ -233,23 +225,17 @@ public class S2ChatLayerService implements ChatLayerService {
}
String dataSetName = dataSetResp.getName();
// topN dimensions
Set<SchemaElementMatch> dimensions =
semanticSchema.getDimensions(dataSetId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1)
.map(mergeFunction())
.collect(Collectors.toSet());
Set<SchemaElementMatch> dimensions = semanticSchema.getDimensions(dataSetId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
dimensions.add(timeDimensionMatch);
// topN metrics
Set<SchemaElementMatch> metrics =
semanticSchema.getMetrics(dataSetId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN)
.map(mergeFunction())
.collect(Collectors.toSet());
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()).limit(topN)
.map(mergeFunction()).collect(Collectors.toSet());
dimensions.addAll(metrics);
result.put(dataSetId, new ArrayList<>(dimensions));
@@ -257,8 +243,8 @@ public class S2ChatLayerService implements ChatLayerService {
return result;
}
private Map<String, List<SchemaElementMatch>> getTerms(
SchemaMapInfo mapInfo, Map<Long, DataSetResp> dataSetNameMap) {
private Map<String, List<SchemaElementMatch>> getTerms(SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetNameMap) {
Map<String, List<SchemaElementMatch>> termMap = new HashMap<>();
Map<Long, List<SchemaElementMatch>> dataSetElementMatches =
mapInfo.getDataSetElementMatches();
@@ -267,13 +253,10 @@ public class S2ChatLayerService implements ChatLayerService {
if (dataSetResp == null) {
continue;
}
List<SchemaElementMatch> terms =
entry.getValue().stream()
.filter(
schemaElementMatch ->
SchemaElementType.TERM.equals(
schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
List<SchemaElementMatch> terms = entry.getValue().stream()
.filter(schemaElementMatch -> SchemaElementType.TERM
.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
termMap.put(dataSetResp.getName(), terms);
}
return termMap;
@@ -287,34 +270,21 @@ public class S2ChatLayerService implements ChatLayerService {
* @return
*/
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
SchemaElement element =
SchemaElement.builder()
.dataSetId(dataSetId)
.dataSetName(dataSetName)
.type(SchemaElementType.DIMENSION)
.bizName(TimeDimensionEnum.DAY.getName())
.build();
SchemaElement element = SchemaElement.builder().dataSetId(dataSetId)
.dataSetName(dataSetName).type(SchemaElementType.DIMENSION)
.bizName(TimeDimensionEnum.DAY.getName()).build();
SchemaElementMatch timeDimensionMatch =
SchemaElementMatch.builder()
.element(element)
.detectWord(TimeDimensionEnum.DAY.getChName())
.word(TimeDimensionEnum.DAY.getChName())
.similarity(1L)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY)
.build();
SchemaElementMatch timeDimensionMatch = SchemaElementMatch.builder().element(element)
.detectWord(TimeDimensionEnum.DAY.getChName())
.word(TimeDimensionEnum.DAY.getChName()).similarity(1L)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
return timeDimensionMatch;
}
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
return schemaElement ->
SchemaElementMatch.builder()
.element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY)
.word(schemaElement.getName())
.similarity(1)
.detectWord(schemaElement.getName())
.build();
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName())
.similarity(1).detectWord(schemaElement.getName()).build();
}
}

View File

@@ -85,17 +85,11 @@ public class S2SemanticLayerService implements SemanticLayerService {
private final QueryCache queryCache = ComponentFactory.getQueryCache();
private final List<QueryExecutor> queryExecutors = ComponentFactory.getQueryExecutors();
public S2SemanticLayerService(
StatUtils statUtils,
QueryUtils queryUtils,
QueryReqConverter queryReqConverter,
SemanticSchemaManager semanticSchemaManager,
DataSetService dataSetService,
SchemaService schemaService,
SemanticTranslator semanticTranslator,
MetricDrillDownChecker metricDrillDownChecker,
KnowledgeBaseService knowledgeBaseService,
MetricService metricService,
public S2SemanticLayerService(StatUtils statUtils, QueryUtils queryUtils,
QueryReqConverter queryReqConverter, SemanticSchemaManager semanticSchemaManager,
DataSetService dataSetService, SchemaService schemaService,
SemanticTranslator semanticTranslator, MetricDrillDownChecker metricDrillDownChecker,
KnowledgeBaseService knowledgeBaseService, MetricService metricService,
DimensionService dimensionService) {
this.statUtils = statUtils;
this.queryUtils = queryUtils;
@@ -119,11 +113,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
public SemanticTranslateResp translate(SemanticQueryReq queryReq, User user) throws Exception {
QueryStatement queryStatement = buildQueryStatement(queryReq, user);
semanticTranslator.translate(queryStatement);
return SemanticTranslateResp.builder()
.querySQL(queryStatement.getSql())
.isOk(queryStatement.isOk())
.errMsg(queryStatement.getErrMsg())
.build();
return SemanticTranslateResp.builder().querySQL(queryStatement.getSql())
.isOk(queryStatement.isOk()).errMsg(queryStatement.getErrMsg()).build();
}
@Override
@@ -161,8 +152,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
for (QueryExecutor queryExecutor : queryExecutors) {
if (queryExecutor.accept(queryStatement)) {
queryResp = queryExecutor.execute(queryStatement);
queryUtils.populateQueryColumns(
queryResp, queryStatement.getSemanticSchemaResp());
queryUtils.populateQueryColumns(queryResp,
queryStatement.getSemanticSchemaResp());
}
}
@@ -208,8 +199,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
return semanticQueryResp;
}
private List<String> getDimensionValuesFromDict(
DimensionValueReq dimensionValueReq, Set<Long> dataSetIds) {
private List<String> getDimensionValuesFromDict(DimensionValueReq dimensionValueReq,
Set<Long> dataSetIds) {
if (StringUtils.isBlank(dimensionValueReq.getValue())) {
return SearchService.getDimensionValue(dimensionValueReq);
}
@@ -217,28 +208,19 @@ public class S2SemanticLayerService implements SemanticLayerService {
Map<Long, List<Long>> modelIdToDataSetIds = new HashMap<>();
modelIdToDataSetIds.put(dimensionValueReq.getModelId(), new ArrayList<>(dataSetIds));
List<HanlpMapResult> hanlpMapResultList =
knowledgeBaseService.prefixSearch(
dimensionValueReq.getValue(), 2000, modelIdToDataSetIds, dataSetIds);
List<HanlpMapResult> hanlpMapResultList = knowledgeBaseService
.prefixSearch(dimensionValueReq.getValue(), 2000, modelIdToDataSetIds, dataSetIds);
HanlpHelper.transLetterOriginal(hanlpMapResultList);
return hanlpMapResultList.stream()
.filter(
o ->
o.getNatures().stream()
.map(NatureHelper::getElementID)
.anyMatch(
elementID ->
dimensionValueReq
.getElementID()
.equals(elementID)))
.map(MapResult::getName)
.collect(Collectors.toList());
.filter(o -> o.getNatures().stream().map(NatureHelper::getElementID)
.anyMatch(elementID -> dimensionValueReq.getElementID().equals(elementID)))
.map(MapResult::getName).collect(Collectors.toList());
}
private SemanticQueryResp getDimensionValuesFromDb(
DimensionValueReq dimensionValueReq, User user) {
private SemanticQueryResp getDimensionValuesFromDb(DimensionValueReq dimensionValueReq,
User user) {
QuerySqlReq querySqlReq = buildQuerySqlReq(dimensionValueReq);
return queryByReq(querySqlReq, user);
}
@@ -255,16 +237,13 @@ public class S2SemanticLayerService implements SemanticLayerService {
return columns;
}
private List<Map<String, Object>> createResultList(
DimensionValueReq dimensionValueReq, List<String> dimensionValues) {
return dimensionValues.stream()
.map(
value -> {
Map<String, Object> map = new HashMap<>();
map.put(dimensionValueReq.getBizName(), value);
return map;
})
.collect(Collectors.toList());
private List<Map<String, Object>> createResultList(DimensionValueReq dimensionValueReq,
List<String> dimensionValues) {
return dimensionValues.stream().map(value -> {
Map<String, Object> map = new HashMap<>();
map.put(dimensionValueReq.getBizName(), value);
return map;
}).collect(Collectors.toList());
}
private DimensionResp getDimension(DimensionValueReq dimensionValueReq) {
@@ -278,8 +257,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
return dimensionResp;
}
public EntityInfo getEntityInfo(
SemanticParseInfo parseInfo, DataSetSchema dataSetSchema, User user) {
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, DataSetSchema dataSetSchema,
User user) {
if (parseInfo != null && parseInfo.getDataSetId() != null && parseInfo.getDataSetId() > 0) {
EntityInfo entityInfo = getEntityBasicInfo(dataSetSchema);
if (parseInfo.getDimensionFilters().size() <= 0
@@ -292,8 +271,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
if (StringUtils.isNotBlank(primaryKey)) {
String entityId = "";
for (QueryFilter chatFilter : parseInfo.getDimensionFilters()) {
if (chatFilter != null
&& chatFilter.getBizName() != null
if (chatFilter != null && chatFilter.getBizName() != null
&& chatFilter.getBizName().equals(primaryKey)) {
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
entityId = chatFilter.getValue().toString();
@@ -377,8 +355,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
queryStatement =
buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq, user);
}
if (Objects.nonNull(queryStatement)
&& Objects.nonNull(semanticQueryReq.getSqlInfo())
if (Objects.nonNull(queryStatement) && Objects.nonNull(semanticQueryReq.getSqlInfo())
&& StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) {
queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
@@ -402,8 +379,8 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryStatement;
}
private QueryStatement buildMultiStructQueryStatement(
QueryMultiStructReq queryMultiStructReq, User user) throws Exception {
private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq,
User user) throws Exception {
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq, user);
@@ -429,32 +406,20 @@ public class S2SemanticLayerService implements SemanticLayerService {
QuerySqlReq querySqlReq = new QuerySqlReq();
List<ModelResp> modelResps =
schemaService.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp =
schemaService.getDimension(
queryDimValueReq.getBizName(), queryDimValueReq.getModelId());
DimensionResp dimensionResp = schemaService.getDimension(queryDimValueReq.getBizName(),
queryDimValueReq.getModelId());
ModelResp modelResp = modelResps.get(0);
String sql =
String.format(
"select distinct %s from %s where 1=1",
dimensionResp.getName(), modelResp.getName());
String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(),
modelResp.getName());
List<Dim> timeDims = modelResp.getTimeDimension();
if (CollectionUtils.isNotEmpty(timeDims)) {
sql =
String.format(
"%s and %s >= '%s' and %s <= '%s'",
sql,
TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getStartDate(),
TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getEndDate());
sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(),
TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate());
}
if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
sql +=
" AND "
+ queryDimValueReq.getBizName()
+ " LIKE '%"
+ queryDimValueReq.getValue()
+ "%'";
sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%"
+ queryDimValueReq.getValue() + "%'";
}
querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
querySqlReq.setSql(sql);
@@ -488,48 +453,32 @@ public class S2SemanticLayerService implements SemanticLayerService {
|| detailTypeDefaultConfig.getDefaultDisplayInfo() == null) {
return entityInfo;
}
List<DataInfo> dimensions =
detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(
id -> {
SchemaElement element =
dataSetSchema.getElement(
SchemaElementType.DIMENSION, id);
if (element == null) {
return null;
}
return new DataInfo(
element.getId().intValue(),
element.getName(),
element.getBizName(),
null);
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
List<DataInfo> metrics =
detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds().stream()
.map(
id -> {
SchemaElement element =
dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (element == null) {
return null;
}
return new DataInfo(
element.getId().intValue(),
element.getName(),
element.getBizName(),
null);
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
List<DataInfo> dimensions = detailTypeDefaultConfig.getDefaultDisplayInfo()
.getDimensionIds().stream().map(id -> {
SchemaElement element =
dataSetSchema.getElement(SchemaElementType.DIMENSION, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(),
element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
List<DataInfo> metrics = detailTypeDefaultConfig.getDefaultDisplayInfo().getDimensionIds()
.stream().map(id -> {
SchemaElement element = dataSetSchema.getElement(SchemaElementType.METRIC, id);
if (element == null) {
return null;
}
return new DataInfo(element.getId().intValue(), element.getName(),
element.getBizName(), null);
}).filter(Objects::nonNull).collect(Collectors.toList());
entityInfo.setDimensions(dimensions);
entityInfo.setMetrics(metrics);
return entityInfo;
}
private void fillEntityInfoValue(
EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
private void fillEntityInfoValue(EntityInfo entityInfo, DataSetSchema dataSetSchema,
User user) {
SemanticQueryResp queryResultWithColumns =
getQueryResultWithSchemaResp(entityInfo, dataSetSchema, user);
if (queryResultWithColumns != null) {
@@ -540,19 +489,17 @@ public class S2SemanticLayerService implements SemanticLayerService {
if (entry.getValue() == null || entryKey == null) {
continue;
}
entityInfo.getDimensions().stream()
.filter(i -> entryKey.equals(i.getBizName()))
entityInfo.getDimensions().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
entityInfo.getMetrics().stream()
.filter(i -> entryKey.equals(i.getBizName()))
entityInfo.getMetrics().stream().filter(i -> entryKey.equals(i.getBizName()))
.forEach(i -> i.setValue(entry.getValue().toString()));
}
}
}
}
private SemanticQueryResp getQueryResultWithSchemaResp(
EntityInfo entityInfo, DataSetSchema dataSetSchema, User user) {
private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
DataSetSchema dataSetSchema, User user) {
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
semanticParseInfo.setQueryType(QueryType.DETAIL);

View File

@@ -21,9 +21,11 @@ import java.util.List;
@Slf4j
public class MetaEmbeddingListener implements ApplicationListener<DataEvent> {
@Autowired private EmbeddingConfig embeddingConfig;
@Autowired
private EmbeddingConfig embeddingConfig;
@Autowired private EmbeddingService embeddingService;
@Autowired
private EmbeddingService embeddingService;
@Value("${s2.embedding.operation.sleep.time:3000}")
private Integer embeddingOperationSleepTime;

View File

@@ -22,28 +22,24 @@ public class SchemaDictUpdateListener implements ApplicationListener<DataEvent>
if (CollectionUtils.isEmpty(dataEvent.getDataItems())) {
return;
}
dataEvent
.getDataItems()
.forEach(
dataItem -> {
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());
String sign = DictWordType.NATURE_SPILT;
String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
String nature =
sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);
if (EventType.ADD.equals(dataEvent.getEventType())) {
HanlpHelper.addToCustomDictionary(dictWord);
} else if (EventType.DELETE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
} else if (EventType.UPDATE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
dictWord.setWord(dataItem.getNewName());
HanlpHelper.addToCustomDictionary(dictWord);
}
});
dataEvent.getDataItems().forEach(dataItem -> {
DictWord dictWord = new DictWord();
dictWord.setWord(dataItem.getName());
String sign = DictWordType.NATURE_SPILT;
String suffixNature = DictWordType.getSuffixNature(dataItem.getType());
String nature = sign + dataItem.getModelId() + dataItem.getId() + suffixNature;
String natureWithFrequency = nature + " " + Constants.DEFAULT_FREQUENCY;
dictWord.setNature(nature);
dictWord.setNatureWithFrequency(natureWithFrequency);
if (EventType.ADD.equals(dataEvent.getEventType())) {
HanlpHelper.addToCustomDictionary(dictWord);
} else if (EventType.DELETE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
} else if (EventType.UPDATE.equals(dataEvent.getEventType())) {
HanlpHelper.removeFromCustomDictionary(dictWord);
dictWord.setWord(dataItem.getNewName());
HanlpHelper.addToCustomDictionary(dictWord);
}
});
}
}

View File

@@ -22,14 +22,9 @@ public class DimensionYamlManager {
return new ArrayList<>();
}
return dimensions.stream()
.filter(
dimension ->
!dimension
.getType()
.name()
.equalsIgnoreCase(IdentifyType.primary.name()))
.map(DimensionYamlManager::convert2DimensionYamlTpl)
.collect(Collectors.toList());
.filter(dimension -> !dimension.getType().name()
.equalsIgnoreCase(IdentifyType.primary.name()))
.map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList());
}
public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) {

View File

@@ -28,8 +28,8 @@ import java.util.stream.Collectors;
@Slf4j
public class ModelYamlManager {
public static synchronized DataModelYamlTpl convert2YamlObj(
ModelResp modelResp, DatabaseResp databaseResp) {
public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp,
DatabaseResp databaseResp) {
ModelDetail modelDetail = modelResp.getModelDetail();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
@@ -37,18 +37,12 @@ public class ModelYamlManager {
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
dataModelYamlTpl.setType(databaseResp.getType());
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);
dataModelYamlTpl.setIdentifiers(
modelDetail.getIdentifiers().stream()
.map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setDimensions(
modelDetail.getDimensions().stream()
.map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setMeasures(
modelDetail.getMeasures().stream()
.map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setIdentifiers(modelDetail.getIdentifiers().stream()
.map(ModelYamlManager::convert).collect(Collectors.toList()));
dataModelYamlTpl.setDimensions(modelDetail.getDimensions().stream()
.map(ModelYamlManager::convert).collect(Collectors.toList()));
dataModelYamlTpl.setMeasures(modelDetail.getMeasures().stream()
.map(ModelYamlManager::convert).collect(Collectors.toList()));
dataModelYamlTpl.setName(modelResp.getBizName());
dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
if (modelDetail.getQueryType().equalsIgnoreCase(ModelDefineType.SQL_QUERY.getName())) {

View File

@@ -64,12 +64,8 @@ public class SemanticSchemaManager {
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
Map<Long, String> modelIdName = new HashMap<>();
schemaService.getSchemaYamlTpl(
semanticSchemaResp,
dimensionYamlTpls,
dataModelYamlTpls,
metricYamlTpls,
modelIdName);
schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls, dataModelYamlTpls,
metricYamlTpls, modelIdName);
DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp();
semanticModel.setDatabase(DatabaseConverter.convert(databaseResp));
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) {
@@ -78,11 +74,8 @@ public class SemanticSchemaManager {
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap =
dataModelYamlTpls.stream()
.map(SemanticSchemaManager::getDatasource)
.collect(
Collectors.toMap(
DataSource::getName, item -> item, (k1, k2) -> k1));
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect(
Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
semanticModel.setDatasourceMap(dataSourceMap);
}
if (!dimensionYamlTpls.isEmpty()) {
@@ -114,8 +107,8 @@ public class SemanticSchemaManager {
}
if (Objects.nonNull(semanticModel.getDatasourceMap())
&& !semanticModel.getDatasourceMap().isEmpty()) {
for (Map.Entry<String, DataSource> entry :
semanticModel.getDatasourceMap().entrySet()) {
for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap()
.entrySet()) {
List<Dimension> modelDimensions = new ArrayList<>();
if (!semanticModel.getDimensionMap().containsKey(entry.getKey())) {
semanticModel.getDimensionMap().put(entry.getKey(), modelDimensions);
@@ -133,18 +126,16 @@ public class SemanticSchemaManager {
return semanticModel;
}
private void addTagModel(
TagResp tagResp, List<Dimension> modelDimensions, List<Metric> modelMetrics)
throws Exception {
private void addTagModel(TagResp tagResp, List<Dimension> modelDimensions,
List<Metric> modelMetrics) throws Exception {
TagDefineType tagDefineType = TagDefineType.valueOf(tagResp.getTagDefineType());
switch (tagDefineType) {
case FIELD:
case DIMENSION:
if (TagDefineType.DIMENSION.equals(tagResp.getTagDefineType())) {
Optional<Dimension> modelDimension =
modelDimensions.stream()
// .filter(d -> d.getBizName().equals(tagResp.getExpr()))
.findFirst();
Optional<Dimension> modelDimension = modelDimensions.stream()
// .filter(d -> d.getBizName().equals(tagResp.getExpr()))
.findFirst();
if (modelDimension.isPresent()) {
modelDimension.get().setName(tagResp.getBizName());
return;
@@ -152,7 +143,7 @@ public class SemanticSchemaManager {
}
Dimension dimension = Dimension.builder().build();
dimension.setType("");
// dimension.setExpr(tagResp.getExpr());
// dimension.setExpr(tagResp.getExpr());
dimension.setName(tagResp.getBizName());
dimension.setOwners("");
dimension.setBizName(tagResp.getBizName());
@@ -165,10 +156,9 @@ public class SemanticSchemaManager {
modelDimensions.add(dimension);
return;
case METRIC:
Optional<Metric> modelMetric =
modelMetrics.stream()
// .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr()))
.findFirst();
Optional<Metric> modelMetric = modelMetrics.stream()
// .filter(m -> m.getName().equalsIgnoreCase(tagResp.getExpr()))
.findFirst();
if (modelMetric.isPresent()) {
modelMetric.get().setName(tagResp.getBizName());
} else {
@@ -189,37 +179,22 @@ public class SemanticSchemaManager {
}
public static DataSource getDatasource(final DataModelYamlTpl d) {
DataSource datasource =
DataSource.builder()
.id(d.getId())
.sourceId(d.getSourceId())
.type(d.getType())
.sqlQuery(d.getSqlQuery())
.name(d.getName())
.tableQuery(d.getTableQuery())
.identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasureParams(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions()))
.build();
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId())
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName())
.tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers()))
.measures(getMeasureParams(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions())).build();
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
}
if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
Set<String> measures =
datasource.getMeasures().stream()
.map(mm -> mm.getName())
.collect(Collectors.toSet());
Set<String> measures = datasource.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet());
for (Field f : d.getFields()) {
if (!measures.contains(f.getFieldName())) {
datasource
.getMeasures()
.add(
Measure.builder()
.expr(f.getFieldName())
.name(f.getFieldName())
.agg("")
.build());
datasource.getMeasures().add(Measure.builder().expr(f.getFieldName())
.name(f.getFieldName()).agg("").build());
}
}
}
@@ -227,10 +202,9 @@ public class SemanticSchemaManager {
}
private static String getDataSourceAggTime(List<Dimension> dimensions) {
Optional<Dimension> timeDimension =
dimensions.stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
Optional<Dimension> timeDimension = dimensions.stream()
.filter(d -> Constants.DIMENSION_TYPE_TIME.equalsIgnoreCase(d.getType()))
.findFirst();
if (timeDimension.isPresent()
&& Objects.nonNull(timeDimension.get().getDimensionTimeTypeParams())) {
return timeDimension.get().getDimensionTimeTypeParams().getTimeGranularity();
@@ -340,8 +314,8 @@ public class SemanticSchemaManager {
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl) {
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
if (dimensionTimeTypeParamsTpl != null) {
dimensionTimeTypeParams.setTimeGranularity(
dimensionTimeTypeParamsTpl.getTimeGranularity());
dimensionTimeTypeParams
.setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity());
dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary());
}
return dimensionTimeTypeParams;
@@ -358,38 +332,27 @@ public class SemanticSchemaManager {
return identifies;
}
private static List<JoinRelation> getJoinRelation(
List<ModelRela> modelRelas, Map<Long, String> modelIdName) {
private static List<JoinRelation> getJoinRelation(List<ModelRela> modelRelas,
Map<Long, String> modelIdName) {
List<JoinRelation> joinRelations = new ArrayList<>();
modelRelas.stream()
.forEach(
r -> {
if (modelIdName.containsKey(r.getFromModelId())
&& modelIdName.containsKey(r.getToModelId())) {
JoinRelation joinRelation =
JoinRelation.builder()
.left(modelIdName.get(r.getFromModelId()))
.right(modelIdName.get(r.getToModelId()))
.joinType(r.getJoinType())
.build();
List<Triple<String, String, String>> conditions = new ArrayList<>();
r.getJoinConditions().stream()
.forEach(
rr -> {
if (FilterOperatorEnum.isValueCompare(
rr.getOperator())) {
conditions.add(
Triple.of(
rr.getLeftField(),
rr.getOperator().getValue(),
rr.getRightField()));
}
});
joinRelation.setId(r.getId());
joinRelation.setJoinCondition(conditions);
joinRelations.add(joinRelation);
}
});
modelRelas.stream().forEach(r -> {
if (modelIdName.containsKey(r.getFromModelId())
&& modelIdName.containsKey(r.getToModelId())) {
JoinRelation joinRelation = JoinRelation.builder()
.left(modelIdName.get(r.getFromModelId()))
.right(modelIdName.get(r.getToModelId())).joinType(r.getJoinType()).build();
List<Triple<String, String, String>> conditions = new ArrayList<>();
r.getJoinConditions().stream().forEach(rr -> {
if (FilterOperatorEnum.isValueCompare(rr.getOperator())) {
conditions.add(Triple.of(rr.getLeftField(), rr.getOperator().getValue(),
rr.getRightField()));
}
});
joinRelation.setId(r.getId());
joinRelation.setJoinCondition(conditions);
joinRelations.add(joinRelation);
}
});
return joinRelations;
}
@@ -405,8 +368,7 @@ public class SemanticSchemaManager {
String dataSourceName = datasourceYamlTpl.getName();
Optional<Entry<String, DataSource>> datasourceYamlTplMap =
schema.getDatasource().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName))
.findFirst();
.filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst();
if (datasourceYamlTplMap.isPresent()) {
datasourceYamlTplMap.get().setValue(datasourceYamlTpl);
} else {
@@ -415,14 +377,12 @@ public class SemanticSchemaManager {
}
}
public static void update(
SemanticSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
throws Exception {
public static void update(SemanticSchema schema, String datasourceBizName,
List<Dimension> dimensionYamlTpls) throws Exception {
if (schema != null) {
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap =
schema.getDimension().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName))
.findFirst();
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema
.getDimension().entrySet().stream()
.filter(t -> t.getKey().equalsIgnoreCase(datasourceBizName)).findFirst();
if (datasourceYamlTplMap.isPresent()) {
updateDimension(dimensionYamlTpls, datasourceYamlTplMap.get().getValue());
} else {
@@ -433,8 +393,8 @@ public class SemanticSchemaManager {
}
}
private static void updateDimension(
List<Dimension> dimensionYamlTpls, List<Dimension> dimensions) {
private static void updateDimension(List<Dimension> dimensionYamlTpls,
List<Dimension> dimensions) {
if (CollectionUtils.isEmpty(dimensionYamlTpls)) {
return;
}

View File

@@ -139,8 +139,8 @@ public class DatabaseDOExample {
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(
String condition, Object value1, Object value2, String property) {
protected void addCriterion(String condition, Object value1, Object value2,
String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
@@ -969,8 +969,8 @@ public class DatabaseDOExample {
this(condition, value, null);
}
protected Criterion(
String condition, Object value, Object secondValue, String typeHandler) {
protected Criterion(String condition, Object value, Object secondValue,
String typeHandler) {
super();
this.condition = condition;
this.value = value;

View File

@@ -139,8 +139,8 @@ public class DomainDOExample {
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(
String condition, Object value1, Object value2, String property) {
protected void addCriterion(String condition, Object value1, Object value2,
String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
@@ -1117,8 +1117,8 @@ public class DomainDOExample {
this(condition, value, null);
}
protected Criterion(
String condition, Object value, Object secondValue, String typeHandler) {
protected Criterion(String condition, Object value, Object secondValue,
String typeHandler) {
super();
this.condition = condition;
this.value = value;

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.AppDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface AppMapper extends BaseMapper<AppDO> {}
public interface AppMapper extends BaseMapper<AppDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.CanvasDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface CanvasDOMapper extends BaseMapper<CanvasDO> {}
public interface CanvasDOMapper extends BaseMapper<CanvasDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ClassDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ClassMapper extends BaseMapper<ClassDO> {}
public interface ClassMapper extends BaseMapper<ClassDO> {
}

View File

@@ -11,4 +11,5 @@ import org.apache.ibatis.annotations.Mapper;
* @since 2023-11-09 03:49:33
*/
@Mapper
public interface CollectMapper extends BaseMapper<CollectDO> {}
public interface CollectMapper extends BaseMapper<CollectDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DataSetDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DataSetDOMapper extends BaseMapper<DataSetDO> {}
public interface DataSetDOMapper extends BaseMapper<DataSetDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DatabaseDOMapper extends BaseMapper<DatabaseDO> {}
public interface DatabaseDOMapper extends BaseMapper<DatabaseDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DictConfMapper extends BaseMapper<DictConfDO> {}
public interface DictConfMapper extends BaseMapper<DictConfDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DictTaskMapper extends BaseMapper<DictTaskDO> {}
public interface DictTaskMapper extends BaseMapper<DictTaskDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DimensionDOMapper extends BaseMapper<DimensionDO> {}
public interface DimensionDOMapper extends BaseMapper<DimensionDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DomainDOMapper extends BaseMapper<DomainDO> {}
public interface DomainDOMapper extends BaseMapper<DomainDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface MetricDOMapper extends BaseMapper<MetricDO> {}
public interface MetricDOMapper extends BaseMapper<MetricDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQuery
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface MetricQueryDefaultConfigDOMapper extends BaseMapper<MetricQueryDefaultConfigDO> {}
public interface MetricQueryDefaultConfigDOMapper extends BaseMapper<MetricQueryDefaultConfigDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ModelDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ModelDOMapper extends BaseMapper<ModelDO> {}
public interface ModelDOMapper extends BaseMapper<ModelDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.ModelRelaDO
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface ModelRelaDOMapper extends BaseMapper<ModelRelaDO> {}
public interface ModelRelaDOMapper extends BaseMapper<ModelRelaDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.QueryRuleDO
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface QueryRuleMapper extends BaseMapper<QueryRuleDO> {}
public interface QueryRuleMapper extends BaseMapper<QueryRuleDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TagMapper extends BaseMapper<TagDO> {}
public interface TagMapper extends BaseMapper<TagDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TagObjectDO
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TagObjectMapper extends BaseMapper<TagObjectDO> {}
public interface TagObjectMapper extends BaseMapper<TagObjectDO> {
}

View File

@@ -5,4 +5,5 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.TermDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TermMapper extends BaseMapper<TermDO> {}
public interface TermMapper extends BaseMapper<TermDO> {
}

View File

@@ -26,7 +26,8 @@ public class DateInfoRepositoryImpl implements DateInfoRepository {
private ObjectMapper mapper = new ObjectMapper();
@Autowired private DateInfoMapper dateInfoMapper;
@Autowired
private DateInfoMapper dateInfoMapper;
@Override
public Integer upsertDateInfo(List<DateInfoReq> dateInfoCommends) {
@@ -36,22 +37,19 @@ public class DateInfoRepositoryImpl implements DateInfoRepository {
return 0;
}
dateInfoCommends.stream()
.forEach(
commend -> {
DateInfoDO dateInfoDO = new DateInfoDO();
BeanUtils.copyProperties(commend, dateInfoDO);
try {
dateInfoDO.setUnavailableDateList(
mapper.writeValueAsString(
commend.getUnavailableDateList()));
dateInfoDO.setCreatedBy(Constants.ADMIN_LOWER);
dateInfoDO.setUpdatedBy(Constants.ADMIN_LOWER);
} catch (JsonProcessingException e) {
log.info("e,", e);
}
dateInfoDOList.add(dateInfoDO);
});
dateInfoCommends.stream().forEach(commend -> {
DateInfoDO dateInfoDO = new DateInfoDO();
BeanUtils.copyProperties(commend, dateInfoDO);
try {
dateInfoDO.setUnavailableDateList(
mapper.writeValueAsString(commend.getUnavailableDateList()));
dateInfoDO.setCreatedBy(Constants.ADMIN_LOWER);
dateInfoDO.setUpdatedBy(Constants.ADMIN_LOWER);
} catch (JsonProcessingException e) {
log.info("e,", e);
}
dateInfoDOList.add(dateInfoDO);
});
return batchUpsert(dateInfoDOList);
}

View File

@@ -36,11 +36,8 @@ public class DictRepositoryImpl implements DictRepository {
private final DictUtils dictConverter;
private final DimensionService dimensionService;
public DictRepositoryImpl(
DictTaskMapper dictTaskMapper,
DictConfMapper dictConfMapper,
DictUtils dictConverter,
DimensionService dimensionService) {
public DictRepositoryImpl(DictTaskMapper dictTaskMapper, DictConfMapper dictConfMapper,
DictUtils dictConverter, DimensionService dimensionService) {
this.dictTaskMapper = dictTaskMapper;
this.dictConfMapper = dictConfMapper;
this.dictConverter = dictConverter;
@@ -90,11 +87,9 @@ public class DictRepositoryImpl implements DictRepository {
QueryWrapper<DictTaskDO> wrapper = new QueryWrapper<>();
wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId());
wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType());
List<DictTaskDO> dictTaskDOList =
dictTaskMapper.selectList(wrapper).stream()
.sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
.limit(dictTaskNum)
.collect(Collectors.toList());
List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper).stream()
.sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
.limit(dictTaskNum).collect(Collectors.toList());
if (CollectionUtils.isEmpty(dictTaskDOList)) {
return taskResp;
}
@@ -114,10 +109,8 @@ public class DictRepositoryImpl implements DictRepository {
@Override
public Long editDictConf(DictConfDO dictConfDO) {
DictItemFilter filter =
DictItemFilter.builder()
.type(TypeEnums.valueOf(dictConfDO.getType()))
.itemId(dictConfDO.getItemId())
.build();
DictItemFilter.builder().type(TypeEnums.valueOf(dictConfDO.getType()))
.itemId(dictConfDO.getItemId()).build();
List<DictConfDO> dictConfDOList = getDictConfDOList(filter);
if (CollectionUtils.isEmpty(dictConfDOList)) {

View File

@@ -17,8 +17,8 @@ public class DimensionRepositoryImpl implements DimensionRepository {
private DimensionDOCustomMapper dimensionDOCustomMapper;
public DimensionRepositoryImpl(
DimensionDOMapper dimensionDOMapper, DimensionDOCustomMapper dimensionDOCustomMapper) {
public DimensionRepositoryImpl(DimensionDOMapper dimensionDOMapper,
DimensionDOCustomMapper dimensionDOCustomMapper) {
this.dimensionDOMapper = dimensionDOMapper;
this.dimensionDOCustomMapper = dimensionDOCustomMapper;
}

View File

@@ -22,8 +22,7 @@ public class MetricRepositoryImpl implements MetricRepository {
private MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper;
public MetricRepositoryImpl(
MetricDOMapper metricDOMapper,
public MetricRepositoryImpl(MetricDOMapper metricDOMapper,
MetricDOCustomMapper metricDOCustomMapper,
MetricQueryDefaultConfigDOMapper metricQueryDefaultConfigDOMapper) {
this.metricDOMapper = metricDOMapper;
@@ -95,9 +94,7 @@ public class MetricRepositoryImpl implements MetricRepository {
@Override
public MetricQueryDefaultConfigDO getDefaultQueryConfig(Long metricId, String userName) {
QueryWrapper<MetricQueryDefaultConfigDO> queryWrapper = new QueryWrapper<>();
queryWrapper
.lambda()
.eq(MetricQueryDefaultConfigDO::getMetricId, metricId)
queryWrapper.lambda().eq(MetricQueryDefaultConfigDO::getMetricId, metricId)
.eq(MetricQueryDefaultConfigDO::getCreatedBy, userName);
return metricQueryDefaultConfigDOMapper.selectOne(queryWrapper);
}

View File

@@ -19,8 +19,8 @@ public class ModelRepositoryImpl implements ModelRepository {
private ModelDOCustomMapper modelDOCustomMapper;
public ModelRepositoryImpl(
ModelDOMapper modelDOMapper, ModelDOCustomMapper modelDOCustomMapper) {
public ModelRepositoryImpl(ModelDOMapper modelDOMapper,
ModelDOCustomMapper modelDOCustomMapper) {
this.modelDOMapper = modelDOMapper;
this.modelDOCustomMapper = modelDOCustomMapper;
}

View File

@@ -45,34 +45,24 @@ public class StatRepositoryImpl implements StatRepository {
List<ItemUseResp> result = new ArrayList<>();
List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
Map<String, Long> map = new ConcurrentHashMap<>();
statInfos.stream()
.forEach(
stat -> {
String dimensions = stat.getDimensions();
String metrics = stat.getMetrics();
if (Objects.nonNull(stat.getDataSetId())) {
updateStatMapInfo(
map,
dimensions,
TypeEnums.DIMENSION.name().toLowerCase(),
stat.getDataSetId());
updateStatMapInfo(
map,
metrics,
TypeEnums.METRIC.name().toLowerCase(),
stat.getDataSetId());
}
});
map.forEach(
(k, v) -> {
Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]);
String type = k.split(AT_SYMBOL + AT_SYMBOL)[1];
String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2];
result.add(new ItemUseResp(classId, type, nameEn, v));
});
statInfos.stream().forEach(stat -> {
String dimensions = stat.getDimensions();
String metrics = stat.getMetrics();
if (Objects.nonNull(stat.getDataSetId())) {
updateStatMapInfo(map, dimensions, TypeEnums.DIMENSION.name().toLowerCase(),
stat.getDataSetId());
updateStatMapInfo(map, metrics, TypeEnums.METRIC.name().toLowerCase(),
stat.getDataSetId());
}
});
map.forEach((k, v) -> {
Long classId = Long.parseLong(k.split(AT_SYMBOL + AT_SYMBOL)[0]);
String type = k.split(AT_SYMBOL + AT_SYMBOL)[1];
String nameEn = k.split(AT_SYMBOL + AT_SYMBOL)[2];
result.add(new ItemUseResp(classId, type, nameEn, v));
});
return result.stream()
.sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
return result.stream().sorted(Comparator.comparing(ItemUseResp::getUseCnt).reversed())
.collect(Collectors.toList());
}
@@ -81,24 +71,21 @@ public class StatRepositoryImpl implements StatRepository {
return statMapper.getStatInfo(itemUseCommend);
}
private void updateStatMapInfo(
Map<String, Long> map, String dimensions, String type, Long dataSetId) {
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type,
Long dataSetId) {
if (StringUtils.isNotEmpty(dimensions)) {
try {
List<String> dimensionList =
mapper.readValue(dimensions, new TypeReference<List<String>>() {});
dimensionList.stream()
.forEach(
dimension -> {
String key =
dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL
+ AT_SYMBOL + dimension;
if (map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {
map.put(key, 1L);
}
});
dimensionList.stream().forEach(dimension -> {
String key = dataSetId + AT_SYMBOL + AT_SYMBOL + type + AT_SYMBOL + AT_SYMBOL
+ dimension;
if (map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {
map.put(key, 1L);
}
});
} catch (Exception e) {
log.warn("e:{}", e);
}

View File

@@ -63,8 +63,8 @@ public class TagRepositoryImpl implements TagRepository {
}
if (Objects.nonNull(tagDeleteReq.getTagDefineType())
&& CollectionUtils.isNotEmpty(tagDeleteReq.getItemIds())) {
tagCustomMapper.deleteBatchByType(
tagDeleteReq.getItemIds(), tagDeleteReq.getTagDefineType().name());
tagCustomMapper.deleteBatchByType(tagDeleteReq.getItemIds(),
tagDeleteReq.getTagDefineType().name());
}
}
}

View File

@@ -16,24 +16,18 @@ public class EntityInfoProcessor implements ResultProcessor {
@Override
public void process(ParseResp parseResp, ChatQueryContext chatQueryContext) {
parseResp
.getSelectedParses()
.forEach(
parseInfo -> {
String queryMode = parseInfo.getQueryMode();
if (!QueryManager.isDetailQuery(queryMode)
&& !QueryManager.isMetricQuery(queryMode)) {
return;
}
parseResp.getSelectedParses().forEach(parseInfo -> {
String queryMode = parseInfo.getQueryMode();
if (!QueryManager.isDetailQuery(queryMode) && !QueryManager.isMetricQuery(queryMode)) {
return;
}
SemanticLayerService semanticService =
ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dataSetSchema =
semanticService.getDataSetSchema(parseInfo.getDataSetId());
EntityInfo entityInfo =
semanticService.getEntityInfo(
parseInfo, dataSetSchema, chatQueryContext.getUser());
parseInfo.setEntityInfo(entityInfo);
});
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
DataSetSchema dataSetSchema =
semanticService.getDataSetSchema(parseInfo.getDataSetId());
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema,
chatQueryContext.getUser());
parseInfo.setEntityInfo(entityInfo);
});
}
}

View File

@@ -82,39 +82,33 @@ public class ParseInfoProcessor implements ResultProcessor {
} else if (QueryType.DETAIL.equals(parseInfo.getQueryType())) {
List<String> selectFields = SqlSelectHelper.getSelectFields(s2SQL);
List<String> selectDimensions = filterDateField(dsSchema, selectFields);
parseInfo.setDimensions(
matchSchemaElements(selectDimensions, dsSchema.getDimensions()));
parseInfo
.setDimensions(matchSchemaElements(selectDimensions, dsSchema.getDimensions()));
}
}
private Set<SchemaElement> matchSchemaElements(
List<String> allFields, Set<SchemaElement> elements) {
return elements.stream()
.filter(
schemaElement -> {
if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
return allFields.contains(schemaElement.getName());
}
Set<String> allFieldsSet = new HashSet<>(allFields);
Set<String> aliasSet = new HashSet<>(schemaElement.getAlias());
List<String> intersection =
allFieldsSet.stream()
.filter(aliasSet::contains)
.collect(Collectors.toList());
return allFields.contains(schemaElement.getName())
|| !CollectionUtils.isEmpty(intersection);
})
.collect(Collectors.toSet());
private Set<SchemaElement> matchSchemaElements(List<String> allFields,
Set<SchemaElement> elements) {
return elements.stream().filter(schemaElement -> {
if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
return allFields.contains(schemaElement.getName());
}
Set<String> allFieldsSet = new HashSet<>(allFields);
Set<String> aliasSet = new HashSet<>(schemaElement.getAlias());
List<String> intersection =
allFieldsSet.stream().filter(aliasSet::contains).collect(Collectors.toList());
return allFields.contains(schemaElement.getName())
|| !CollectionUtils.isEmpty(intersection);
}).collect(Collectors.toSet());
}
private List<String> filterDateField(DataSetSchema dataSetSchema, List<String> allFields) {
return allFields.stream()
.filter(entry -> !isPartitionDimension(dataSetSchema, entry))
return allFields.stream().filter(entry -> !isPartitionDimension(dataSetSchema, entry))
.collect(Collectors.toList());
}
private List<QueryFilter> extractDimensionFilter(
DataSetSchema dsSchema, List<FieldExpression> fieldExpressions) {
private List<QueryFilter> extractDimensionFilter(DataSetSchema dsSchema,
List<FieldExpression> fieldExpressions) {
Map<String, SchemaElement> fieldNameToElement = getNameToElement(dsSchema);
List<QueryFilter> result = Lists.newArrayList();
@@ -139,15 +133,11 @@ public class ParseInfoProcessor implements ResultProcessor {
return result;
}
private DateConf extractDateFilter(
List<FieldExpression> fieldExpressions, DataSetSchema dataSetSchema) {
List<FieldExpression> dateExpressions =
fieldExpressions.stream()
.filter(
expression ->
isPartitionDimension(
dataSetSchema, expression.getFieldName()))
.collect(Collectors.toList());
private DateConf extractDateFilter(List<FieldExpression> fieldExpressions,
DataSetSchema dataSetSchema) {
List<FieldExpression> dateExpressions = fieldExpressions.stream().filter(
expression -> isPartitionDimension(dataSetSchema, expression.getFieldName()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(dateExpressions)) {
return null;
}
@@ -164,20 +154,14 @@ public class ParseInfoProcessor implements ResultProcessor {
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
return dateInfo;
}
if (containOperators(
firstExpression,
firstOperator,
FilterOperatorEnum.GREATER_THAN,
if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.GREATER_THAN,
FilterOperatorEnum.GREATER_THAN_EQUALS)) {
dateInfo.setStartDate(firstExpression.getFieldValue().toString());
if (hasSecondDate(dateExpressions)) {
dateInfo.setEndDate(dateExpressions.get(1).getFieldValue().toString());
}
}
if (containOperators(
firstExpression,
firstOperator,
FilterOperatorEnum.MINOR_THAN,
if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.MINOR_THAN,
FilterOperatorEnum.MINOR_THAN_EQUALS)) {
dateInfo.setEndDate(firstExpression.getFieldValue().toString());
if (hasSecondDate(dateExpressions)) {
@@ -191,17 +175,14 @@ public class ParseInfoProcessor implements ResultProcessor {
if (TimeDimensionEnum.containsTimeDimension(sqlFieldName)) {
return true;
}
if (Objects.isNull(dataSetSchema)
|| Objects.isNull(dataSetSchema.getPartitionDimension())
if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension())
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
return false;
}
return sqlFieldName.equalsIgnoreCase(dataSetSchema.getPartitionDimension().getName());
}
private boolean containOperators(
FieldExpression expression,
FilterOperatorEnum firstOperator,
private boolean containOperators(FieldExpression expression, FilterOperatorEnum firstOperator,
FilterOperatorEnum... operatorEnums) {
return (Arrays.asList(operatorEnums).contains(firstOperator)
&& Objects.nonNull(expression.getFieldValue()));
@@ -220,21 +201,16 @@ public class ParseInfoProcessor implements ResultProcessor {
allElements.addAll(dimensions);
allElements.addAll(metrics);
// support alias
return allElements.stream()
.flatMap(
schemaElement -> {
Set<Pair<String, SchemaElement>> result = new HashSet<>();
result.add(Pair.of(schemaElement.getName(), schemaElement));
List<String> aliasList = schemaElement.getAlias();
if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) {
for (String alias : aliasList) {
result.add(Pair.of(alias, schemaElement));
}
}
return result.stream();
})
.collect(
Collectors.toMap(
Pair::getLeft, Pair::getRight, (value1, value2) -> value2));
return allElements.stream().flatMap(schemaElement -> {
Set<Pair<String, SchemaElement>> result = new HashSet<>();
result.add(Pair.of(schemaElement.getName(), schemaElement));
List<String> aliasList = schemaElement.getAlias();
if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) {
for (String alias : aliasList) {
result.add(Pair.of(alias, schemaElement));
}
}
return result.stream();
}).collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (value1, value2) -> value2));
}
}

View File

@@ -25,28 +25,27 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping("/api/semantic/app")
public class AppController {
@Autowired private AppService appService;
@Autowired
private AppService appService;
@PostMapping
public boolean save(
@RequestBody AppReq app, HttpServletRequest request, HttpServletResponse response) {
public boolean save(@RequestBody AppReq app, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.save(app, user);
return true;
}
@PutMapping
public boolean update(
@RequestBody AppReq app, HttpServletRequest request, HttpServletResponse response) {
public boolean update(@RequestBody AppReq app, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.update(app, user);
return true;
}
@PutMapping("/online/{id}")
public boolean online(
@PathVariable("id") Integer id,
HttpServletRequest request,
public boolean online(@PathVariable("id") Integer id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.online(id, user);
@@ -54,9 +53,7 @@ public class AppController {
}
@PutMapping("/offline/{id}")
public boolean offline(
@PathVariable("id") Integer id,
HttpServletRequest request,
public boolean offline(@PathVariable("id") Integer id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.offline(id, user);
@@ -64,9 +61,7 @@ public class AppController {
}
@DeleteMapping("/{id}")
public boolean delete(
@PathVariable("id") Integer id,
HttpServletRequest request,
public boolean delete(@PathVariable("id") Integer id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
appService.delete(id, user);
@@ -74,19 +69,15 @@ public class AppController {
}
@GetMapping("/{id}")
public AppDetailResp getApp(
@PathVariable("id") Integer id,
HttpServletRequest request,
public AppDetailResp getApp(@PathVariable("id") Integer id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return appService.getApp(id, user);
}
@PostMapping("/page")
public PageInfo<AppResp> pageApp(
@RequestBody AppQueryReq appQueryReq,
HttpServletRequest request,
HttpServletResponse response) {
public PageInfo<AppResp> pageApp(@RequestBody AppQueryReq appQueryReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return appService.pageApp(appQueryReq, user);
}

View File

@@ -24,13 +24,12 @@ import java.util.List;
@RequestMapping("/api/semantic/viewInfo")
public class CanvasController {
@Autowired private CanvasService canvasService;
@Autowired
private CanvasService canvasService;
@PostMapping("/createOrUpdateViewInfo")
public CanvasDO createOrUpdateCanvas(
@RequestBody CanvasReq canvasReq,
HttpServletRequest request,
HttpServletResponse response) {
public CanvasDO createOrUpdateCanvas(@RequestBody CanvasReq canvasReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return canvasService.createOrUpdateCanvas(canvasReq, user);
}
@@ -46,10 +45,8 @@ public class CanvasController {
}
@GetMapping("/getDomainSchemaRela/{domainId}")
public List<CanvasSchemaResp> getDomainSchema(
@PathVariable("domainId") Long domainId,
HttpServletRequest request,
HttpServletResponse response) {
public List<CanvasSchemaResp> getDomainSchema(@PathVariable("domainId") Long domainId,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return canvasService.getCanvasSchema(domainId, user);
}

View File

@@ -40,9 +40,7 @@ public class ClassController {
* @return
*/
@PostMapping("/create")
public ClassResp create(
@RequestBody @Valid ClassReq classReq,
HttpServletRequest request,
public ClassResp create(@RequestBody @Valid ClassReq classReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return classService.create(classReq, user);
@@ -57,9 +55,7 @@ public class ClassController {
* @return
*/
@PutMapping("/update")
public ClassResp update(
@RequestBody @Valid ClassReq classReq,
HttpServletRequest request,
public ClassResp update(@RequestBody @Valid ClassReq classReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return classService.update(classReq, user);
@@ -75,12 +71,8 @@ public class ClassController {
* @throws Exception
*/
@DeleteMapping("delete/{id}/{force}")
public Boolean delete(
@PathVariable("id") Long id,
@PathVariable("force") Boolean force,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Boolean delete(@PathVariable("id") Long id, @PathVariable("force") Boolean force,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return classService.delete(id, force, user);
}
@@ -95,9 +87,7 @@ public class ClassController {
* @throws Exception
*/
@GetMapping("delete/{id}/{force}")
public List<ClassResp> get(
@RequestBody @Valid ClassFilter filter,
HttpServletRequest request,
public List<ClassResp> get(@RequestBody @Valid ClassFilter filter, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return classService.getClassList(filter, user);

View File

@@ -26,27 +26,23 @@ public class CollectController {
}
@PostMapping("/createCollectionIndicators")
public boolean createCollectionIndicators(
@RequestBody CollectDO collectDO,
HttpServletRequest request,
HttpServletResponse response) {
public boolean createCollectionIndicators(@RequestBody CollectDO collectDO,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return collectService.collect(user, collectDO);
}
@Deprecated
@DeleteMapping("/deleteCollectionIndicators/{id}")
public boolean deleteCollectionIndicators(
@PathVariable Long id, HttpServletRequest request, HttpServletResponse response) {
public boolean deleteCollectionIndicators(@PathVariable Long id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return collectService.unCollect(user, id);
}
@PostMapping("/deleteCollectionIndicators")
public boolean deleteCollectionIndicators(
@RequestBody CollectDO collectDO,
HttpServletRequest request,
HttpServletResponse response) {
public boolean deleteCollectionIndicators(@RequestBody CollectDO collectDO,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return collectService.unCollect(user, collectDO);
}

View File

@@ -26,21 +26,18 @@ import java.util.List;
@RequestMapping("/api/semantic/dataSet")
public class DataSetController {
@Autowired private DataSetService dataSetService;
@Autowired
private DataSetService dataSetService;
@PostMapping
public DataSetResp save(
@RequestBody DataSetReq dataSetReq,
HttpServletRequest request,
public DataSetResp save(@RequestBody DataSetReq dataSetReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return dataSetService.save(dataSetReq, user);
}
@PutMapping
public DataSetResp update(
@RequestBody DataSetReq dataSetReq,
HttpServletRequest request,
public DataSetResp update(@RequestBody DataSetReq dataSetReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return dataSetService.update(dataSetReq, user);
@@ -59,8 +56,8 @@ public class DataSetController {
}
@DeleteMapping("/{id}")
public Boolean delete(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
dataSetService.delete(id, user);
return true;

View File

@@ -37,33 +37,29 @@ public class DatabaseController {
}
@PostMapping("/testConnect")
public boolean testConnect(
@RequestBody DatabaseReq databaseReq,
HttpServletRequest request,
public boolean testConnect(@RequestBody DatabaseReq databaseReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.testConnect(databaseReq, user);
}
@PostMapping("/createOrUpdateDatabase")
public DatabaseResp createOrUpdateDatabase(
@RequestBody DatabaseReq databaseReq,
HttpServletRequest request,
HttpServletResponse response) {
public DatabaseResp createOrUpdateDatabase(@RequestBody DatabaseReq databaseReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.createOrUpdateDatabase(databaseReq, user);
}
@GetMapping("/{id}")
public DatabaseResp getDatabase(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
public DatabaseResp getDatabase(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.getDatabase(id, user);
}
@GetMapping("/getDatabaseList")
public List<DatabaseResp> getDatabaseList(
HttpServletRequest request, HttpServletResponse response) {
public List<DatabaseResp> getDatabaseList(HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.getDatabaseList(user);
}
@@ -75,10 +71,8 @@ public class DatabaseController {
}
@PostMapping("/executeSql")
public SemanticQueryResp executeSql(
@RequestBody SqlExecuteReq sqlExecuteReq,
HttpServletRequest request,
HttpServletResponse response) {
public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user);
}
@@ -89,17 +83,14 @@ public class DatabaseController {
}
@RequestMapping("/getTables")
public List<String> getTables(
@RequestParam("databaseId") Long databaseId, @RequestParam("db") String db)
throws SQLException {
public List<String> getTables(@RequestParam("databaseId") Long databaseId,
@RequestParam("db") String db) throws SQLException {
return databaseService.getTables(databaseId, db);
}
@RequestMapping("/getColumnsByName")
public List<DBColumn> getColumnsByName(
@RequestParam("databaseId") Long databaseId,
@RequestParam("db") String db,
@RequestParam("table") String table)
public List<DBColumn> getColumnsByName(@RequestParam("databaseId") Long databaseId,
@RequestParam("db") String db, @RequestParam("table") String table)
throws SQLException {
return databaseService.getColumns(databaseId, db, table);
}
@@ -110,8 +101,8 @@ public class DatabaseController {
}
@GetMapping("/getDatabaseParameters")
public Map<String, List<DatabaseParameter>> getDatabaseParameters(
HttpServletRequest request, HttpServletResponse response) {
public Map<String, List<DatabaseParameter>> getDatabaseParameters(HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return databaseService.getDatabaseParameters(user);
}

View File

@@ -34,9 +34,11 @@ import java.util.List;
@RequestMapping("/api/semantic/dimension")
public class DimensionController {
@Autowired private DimensionService dimensionService;
@Autowired
private DimensionService dimensionService;
@Autowired private SemanticLayerService queryService;
@Autowired
private SemanticLayerService queryService;
/**
* 创建维度
@@ -44,60 +46,46 @@ public class DimensionController {
* @param dimensionReq
*/
@PostMapping("/createDimension")
public DimensionResp createDimension(
@RequestBody DimensionReq dimensionReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public DimensionResp createDimension(@RequestBody DimensionReq dimensionReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return dimensionService.createDimension(dimensionReq, user);
}
@PostMapping("/updateDimension")
public Boolean updateDimension(
@RequestBody DimensionReq dimensionReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Boolean updateDimension(@RequestBody DimensionReq dimensionReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
dimensionService.updateDimension(dimensionReq, user);
return true;
}
@PostMapping("/batchUpdateStatus")
public Boolean batchUpdateStatus(
@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
dimensionService.batchUpdateStatus(metaBatchReq, user);
return true;
}
@PostMapping("/batchUpdateSensitiveLevel")
public Boolean batchUpdateSensitiveLevel(
@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean batchUpdateSensitiveLevel(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
dimensionService.batchUpdateSensitiveLevel(metaBatchReq, user);
return true;
}
@PostMapping("/mockDimensionAlias")
public List<String> mockMetricAlias(
@RequestBody DimensionReq dimensionReq,
HttpServletRequest request,
HttpServletResponse response) {
public List<String> mockMetricAlias(@RequestBody DimensionReq dimensionReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return dimensionService.mockAlias(dimensionReq, "dimension", user);
}
@PostMapping("/mockDimensionValuesAlias")
public List<DimValueMap> mockDimensionValuesAlias(
@RequestBody DimensionReq dimensionReq,
HttpServletRequest request,
HttpServletResponse response) {
public List<DimValueMap> mockDimensionValuesAlias(@RequestBody DimensionReq dimensionReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return dimensionService.mockDimensionValueAlias(dimensionReq, user);
}
@@ -115,8 +103,7 @@ public class DimensionController {
}
@GetMapping("/{modelId}/{dimensionName}")
public DimensionResp getDimensionDescByNameAndId(
@PathVariable("modelId") Long modelId,
public DimensionResp getDimensionDescByNameAndId(@PathVariable("modelId") Long modelId,
@PathVariable("dimensionName") String dimensionBizName) {
return dimensionService.getDimension(dimensionBizName, modelId);
}
@@ -127,17 +114,15 @@ public class DimensionController {
}
@PostMapping("/queryDimValue")
public SemanticQueryResp queryDimValue(
@RequestBody DimensionValueReq dimensionValueReq,
HttpServletRequest request,
HttpServletResponse response) {
public SemanticQueryResp queryDimValue(@RequestBody DimensionValueReq dimensionValueReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return queryService.queryDimensionValue(dimensionValueReq, user);
}
@DeleteMapping("deleteDimension/{id}")
public Boolean deleteDimension(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
public Boolean deleteDimension(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
dimensionService.deleteDimension(id, user);
return true;

View File

@@ -32,19 +32,15 @@ public class DomainController {
}
@PostMapping("/createDomain")
public DomainResp createDomain(
@RequestBody DomainReq domainReq,
HttpServletRequest request,
public DomainResp createDomain(@RequestBody DomainReq domainReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return domainService.createDomain(domainReq, user);
}
@PostMapping("/updateDomain")
public DomainResp updateDomain(
@RequestBody DomainUpdateReq domainUpdateReq,
HttpServletRequest request,
HttpServletResponse response) {
public DomainResp updateDomain(@RequestBody DomainUpdateReq domainUpdateReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return domainService.updateDomain(domainUpdateReq, user);
}
@@ -56,8 +52,8 @@ public class DomainController {
}
@GetMapping("/getDomainList")
public List<DomainResp> getDomainList(
HttpServletRequest request, HttpServletResponse response) {
public List<DomainResp> getDomainList(HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return domainService.getDomainListWithAdminAuth(user);
}
@@ -69,9 +65,7 @@ public class DomainController {
@GetMapping("/getDomainListByIds/{domainIds}")
public List<DomainResp> getDomainListByIds(@PathVariable("domainIds") String domainIds) {
return domainService.getDomainList(
Arrays.stream(domainIds.split(","))
.map(Long::parseLong)
.collect(Collectors.toList()));
return domainService.getDomainList(Arrays.stream(domainIds.split(",")).map(Long::parseLong)
.collect(Collectors.toList()));
}
}

View File

@@ -34,17 +34,23 @@ import java.util.List;
@RequestMapping("/api/semantic/knowledge")
public class KnowledgeController {
@Autowired private DictTaskService taskService;
@Autowired
private DictTaskService taskService;
@Autowired private DictConfService confService;
@Autowired
private DictConfService confService;
@Autowired private MetaEmbeddingTask metaEmbeddingTask;
@Autowired
private MetaEmbeddingTask metaEmbeddingTask;
@Autowired private DictionaryReloadTask dictionaryReloadTask;
@Autowired
private DictionaryReloadTask dictionaryReloadTask;
@Autowired private ExemplarService exemplarService;
@Autowired
private ExemplarService exemplarService;
@Autowired private EmbeddingService embeddingService;
@Autowired
private EmbeddingService embeddingService;
/**
* addDictConf-新增item的字典配置 Add configuration information for dictionary entries
@@ -52,10 +58,8 @@ public class KnowledgeController {
* @param dictItemReq
*/
@PostMapping("/conf")
public DictItemResp addDictConf(
@RequestBody @Valid DictItemReq dictItemReq,
HttpServletRequest request,
HttpServletResponse response) {
public DictItemResp addDictConf(@RequestBody @Valid DictItemReq dictItemReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return confService.addDictConf(dictItemReq, user);
}
@@ -66,10 +70,8 @@ public class KnowledgeController {
* @param dictItemReq
*/
@PutMapping("/conf")
public DictItemResp editDictConf(
@RequestBody @Valid DictItemReq dictItemReq,
HttpServletRequest request,
HttpServletResponse response) {
public DictItemResp editDictConf(@RequestBody @Valid DictItemReq dictItemReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return confService.editDictConf(dictItemReq, user);
}
@@ -80,10 +82,8 @@ public class KnowledgeController {
* @param filter
*/
@PostMapping("/conf/query")
public List<DictItemResp> queryDictConf(
@RequestBody @Valid DictItemFilter filter,
HttpServletRequest request,
HttpServletResponse response) {
public List<DictItemResp> queryDictConf(@RequestBody @Valid DictItemFilter filter,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return confService.queryDictConf(filter, user);
}
@@ -94,9 +94,7 @@ public class KnowledgeController {
* @param taskReq
*/
@PostMapping("/task")
public Long addDictTask(
@RequestBody DictSingleTaskReq taskReq,
HttpServletRequest request,
public Long addDictTask(@RequestBody DictSingleTaskReq taskReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return taskService.addDictTask(taskReq, user);
@@ -108,9 +106,7 @@ public class KnowledgeController {
* @param taskReq
*/
@PutMapping("/task/delete")
public Long deleteDictTask(
@RequestBody DictSingleTaskReq taskReq,
HttpServletRequest request,
public Long deleteDictTask(@RequestBody DictSingleTaskReq taskReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return taskService.deleteDictTask(taskReq, user);
@@ -128,10 +124,8 @@ public class KnowledgeController {
* @param taskReq
*/
@PostMapping("/task/search")
public DictTaskResp queryLatestDictTask(
@RequestBody DictSingleTaskReq taskReq,
HttpServletRequest request,
HttpServletResponse response) {
public DictTaskResp queryLatestDictTask(@RequestBody DictSingleTaskReq taskReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return taskService.queryLatestDictTask(taskReq, user);
}
@@ -161,10 +155,8 @@ public class KnowledgeController {
* @param dictValueReq
*/
@PostMapping("/dict/data")
public PageInfo<DictValueResp> queryDictValue(
@RequestBody @Valid DictValueReq dictValueReq,
HttpServletRequest request,
HttpServletResponse response) {
public PageInfo<DictValueResp> queryDictValue(@RequestBody @Valid DictValueReq dictValueReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return taskService.queryDictValue(dictValueReq, user);
}
@@ -175,10 +167,8 @@ public class KnowledgeController {
* @param dictValueReq
*/
@PostMapping("/dict/file")
public String queryDictFilePath(
@RequestBody @Valid DictValueReq dictValueReq,
HttpServletRequest request,
HttpServletResponse response) {
public String queryDictFilePath(@RequestBody @Valid DictValueReq dictValueReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return taskService.queryDictFilePath(dictValueReq, user);
}

View File

@@ -40,39 +40,29 @@ public class MetricController {
}
@PostMapping("/createMetric")
public MetricResp createMetric(
@RequestBody MetricReq metricReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public MetricResp createMetric(@RequestBody MetricReq metricReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return metricService.createMetric(metricReq, user);
}
@PostMapping("/updateMetric")
public MetricResp updateMetric(
@RequestBody MetricReq metricReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public MetricResp updateMetric(@RequestBody MetricReq metricReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return metricService.updateMetric(metricReq, user);
}
@PostMapping("/batchUpdateStatus")
public Boolean batchUpdateStatus(
@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
metricService.batchUpdateStatus(metaBatchReq, user);
return true;
}
@PostMapping("/batchPublish")
public Boolean batchPublish(
@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
public Boolean batchPublish(@RequestBody MetaBatchReq metaBatchReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
metricService.batchPublish(metaBatchReq.getIds(), user);
@@ -80,40 +70,32 @@ public class MetricController {
}
@PostMapping("/batchUnPublish")
public Boolean batchUnPublish(
@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean batchUnPublish(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
metricService.batchUnPublish(metaBatchReq.getIds(), user);
return true;
}
@PostMapping("/batchUpdateClassifications")
public Boolean batchUpdateClassifications(
@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean batchUpdateClassifications(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
metricService.batchUpdateClassifications(metaBatchReq, user);
return true;
}
@PostMapping("/batchUpdateSensitiveLevel")
public Boolean batchUpdateSensitiveLevel(
@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean batchUpdateSensitiveLevel(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
metricService.batchUpdateSensitiveLevel(metaBatchReq, user);
return true;
}
@PostMapping("/mockMetricAlias")
public List<String> mockMetricAlias(
@RequestBody MetricBaseReq metricReq,
HttpServletRequest request,
HttpServletResponse response) {
public List<String> mockMetricAlias(@RequestBody MetricBaseReq metricReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return metricService.mockAlias(metricReq, "indicator", user);
}
@@ -130,32 +112,29 @@ public class MetricController {
}
@PostMapping("/queryMetric")
public PageInfo<MetricResp> queryMetric(
@RequestBody PageMetricReq pageMetricReq,
HttpServletRequest request,
HttpServletResponse response) {
public PageInfo<MetricResp> queryMetric(@RequestBody PageMetricReq pageMetricReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return metricService.queryMetricMarket(pageMetricReq, user);
}
@Deprecated
@GetMapping("getMetric/{modelId}/{bizName}")
public MetricResp getMetric(
@PathVariable("modelId") Long modelId, @PathVariable("bizName") String bizName) {
public MetricResp getMetric(@PathVariable("modelId") Long modelId,
@PathVariable("bizName") String bizName) {
return metricService.getMetric(modelId, bizName);
}
@GetMapping("getMetric/{id}")
public MetricResp getMetric(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
public MetricResp getMetric(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return metricService.getMetric(id, user);
}
@DeleteMapping("deleteMetric/{id}")
public Boolean deleteMetric(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response)
throws Exception {
public Boolean deleteMetric(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
metricService.deleteMetric(id, user);
return true;
@@ -186,8 +165,7 @@ public class MetricController {
@PostMapping("/saveMetricQueryDefaultConfig")
public boolean saveMetricQueryDefaultConfig(
@RequestBody MetricQueryDefaultConfig queryDefaultConfig,
HttpServletRequest request,
@RequestBody MetricQueryDefaultConfig queryDefaultConfig, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
metricService.saveMetricQueryDefaultConfig(queryDefaultConfig, user);
@@ -196,8 +174,7 @@ public class MetricController {
@RequestMapping("getMetricQueryDefaultConfig/{metricId}")
public MetricQueryDefaultConfig getMetricQueryDefaultConfig(
@PathVariable("metricId") Long metricId,
HttpServletRequest request,
@PathVariable("metricId") Long metricId, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return metricService.getMetricQueryDefaultConfig(metricId, user);

View File

@@ -39,31 +39,23 @@ public class ModelController {
}
@PostMapping("/createModel")
public Boolean createModel(
@RequestBody ModelReq modelReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Boolean createModel(@RequestBody ModelReq modelReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
modelService.createModel(modelReq, user);
return true;
}
@PostMapping("/updateModel")
public Boolean updateModel(
@RequestBody ModelReq modelReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Boolean updateModel(@RequestBody ModelReq modelReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
modelService.updateModel(modelReq, user);
return true;
}
@DeleteMapping("/deleteModel/{modelId}")
public Boolean deleteModel(
@PathVariable("modelId") Long modelId,
HttpServletRequest request,
public Boolean deleteModel(@PathVariable("modelId") Long modelId, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
modelService.deleteModel(modelId, user);
@@ -71,10 +63,8 @@ public class ModelController {
}
@GetMapping("/getModelList/{domainId}")
public List<ModelResp> getModelList(
@PathVariable("domainId") Long domainId,
HttpServletRequest request,
HttpServletResponse response) {
public List<ModelResp> getModelList(@PathVariable("domainId") Long domainId,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return modelService.getModelListWithAuth(user, domainId, AuthType.ADMIN);
}
@@ -86,10 +76,8 @@ public class ModelController {
@GetMapping("/getModelListByIds/{modelIds}")
public List<ModelResp> getModelListByIds(@PathVariable("modelIds") String modelIds) {
List<Long> ids =
Arrays.stream(modelIds.split(","))
.map(Long::parseLong)
.collect(Collectors.toList());
List<Long> ids = Arrays.stream(modelIds.split(",")).map(Long::parseLong)
.collect(Collectors.toList());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setIds(ids);
return modelService.getModelList(modelFilter);
@@ -106,10 +94,8 @@ public class ModelController {
}
@PostMapping("/batchUpdateStatus")
public Boolean batchUpdateStatus(
@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request,
HttpServletResponse response) {
public Boolean batchUpdateStatus(@RequestBody MetaBatchReq metaBatchReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
modelService.batchUpdateStatus(metaBatchReq, user);
return true;

View File

@@ -19,7 +19,8 @@ import java.util.List;
@RequestMapping("/api/semantic/modelRela")
public class ModelRelaController {
@Autowired private ModelRelaService modelRelaService;
@Autowired
private ModelRelaService modelRelaService;
@PostMapping
public boolean save(@RequestBody ModelRela modelRela, User user) {

View File

@@ -39,10 +39,8 @@ public class QueryRuleController {
* @throws Exception
*/
@PostMapping("/create")
public QueryRuleResp create(
@RequestBody @Validated QueryRuleReq queryRuleReq,
HttpServletRequest request,
HttpServletResponse response) {
public QueryRuleResp create(@RequestBody @Validated QueryRuleReq queryRuleReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return queryRuleService.addQueryRule(queryRuleReq, user);
}
@@ -57,10 +55,8 @@ public class QueryRuleController {
* @throws Exception
*/
@PostMapping("/update")
public QueryRuleResp update(
@RequestBody @Validated QueryRuleReq queryRuleReq,
HttpServletRequest request,
HttpServletResponse response) {
public QueryRuleResp update(@RequestBody @Validated QueryRuleReq queryRuleReq,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return queryRuleService.updateQueryRule(queryRuleReq, user);
}
@@ -74,8 +70,8 @@ public class QueryRuleController {
* @return
*/
@DeleteMapping("delete/{id}")
public Boolean delete(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return queryRuleService.dropQueryRule(id, user);
}
@@ -88,10 +84,8 @@ public class QueryRuleController {
* @return
*/
@PostMapping("query")
public List<QueryRuleResp> query(
@RequestBody @Validated QueryRuleFilter queryRuleFilter,
HttpServletRequest request,
HttpServletResponse response) {
public List<QueryRuleResp> query(@RequestBody @Validated QueryRuleFilter queryRuleFilter,
HttpServletRequest request, HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return queryRuleService.getQueryRuleList(queryRuleFilter, user);
}

View File

@@ -21,20 +21,19 @@ import java.util.List;
@RequestMapping("/api/semantic/schema")
public class SchemaController {
@Autowired private SchemaService schemaService;
@Autowired
private SchemaService schemaService;
@GetMapping("/domain/list")
public List<DomainResp> getDomainList(
HttpServletRequest request, HttpServletResponse response) {
public List<DomainResp> getDomainList(HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.getDomainList(user);
}
@GetMapping("/model/list")
public List<ModelResp> getModelList(
@RequestParam("domainId") Long domainId,
@RequestParam("authType") String authType,
HttpServletRequest request,
public List<ModelResp> getModelList(@RequestParam("domainId") Long domainId,
@RequestParam("authType") String authType, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return schemaService.getModelList(user, AuthType.valueOf(authType), domainId);

View File

@@ -49,9 +49,8 @@ public class TagController {
* @throws Exception
*/
@PostMapping("/create")
public TagResp create(
@RequestBody TagReq tagReq, HttpServletRequest request, HttpServletResponse response)
throws Exception {
public TagResp create(@RequestBody TagReq tagReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagMetaService.create(tagReq, user);
}
@@ -66,11 +65,8 @@ public class TagController {
* @throws Exception
*/
@PostMapping("/create/batch")
public Integer createBatch(
@RequestBody @Valid List<TagReq> tagReqList,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Integer createBatch(@RequestBody @Valid List<TagReq> tagReqList,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagMetaService.createBatch(tagReqList, user);
}
@@ -85,11 +81,8 @@ public class TagController {
* @throws Exception
*/
@PostMapping("/delete/batch")
public Boolean deleteBatch(
@RequestBody @Valid List<TagDeleteReq> tagDeleteReqList,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public Boolean deleteBatch(@RequestBody @Valid List<TagDeleteReq> tagDeleteReqList,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagMetaService.deleteBatch(tagDeleteReqList, user);
}
@@ -104,8 +97,8 @@ public class TagController {
* @throws Exception
*/
@DeleteMapping("delete/{id}")
public Boolean delete(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
tagMetaService.delete(id, user);
return true;
@@ -120,8 +113,8 @@ public class TagController {
* @return
*/
@GetMapping("getTag/{id}")
public TagResp getTag(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response) {
public TagResp getTag(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return tagMetaService.getTag(id, user);
}
@@ -147,11 +140,8 @@ public class TagController {
* @throws Exception
*/
@PostMapping("/value/distribution")
public ItemValueResp queryTagValue(
@RequestBody ItemValueReq itemValueReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public ItemValueResp queryTagValue(@RequestBody ItemValueReq itemValueReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagQueryService.queryTagValue(itemValueReq, user);
}
@@ -166,11 +156,8 @@ public class TagController {
* @throws Exception
*/
@PostMapping("/queryTag/market")
public PageInfo<TagResp> queryTagMarketPage(
@RequestBody TagFilterPageReq tagMarketPageReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public PageInfo<TagResp> queryTagMarketPage(@RequestBody TagFilterPageReq tagMarketPageReq,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagMetaService.queryTagMarketPage(tagMarketPageReq, user);
}

View File

@@ -38,11 +38,8 @@ public class TagObjectController {
* @throws Exception
*/
@PostMapping("/create")
public TagObjectResp create(
@RequestBody TagObjectReq tagObjectReq,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public TagObjectResp create(@RequestBody TagObjectReq tagObjectReq, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagObjectService.create(tagObjectReq, user);
}
@@ -56,9 +53,7 @@ public class TagObjectController {
* @return
*/
@PostMapping("/update")
public TagObjectResp update(
@RequestBody TagObjectReq tagObjectReq,
HttpServletRequest request,
public TagObjectResp update(@RequestBody TagObjectReq tagObjectReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return tagObjectService.update(tagObjectReq, user);
@@ -74,9 +69,8 @@ public class TagObjectController {
* @throws Exception
*/
@DeleteMapping("delete/{id}")
public Boolean delete(
@PathVariable("id") Long id, HttpServletRequest request, HttpServletResponse response)
throws Exception {
public Boolean delete(@PathVariable("id") Long id, HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
tagObjectService.delete(id, user, true);
return true;
@@ -92,11 +86,8 @@ public class TagObjectController {
* @throws Exception
*/
@PostMapping("/query")
public List<TagObjectResp> queryTagObject(
@RequestBody TagObjectFilter filter,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
public List<TagObjectResp> queryTagObject(@RequestBody TagObjectFilter filter,
HttpServletRequest request, HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return tagObjectService.getTagObjects(filter, user);
}

View File

@@ -25,12 +25,11 @@ import java.util.List;
@RequestMapping("/api/semantic/term")
public class TermController {
@Autowired private TermService termService;
@Autowired
private TermService termService;
@PostMapping("/saveOrUpdate")
public boolean saveOrUpdate(
@RequestBody TermReq termReq,
HttpServletRequest request,
public boolean saveOrUpdate(@RequestBody TermReq termReq, HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
termService.saveOrUpdate(termReq, user);
@@ -38,8 +37,7 @@ public class TermController {
}
@GetMapping
public List<TermResp> getTerms(
@RequestParam("domainId") Long domainId,
public List<TermResp> getTerms(@RequestParam("domainId") Long domainId,
@RequestParam(name = "queryKey", required = false) String queryKey) {
return termService.getTerms(domainId, queryKey);
}

View File

@@ -8,9 +8,8 @@ import com.tencent.supersonic.headless.api.pojo.request.DownloadMetricReq;
public interface DownloadService {
void downloadByStruct(
DownloadMetricReq downloadStructReq, User user, HttpServletResponse response)
throws Exception;
void downloadByStruct(DownloadMetricReq downloadStructReq, User user,
HttpServletResponse response) throws Exception;
void batchDownload(BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response)
throws Exception;

View File

@@ -57,11 +57,9 @@ public interface SchemaService {
List<ItemResp> getDomainDataSetTree();
void getSchemaYamlTpl(
SemanticSchemaResp semanticSchemaResp,
void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList,
List<MetricYamlTpl> metricYamlTplList,
List<DataModelYamlTpl> dataModelYamlTplList, List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName);
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);

View File

@@ -44,8 +44,8 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
private DimensionService dimensionService;
public AppServiceImpl(
AppMapper appMapper, MetricService metricService, DimensionService dimensionService) {
public AppServiceImpl(AppMapper appMapper, MetricService metricService,
DimensionService dimensionService) {
this.appMapper = appMapper;
this.metricService = metricService;
this.dimensionService = dimensionService;
@@ -110,16 +110,13 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
PageHelper.startPage(appQueryReq.getCurrent(), appQueryReq.getPageSize())
.doSelectPageInfo(() -> queryApp(appQueryReq));
PageInfo<AppResp> appPageInfo = PageUtils.pageInfo2PageInfoVo(appDOPageInfo);
Map<Long, MetricResp> metricResps =
metricService.getMetrics(new MetaFilter()).stream()
.collect(Collectors.toMap(MetricResp::getId, m -> m));
Map<Long, DimensionResp> dimensionResps =
dimensionService.getDimensions(new MetaFilter()).stream()
.collect(Collectors.toMap(DimensionResp::getId, m -> m));
appPageInfo.setList(
appDOPageInfo.getList().stream()
.map(appDO -> convert(appDO, dimensionResps, metricResps, user))
.collect(Collectors.toList()));
Map<Long, MetricResp> metricResps = metricService.getMetrics(new MetaFilter()).stream()
.collect(Collectors.toMap(MetricResp::getId, m -> m));
Map<Long, DimensionResp> dimensionResps = dimensionService.getDimensions(new MetaFilter())
.stream().collect(Collectors.toMap(DimensionResp::getId, m -> m));
appPageInfo.setList(appDOPageInfo.getList().stream()
.map(appDO -> convert(appDO, dimensionResps, metricResps, user))
.collect(Collectors.toList()));
return appPageInfo;
}
@@ -141,12 +138,10 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
@Override
public AppDetailResp getApp(Integer id, User user) {
AppDO appDO = getAppDO(id);
Map<Long, MetricResp> metricResps =
metricService.getMetrics(new MetaFilter()).stream()
.collect(Collectors.toMap(MetricResp::getId, m -> m));
Map<Long, DimensionResp> dimensionResps =
dimensionService.getDimensions(new MetaFilter()).stream()
.collect(Collectors.toMap(DimensionResp::getId, m -> m));
Map<Long, MetricResp> metricResps = metricService.getMetrics(new MetaFilter()).stream()
.collect(Collectors.toMap(MetricResp::getId, m -> m));
Map<Long, DimensionResp> dimensionResps = dimensionService.getDimensions(new MetaFilter())
.stream().collect(Collectors.toMap(DimensionResp::getId, m -> m));
checkAuth(appDO, user);
return convertDetail(appDO, dimensionResps, metricResps);
}
@@ -179,11 +174,8 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
&& appDO.getOwner().contains(user.getName());
}
private AppResp convert(
AppDO appDO,
Map<Long, DimensionResp> dimensionMap,
Map<Long, MetricResp> metricMap,
User user) {
private AppResp convert(AppDO appDO, Map<Long, DimensionResp> dimensionMap,
Map<Long, MetricResp> metricMap, User user) {
AppResp app = new AppResp();
BeanMapper.mapper(appDO, app);
AppConfig appConfig = JSONObject.parseObject(appDO.getConfig(), AppConfig.class);
@@ -198,8 +190,8 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
return convertDetail(appDO, new HashMap<>(), new HashMap<>());
}
private AppDetailResp convertDetail(
AppDO appDO, Map<Long, DimensionResp> dimensionMap, Map<Long, MetricResp> metricMap) {
private AppDetailResp convertDetail(AppDO appDO, Map<Long, DimensionResp> dimensionMap,
Map<Long, MetricResp> metricMap) {
AppDetailResp app = new AppDetailResp();
BeanMapper.mapper(appDO, app);
AppConfig appConfig = JSONObject.parseObject(appDO.getConfig(), AppConfig.class);
@@ -209,50 +201,24 @@ public class AppServiceImpl extends ServiceImpl<AppMapper, AppDO> implements App
return app;
}
private void fillItemName(
AppConfig appConfig,
Map<Long, DimensionResp> dimensionMap,
private void fillItemName(AppConfig appConfig, Map<Long, DimensionResp> dimensionMap,
Map<Long, MetricResp> metricMap) {
appConfig
.getItems()
.forEach(
metricItem -> {
metricItem.setName(
metricMap
.getOrDefault(metricItem.getId(), new MetricResp())
.getName());
metricItem.setBizName(
metricMap
.getOrDefault(metricItem.getId(), new MetricResp())
.getBizName());
metricItem.setCreatedBy(
metricMap
.getOrDefault(metricItem.getId(), new MetricResp())
.getCreatedBy());
metricItem
.getRelateItems()
.forEach(
dimensionItem -> {
dimensionItem.setName(
dimensionMap
.getOrDefault(
dimensionItem.getId(),
new DimensionResp())
.getName());
dimensionItem.setBizName(
dimensionMap
.getOrDefault(
dimensionItem.getId(),
new DimensionResp())
.getBizName());
dimensionItem.setCreatedBy(
dimensionMap
.getOrDefault(
dimensionItem.getId(),
new DimensionResp())
.getCreatedBy());
});
});
appConfig.getItems().forEach(metricItem -> {
metricItem.setName(
metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getName());
metricItem.setBizName(
metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getBizName());
metricItem.setCreatedBy(
metricMap.getOrDefault(metricItem.getId(), new MetricResp()).getCreatedBy());
metricItem.getRelateItems().forEach(dimensionItem -> {
dimensionItem.setName(dimensionMap
.getOrDefault(dimensionItem.getId(), new DimensionResp()).getName());
dimensionItem.setBizName(dimensionMap
.getOrDefault(dimensionItem.getId(), new DimensionResp()).getBizName());
dimensionItem.setCreatedBy(dimensionMap
.getOrDefault(dimensionItem.getId(), new DimensionResp()).getCreatedBy());
});
});
}
private String getUniqueId() {

View File

@@ -27,11 +27,14 @@ import java.util.List;
public class CanvasServiceImpl extends ServiceImpl<CanvasDOMapper, CanvasDO>
implements CanvasService {
@Autowired private ModelService modelService;
@Autowired
private ModelService modelService;
@Autowired private DimensionService dimensionService;
@Autowired
private DimensionService dimensionService;
@Autowired private MetricService metricService;
@Autowired
private MetricService metricService;
@Override
public List<CanvasDO> getCanvasList(Long domainId) {

View File

@@ -19,7 +19,8 @@ import java.util.List;
public class CollectServiceImpl implements CollectService {
public static final String type = "metric";
@Resource private CollectMapper collectMapper;
@Resource
private CollectMapper collectMapper;
@Override
public Boolean collect(User user, CollectDO collectReq) {

View File

@@ -58,13 +58,20 @@ import java.util.stream.Collectors;
public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
implements DataSetService {
@Autowired private DomainService domainService;
@Autowired
private DomainService domainService;
@Lazy @Autowired private DimensionService dimensionService;
@Lazy
@Autowired
private DimensionService dimensionService;
@Lazy @Autowired private MetricService metricService;
@Lazy
@Autowired
private MetricService metricService;
@Lazy @Autowired private TagMetaService tagMetaService;
@Lazy
@Autowired
private TagMetaService tagMetaService;
@Override
public DataSetResp save(DataSetReq dataSetReq, User user) {
@@ -152,24 +159,21 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
List<DataSetResp> dataSetFilterByAuth = getDataSetFilterByAuth(dataSetResps, user);
dataSetRespSet.addAll(dataSetFilterByAuth);
if (domainId != null && domainId > 0) {
dataSetRespSet =
dataSetRespSet.stream()
.filter(modelResp -> modelResp.getDomainId().equals(domainId))
.collect(Collectors.toSet());
dataSetRespSet = dataSetRespSet.stream()
.filter(modelResp -> modelResp.getDomainId().equals(domainId))
.collect(Collectors.toSet());
}
return dataSetRespSet.stream()
.sorted(Comparator.comparingLong(DataSetResp::getId))
return dataSetRespSet.stream().sorted(Comparator.comparingLong(DataSetResp::getId))
.collect(Collectors.toList());
}
private List<DataSetResp> getDataSetFilterByAuth(List<DataSetResp> dataSetResps, User user) {
return dataSetResps.stream()
.filter(dataSetResp -> checkAdminPermission(user, dataSetResp))
return dataSetResps.stream().filter(dataSetResp -> checkAdminPermission(user, dataSetResp))
.collect(Collectors.toList());
}
private List<DataSetResp> getDataSetFilterByDomainAuth(
List<DataSetResp> dataSetResps, User user) {
private List<DataSetResp> getDataSetFilterByDomainAuth(List<DataSetResp> dataSetResps,
User user) {
Set<DomainResp> domainResps = domainService.getDomainAuthSet(user, AuthType.ADMIN);
if (CollectionUtils.isEmpty(domainResps)) {
return Lists.newArrayList();
@@ -190,14 +194,10 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
dataSetResp.setQueryConfig(
JSONObject.parseObject(dataSetDO.getQueryConfig(), QueryConfig.class));
}
dataSetResp.setAdmins(
StringUtils.isBlank(dataSetDO.getAdmin())
? Lists.newArrayList()
: Arrays.asList(dataSetDO.getAdmin().split(",")));
dataSetResp.setAdminOrgs(
StringUtils.isBlank(dataSetDO.getAdminOrg())
? Lists.newArrayList()
: Arrays.asList(dataSetDO.getAdminOrg().split(",")));
dataSetResp.setAdmins(StringUtils.isBlank(dataSetDO.getAdmin()) ? Lists.newArrayList()
: Arrays.asList(dataSetDO.getAdmin().split(",")));
dataSetResp.setAdminOrgs(StringUtils.isBlank(dataSetDO.getAdminOrg()) ? Lists.newArrayList()
: Arrays.asList(dataSetDO.getAdminOrg().split(",")));
dataSetResp.setTypeEnum(TypeEnums.DATASET);
List<TagItem> dimensionItems =
tagMetaService.getTagItems(dataSetResp.dimensionIds(), TagDefineType.DIMENSION);
@@ -246,14 +246,10 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
metaFilter.setIds(dataSetIds);
List<DataSetResp> dataSetList = getDataSetList(metaFilter);
return dataSetList.stream()
.flatMap(
dataSetResp ->
dataSetResp.getAllModels().stream()
.map(modelId -> Pair.of(modelId, dataSetResp.getId())))
.collect(
Collectors.groupingBy(
Pair::getLeft,
Collectors.mapping(Pair::getRight, Collectors.toList())));
.flatMap(dataSetResp -> dataSetResp.getAllModels().stream()
.map(modelId -> Pair.of(modelId, dataSetResp.getId())))
.collect(Collectors.groupingBy(Pair::getLeft,
Collectors.mapping(Pair::getRight, Collectors.toList())));
}
@Override
@@ -285,14 +281,9 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
}
private <T, R> List<String> findDuplicates(List<T> list, Function<T, R> keyExtractor) {
return list.stream()
.collect(Collectors.groupingBy(keyExtractor, Collectors.counting()))
.entrySet()
.stream()
.filter(entry -> entry.getValue() > 1)
.map(Map.Entry::getKey)
.map(Object::toString)
.collect(Collectors.toList());
return list.stream().collect(Collectors.groupingBy(keyExtractor, Collectors.counting()))
.entrySet().stream().filter(entry -> entry.getValue() > 1).map(Map.Entry::getKey)
.map(Object::toString).collect(Collectors.toList());
}
public Long getDataSetIdFromSql(String sql, User user) {

View File

@@ -45,9 +45,12 @@ import java.util.stream.Collectors;
public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseDO>
implements DatabaseService {
@Autowired private SqlUtils sqlUtils;
@Autowired
private SqlUtils sqlUtils;
@Lazy @Autowired private ModelService datasourceService;
@Lazy
@Autowired
private ModelService datasourceService;
@Override
public boolean testConnect(DatabaseReq databaseReq, User user) {
@@ -84,19 +87,18 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
}
private void fillPermission(List<DatabaseResp> databaseResps, User user) {
databaseResps.forEach(
databaseResp -> {
if (databaseResp.getAdmins().contains(user.getName())
|| user.getName().equalsIgnoreCase(databaseResp.getCreatedBy())
|| user.isSuperAdmin()) {
databaseResp.setHasPermission(true);
databaseResp.setHasEditPermission(true);
databaseResp.setHasUsePermission(true);
}
if (databaseResp.getViewers().contains(user.getName())) {
databaseResp.setHasUsePermission(true);
}
});
databaseResps.forEach(databaseResp -> {
if (databaseResp.getAdmins().contains(user.getName())
|| user.getName().equalsIgnoreCase(databaseResp.getCreatedBy())
|| user.isSuperAdmin()) {
databaseResp.setHasPermission(true);
databaseResp.setHasEditPermission(true);
databaseResp.setHasUsePermission(true);
}
if (databaseResp.getViewers().contains(user.getName())) {
databaseResp.setHasUsePermission(true);
}
});
}
@Override
@@ -135,9 +137,8 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
}
checkPermission(databaseResp, user);
String sql = sqlExecuteReq.getSql();
sql =
SqlVariableParseUtils.parse(
sql, sqlExecuteReq.getSqlVariables(), Lists.newArrayList());
sql = SqlVariableParseUtils.parse(sql, sqlExecuteReq.getSqlVariables(),
Lists.newArrayList());
return executeSql(sql, databaseResp);
}
@@ -161,10 +162,8 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
}
// Add default parameters for unknown databases
if (!CollectionUtils.isEmpty(databaseList)) {
List<String> databaseTypeList =
databaseList.stream()
.map(databaseResp -> databaseResp.getType())
.collect(Collectors.toList());
List<String> databaseTypeList = databaseList.stream()
.map(databaseResp -> databaseResp.getType()).collect(Collectors.toList());
DefaultParametersBuilder defaultParametersBuilder = new DefaultParametersBuilder();
for (String dbType : databaseTypeList) {
if (!parametersBuilderMap.containsKey(dbType)) {
@@ -174,8 +173,7 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
}
// Add the OTHER type at the end
if (parametersBuilderMap.containsKey(EngineType.OTHER.getName())) {
result.put(
EngineType.OTHER.getName(),
result.put(EngineType.OTHER.getName(),
parametersBuilderMap.get(EngineType.OTHER.getName()).build());
}
return result;
@@ -240,14 +238,11 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
private void checkPermission(DatabaseResp databaseResp, User user) {
List<String> admins = databaseResp.getAdmins();
List<String> viewers = databaseResp.getViewers();
if (!admins.contains(user.getName())
&& !viewers.contains(user.getName())
if (!admins.contains(user.getName()) && !viewers.contains(user.getName())
&& !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName())
&& !user.isSuperAdmin()) {
String message =
String.format(
"您暂无当前数据库%s权限, 请联系数据库创建人:%s开通",
databaseResp.getName(), databaseResp.getCreatedBy());
String message = String.format("您暂无当前数据库%s权限, 请联系数据库创建人:%s开通", databaseResp.getName(),
databaseResp.getCreatedBy());
throw new RuntimeException(message);
}
}

View File

@@ -46,12 +46,8 @@ public class DictTaskServiceImpl implements DictTaskService {
private final FileHandler fileHandler;
private final DictWordService dictWordService;
public DictTaskServiceImpl(
DictRepository dictRepository,
DictUtils dictConverter,
DictUtils dictUtils,
FileHandler fileHandler,
DictWordService dictWordService) {
public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter,
DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService) {
this.dictRepository = dictRepository;
this.dictConverter = dictConverter;
this.dictUtils = dictUtils;
@@ -80,11 +76,8 @@ public class DictTaskServiceImpl implements DictTaskService {
}
private DictItemResp fetchDictItemResp(DictSingleTaskReq taskReq) {
DictItemFilter dictItemFilter =
DictItemFilter.builder()
.itemId(taskReq.getItemId())
.type(taskReq.getType())
.build();
DictItemFilter dictItemFilter = DictItemFilter.builder().itemId(taskReq.getItemId())
.type(taskReq.getType()).build();
List<DictItemResp> dictItemRespList = dictRepository.queryDictConf(dictItemFilter);
if (!CollectionUtils.isEmpty(dictItemRespList)) {
return dictItemRespList.get(0);
@@ -159,14 +152,9 @@ public class DictTaskServiceImpl implements DictTaskService {
@Override
public PageInfo<DictValueResp> queryDictValue(DictValueReq dictValueReq, User user) {
String fileName =
String.format(
"dic_value_%d_%s_%s",
dictValueReq.getModelId(),
dictValueReq.getType().name(),
dictValueReq.getItemId())
+ Constants.DOT
+ dictFileType;
String fileName = String.format("dic_value_%d_%s_%s", dictValueReq.getModelId(),
dictValueReq.getType().name(), dictValueReq.getItemId()) + Constants.DOT
+ dictFileType;
PageInfo<DictValueResp> dictValueRespList =
fileHandler.queryDictValue(fileName, dictValueReq);
return dictValueRespList;
@@ -174,14 +162,9 @@ public class DictTaskServiceImpl implements DictTaskService {
@Override
public String queryDictFilePath(DictValueReq dictValueReq, User user) {
String fileName =
String.format(
"dic_value_%d_%s_%s",
dictValueReq.getModelId(),
dictValueReq.getType().name(),
dictValueReq.getItemId())
+ Constants.DOT
+ dictFileType;
String fileName = String.format("dic_value_%d_%s_%s", dictValueReq.getModelId(),
dictValueReq.getType().name(), dictValueReq.getItemId()) + Constants.DOT
+ dictFileType;
return fileHandler.queryDictFilePath(fileName);
}
}

View File

@@ -22,8 +22,10 @@ import java.util.stream.Collectors;
@Slf4j
public class DictWordService {
@Autowired private SchemaService schemaService;
@Autowired private KnowledgeBaseService knowledgeBaseService;
@Autowired
private SchemaService schemaService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
private List<DictWord> preDictWords = new ArrayList<>();
@@ -37,8 +39,8 @@ public class DictWordService {
long startTime = System.currentTimeMillis();
List<DictWord> dictWords = getAllDictWords();
List<DictWord> preDictWords = getPreDictWords();
if (org.apache.commons.collections.CollectionUtils.isEqualCollection(
dictWords, preDictWords)) {
if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords,
preDictWords)) {
log.debug("Dictionary hasn't been reloaded.");
return;
}
@@ -61,8 +63,8 @@ public class DictWordService {
return words;
}
private void addWordsByType(
DictWordType value, List<SchemaElement> metas, List<DictWord> natures) {
private void addWordsByType(DictWordType value, List<SchemaElement> metas,
List<DictWord> natures) {
metas = distinct(metas);
List<DictWord> natureList = WordBuilderFactory.get(value).getDictWords(metas);
log.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
@@ -87,8 +89,6 @@ public class DictWordService {
return metas.stream()
.collect(
Collectors.toMap(SchemaElement::getId, Function.identity(), (e1, e2) -> e1))
.values()
.stream()
.collect(Collectors.toList());
.values().stream().collect(Collectors.toList());
}
}

View File

@@ -74,15 +74,12 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
private TagMetaService tagMetaService;
@Autowired private ApplicationEventPublisher eventPublisher;
@Autowired
private ApplicationEventPublisher eventPublisher;
public DimensionServiceImpl(
DimensionRepository dimensionRepository,
ModelService modelService,
AliasGenerateHelper aliasGenerateHelper,
DatabaseService databaseService,
ModelRelaService modelRelaService,
DataSetService dataSetService,
public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService,
AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService,
ModelRelaService modelRelaService, DataSetService dataSetService,
TagMetaService tagMetaService) {
this.modelService = modelService;
this.dimensionRepository = dimensionRepository;
@@ -110,48 +107,36 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
}
Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensions(modelId);
Map<String, DimensionResp> bizNameMap =
dimensionResps.stream()
.collect(
Collectors.toMap(
DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> nameMap =
dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> nameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
List<DimensionReq> dimensionToInsert = Lists.newArrayList();
dimensionReqs.stream()
.forEach(
dimension -> {
if (!bizNameMap.containsKey(dimension.getBizName())
&& !nameMap.containsKey(dimension.getName())) {
dimensionToInsert.add(dimension);
} else {
DimensionResp dimensionRespByBizName =
bizNameMap.get(dimension.getBizName());
DimensionResp dimensionRespByName =
nameMap.get(dimension.getName());
if (null != dimensionRespByBizName
&& isChange(dimension, dimensionRespByBizName)) {
dimension.setId(dimensionRespByBizName.getId());
this.updateDimension(dimension, user);
} else {
if (null != dimensionRespByName
&& isChange(dimension, dimensionRespByName)) {
dimension.setId(dimensionRespByName.getId());
this.updateDimension(dimension, user);
}
}
}
});
dimensionReqs.stream().forEach(dimension -> {
if (!bizNameMap.containsKey(dimension.getBizName())
&& !nameMap.containsKey(dimension.getName())) {
dimensionToInsert.add(dimension);
} else {
DimensionResp dimensionRespByBizName = bizNameMap.get(dimension.getBizName());
DimensionResp dimensionRespByName = nameMap.get(dimension.getName());
if (null != dimensionRespByBizName && isChange(dimension, dimensionRespByBizName)) {
dimension.setId(dimensionRespByBizName.getId());
this.updateDimension(dimension, user);
} else {
if (null != dimensionRespByName && isChange(dimension, dimensionRespByName)) {
dimension.setId(dimensionRespByName.getId());
this.updateDimension(dimension, user);
}
}
}
});
if (CollectionUtils.isEmpty(dimensionToInsert)) {
return;
}
List<DimensionDO> dimensionDOS =
dimensionToInsert.stream()
.peek(dimension -> dimension.createdBy(user.getName()))
.map(DimensionConverter::convert2DimensionDO)
.collect(Collectors.toList());
dimensionToInsert.stream().peek(dimension -> dimension.createdBy(user.getName()))
.map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
dimensionRepository.createDimensionBatch(dimensionDOS);
sendEventBatch(dimensionDOS, EventType.ADD);
}
@@ -166,13 +151,9 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
dimensionRepository.updateDimension(dimensionDO);
if (!oldName.equals(dimensionDO.getName())) {
sendEvent(
DataItem.builder()
.modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
.newName(dimensionReq.getName())
.name(oldName)
.type(TypeEnums.DIMENSION)
.id(dimensionDO.getId() + Constants.UNDERLINE)
.build(),
DataItem.builder().modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
.newName(dimensionReq.getName()).name(oldName).type(TypeEnums.DIMENSION)
.id(dimensionDO.getId() + Constants.UNDERLINE).build(),
EventType.UPDATE);
}
}
@@ -188,15 +169,11 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
if (CollectionUtils.isEmpty(dimensionDOS)) {
return;
}
dimensionDOS =
dimensionDOS.stream()
.peek(
dimensionDO -> {
dimensionDO.setStatus(metaBatchReq.getStatus());
dimensionDO.setUpdatedAt(new Date());
dimensionDO.setUpdatedBy(user.getName());
})
.collect(Collectors.toList());
dimensionDOS = dimensionDOS.stream().peek(dimensionDO -> {
dimensionDO.setStatus(metaBatchReq.getStatus());
dimensionDO.setUpdatedAt(new Date());
dimensionDO.setUpdatedBy(user.getName());
}).collect(Collectors.toList());
dimensionRepository.batchUpdateStatus(dimensionDOS);
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
@@ -303,8 +280,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
return getDimensions(new MetaFilter(Lists.newArrayList(modelId)));
}
private List<DimensionResp> filterByField(
List<DimensionResp> dimensionResps, List<String> fields) {
private List<DimensionResp> filterByField(List<DimensionResp> dimensionResps,
List<String> fields) {
List<DimensionResp> dimensionFiltered = Lists.newArrayList();
for (DimensionResp dimensionResp : dimensionResps) {
for (String field : fields) {
@@ -338,26 +315,17 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
List<DimensionResp> dimensionResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(dimensionDOS)) {
dimensionResps =
dimensionDOS.stream()
.map(
dimensionDO ->
DimensionConverter.convert2DimensionResp(
dimensionDO, modelMap))
.collect(Collectors.toList());
dimensionResps = dimensionDOS.stream().map(
dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelMap))
.collect(Collectors.toList());
}
return dimensionResps;
}
@Override
public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
String mockAlias =
aliasGenerateHelper.generateAlias(
mockType,
dimensionReq.getName(),
dimensionReq.getBizName(),
"",
dimensionReq.getDescription());
String mockAlias = aliasGenerateHelper.generateAlias(mockType, dimensionReq.getName(),
dimensionReq.getBizName(), "", dimensionReq.getDescription());
String ret = aliasGenerateHelper.extractJsonStringFromAiMessage(mockAlias);
return JSONObject.parseObject(ret, new TypeReference<List<String>>() {});
}
@@ -373,13 +341,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
}
DatabaseResp database = databaseService.getDatabase(modelResp.getDatabaseId());
String sql =
"select ai_talk."
+ dimensionReq.getBizName()
+ " from ("
+ sqlQuery
+ ") as ai_talk group by ai_talk."
+ dimensionReq.getBizName();
String sql = "select ai_talk." + dimensionReq.getBizName() + " from (" + sqlQuery
+ ") as ai_talk group by ai_talk." + dimensionReq.getBizName();
SemanticQueryResp semanticQueryResp = databaseService.executeSql(sql, database);
List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
List<String> valueList = new ArrayList<>();
@@ -403,11 +366,9 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
dimValueMap.setBizName("");
}
try {
dimValueMap.setAlias(
jsonObject
.getJSONObject("alias")
.getJSONArray(stringObjectMap.get(dimensionReq.getBizName()) + "")
.toJavaList(String.class));
dimValueMap.setAlias(jsonObject.getJSONObject("alias")
.getJSONArray(stringObjectMap.get(dimensionReq.getBizName()) + "")
.toJavaList(String.class));
} catch (Exception exception) {
dimValueMap.setAlias(null);
}
@@ -420,39 +381,29 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
private void checkExist(List<DimensionReq> dimensionReqs) {
Long modelId = dimensionReqs.get(0).getModelId();
List<DimensionResp> dimensionResps = getDimensions(modelId);
Map<String, DimensionResp> bizNameMap =
dimensionResps.stream()
.collect(
Collectors.toMap(
DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> nameMap =
dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, DimensionResp> nameMap = dimensionResps.stream()
.collect(Collectors.toMap(DimensionResp::getName, a -> a, (k1, k2) -> k1));
for (DimensionReq dimensionReq : dimensionReqs) {
String forbiddenCharacters =
NameCheckUtils.findForbiddenCharacters(dimensionReq.getName());
if (StringUtils.isNotBlank(forbiddenCharacters)) {
throw new InvalidArgumentException(
String.format(
"名称包含特殊字符, 请修改: %s特殊字符: %s",
dimensionReq.getName(), forbiddenCharacters));
throw new InvalidArgumentException(String.format("名称包含特殊字符, 请修改: %s特殊字符: %s",
dimensionReq.getName(), forbiddenCharacters));
}
if (bizNameMap.containsKey(dimensionReq.getBizName())) {
DimensionResp dimensionResp = bizNameMap.get(dimensionReq.getBizName());
if (!dimensionResp.getId().equals(dimensionReq.getId())) {
throw new RuntimeException(
String.format(
"该主题域下存在相同的维度字段名:%s 创建人:%s",
dimensionReq.getBizName(), dimensionResp.getCreatedBy()));
throw new RuntimeException(String.format("该主题域下存在相同的维度字段名:%s 创建人:%s",
dimensionReq.getBizName(), dimensionResp.getCreatedBy()));
}
}
if (nameMap.containsKey(dimensionReq.getName())) {
DimensionResp dimensionResp = nameMap.get(dimensionReq.getName());
if (!dimensionResp.getId().equals(dimensionReq.getId())) {
throw new RuntimeException(
String.format(
"该主题域下存在相同的维度名:%s 创建人:%s",
dimensionReq.getName(), dimensionResp.getCreatedBy()));
throw new RuntimeException(String.format("该主题域下存在相同的维度名:%s 创建人:%s",
dimensionReq.getName(), dimensionResp.getCreatedBy()));
}
}
}
@@ -478,19 +429,12 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
}
private DataEvent getDataEvent(List<DimensionDO> dimensionDOS, EventType eventType) {
List<DataItem> dataItems =
dimensionDOS.stream()
.map(
dimensionDO ->
DataItem.builder()
.id(dimensionDO.getId() + Constants.UNDERLINE)
.name(dimensionDO.getName())
.modelId(
dimensionDO.getModelId()
+ Constants.UNDERLINE)
.type(TypeEnums.DIMENSION)
.build())
.collect(Collectors.toList());
List<DataItem> dataItems = dimensionDOS.stream()
.map(dimensionDO -> DataItem.builder().id(dimensionDO.getId() + Constants.UNDERLINE)
.name(dimensionDO.getName())
.modelId(dimensionDO.getModelId() + Constants.UNDERLINE)
.type(TypeEnums.DIMENSION).build())
.collect(Collectors.toList());
return new DataEvent(this, dataItems, eventType);
}
@@ -499,10 +443,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
}
private boolean isChange(DimensionReq dimensionReq, DimensionResp dimensionResp) {
boolean isExtChange =
!new EqualsBuilder()
.append(dimensionReq.getExt(), dimensionResp.getExt())
.isEquals();
boolean isExtChange = !new EqualsBuilder()
.append(dimensionReq.getExt(), dimensionResp.getExt()).isEquals();
boolean isTypeParamChange =
!Objects.equals(dimensionReq.getTypeParams(), dimensionResp.getTypeParams());
return isExtChange || isTypeParamChange;

View File

@@ -41,9 +41,7 @@ public class DomainServiceImpl implements DomainService {
private final ModelService modelService;
private final UserService userService;
public DomainServiceImpl(
DomainRepository domainRepository,
@Lazy ModelService modelService,
public DomainServiceImpl(DomainRepository domainRepository, @Lazy ModelService modelService,
UserService userService) {
this.domainRepository = domainRepository;
this.modelService = modelService;
@@ -89,8 +87,7 @@ public class DomainServiceImpl implements DomainService {
@Override
public List<DomainResp> getDomainList(List<Long> domainIds) {
return getDomainList().stream()
.filter(domainDO -> domainIds.contains(domainDO.getId()))
return getDomainList().stream().filter(domainDO -> domainIds.contains(domainDO.getId()))
.collect(Collectors.toList());
}
@@ -113,10 +110,8 @@ public class DomainServiceImpl implements DomainService {
domainResp.setHasModel(true);
}
}
return new ArrayList<>(domainWithAuthAll)
.stream()
.sorted(Comparator.comparingLong(DomainResp::getId))
.collect(Collectors.toList());
return new ArrayList<>(domainWithAuthAll).stream()
.sorted(Comparator.comparingLong(DomainResp::getId)).collect(Collectors.toList());
}
@Override
@@ -125,19 +120,16 @@ public class DomainServiceImpl implements DomainService {
Set<String> orgIds = userService.getUserAllOrgId(user.getName());
Set<DomainResp> domainWithAuth = Sets.newHashSet();
if (authTypeEnum.equals(AuthType.ADMIN)) {
domainWithAuth =
domainResps.stream()
.filter(domainResp -> checkAdminPermission(orgIds, user, domainResp))
.collect(Collectors.toSet());
return domainWithAuth.stream()
.peek(domainResp -> domainResp.setHasEditPermission(true))
domainWithAuth = domainResps.stream()
.filter(domainResp -> checkAdminPermission(orgIds, user, domainResp))
.collect(Collectors.toSet());
return domainWithAuth.stream().peek(domainResp -> domainResp.setHasEditPermission(true))
.collect(Collectors.toSet());
}
if (authTypeEnum.equals(AuthType.VISIBLE)) {
domainWithAuth =
domainResps.stream()
.filter(domainResp -> checkViewPermission(orgIds, user, domainResp))
.collect(Collectors.toSet());
domainWithAuth = domainResps.stream()
.filter(domainResp -> checkViewPermission(orgIds, user, domainResp))
.collect(Collectors.toSet());
}
return domainWithAuth;
@@ -217,9 +209,8 @@ public class DomainServiceImpl implements DomainService {
public Map<Long, String> getDomainFullPathMap() {
Map<Long, String> domainFullPathMap = new HashMap<>();
List<DomainDO> domainDOList = domainRepository.getDomainList();
Map<Long, DomainDO> domainDOMap =
domainDOList.stream()
.collect(Collectors.toMap(DomainDO::getId, a -> a, (k1, k2) -> k1));
Map<Long, DomainDO> domainDOMap = domainDOList.stream()
.collect(Collectors.toMap(DomainDO::getId, a -> a, (k1, k2) -> k1));
for (DomainDO domainDO : domainDOList) {
final Long domainId = domainDO.getId();
StringBuilder fullPath = new StringBuilder(domainDO.getBizName() + "/");

View File

@@ -69,9 +69,7 @@ public class DownloadServiceImpl implements DownloadService {
private SemanticLayerService queryService;
public DownloadServiceImpl(
MetricService metricService,
DimensionService dimensionService,
public DownloadServiceImpl(MetricService metricService, DimensionService dimensionService,
SemanticLayerService queryService) {
this.metricService = metricService;
this.dimensionService = dimensionService;
@@ -79,9 +77,8 @@ public class DownloadServiceImpl implements DownloadService {
}
@Override
public void downloadByStruct(
DownloadMetricReq downloadMetricReq, User user, HttpServletResponse response)
throws Exception {
public void downloadByStruct(DownloadMetricReq downloadMetricReq, User user,
HttpServletResponse response) throws Exception {
String fileName =
String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), dateFormat));
File file = FileUtils.createTmpFile(fileName);
@@ -91,14 +88,10 @@ public class DownloadServiceImpl implements DownloadService {
queryService.queryByReq(queryStructReq.convert(true), user);
DataDownload dataDownload =
buildDataDownload(queryResult, queryStructReq, downloadMetricReq.isTransform());
EasyExcel.write(file)
.sheet("Sheet1")
.head(dataDownload.getHeaders())
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders())
.doWrite(dataDownload.getData());
} catch (RuntimeException e) {
EasyExcel.write(file)
.sheet("Sheet1")
.head(buildErrMessageHead())
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
.doWrite(buildErrMessageData(e.getMessage()));
return;
}
@@ -106,9 +99,8 @@ public class DownloadServiceImpl implements DownloadService {
}
@Override
public void batchDownload(
BatchDownloadReq batchDownloadReq, User user, HttpServletResponse response)
throws Exception {
public void batchDownload(BatchDownloadReq batchDownloadReq, User user,
HttpServletResponse response) throws Exception {
String fileName =
String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), dateFormat));
File file = FileUtils.createTmpFile(fileName);
@@ -127,16 +119,13 @@ public class DownloadServiceImpl implements DownloadService {
metaFilter.setIds(metricIds);
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
Map<String, List<MetricResp>> metricMap = getMetricMap(metricResps);
List<Long> dimensionIds =
metricResps.stream()
.map(metricResp -> metricService.getDrillDownDimension(metricResp.getId()))
.flatMap(Collection::stream)
.map(DrillDownDimension::getDimensionId)
.collect(Collectors.toList());
List<Long> dimensionIds = metricResps.stream()
.map(metricResp -> metricService.getDrillDownDimension(metricResp.getId()))
.flatMap(Collection::stream).map(DrillDownDimension::getDimensionId)
.collect(Collectors.toList());
metaFilter.setIds(dimensionIds);
Map<Long, DimensionResp> dimensionRespMap =
dimensionService.getDimensions(metaFilter).stream()
.collect(Collectors.toMap(DimensionResp::getId, d -> d));
Map<Long, DimensionResp> dimensionRespMap = dimensionService.getDimensions(metaFilter)
.stream().collect(Collectors.toMap(DimensionResp::getId, d -> d));
ExcelWriter excelWriter = EasyExcel.write(file).build();
int sheetCount = 1;
for (List<MetricResp> metrics : metricMap.values()) {
@@ -152,18 +141,13 @@ public class DownloadServiceImpl implements DownloadService {
QuerySqlReq querySqlReq = queryStructReq.convert();
querySqlReq.setNeedAuth(true);
SemanticQueryResp queryResult = queryService.queryByReq(querySqlReq, user);
DataDownload dataDownload =
buildDataDownload(
queryResult, queryStructReq, batchDownloadReq.isTransform());
WriteSheet writeSheet =
EasyExcel.writerSheet("Sheet" + sheetCount)
.head(dataDownload.getHeaders())
.build();
DataDownload dataDownload = buildDataDownload(queryResult, queryStructReq,
batchDownloadReq.isTransform());
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
.head(dataDownload.getHeaders()).build();
excelWriter.write(dataDownload.getData(), writeSheet);
} catch (RuntimeException e) {
EasyExcel.write(file)
.sheet("Sheet1")
.head(buildErrMessageHead())
EasyExcel.write(file).sheet("Sheet1").head(buildErrMessageHead())
.doWrite(buildErrMessageData(e.getMessage()));
return;
}
@@ -220,11 +204,8 @@ public class DownloadServiceImpl implements DownloadService {
return data;
}
private List<List<String>> buildData(
List<List<String>> headers,
Map<String, String> nameMap,
List<Map<String, Object>> dataTransformed,
String metricName) {
private List<List<String>> buildData(List<List<String>> headers, Map<String, String> nameMap,
List<Map<String, Object>> dataTransformed, String metricName) {
List<List<String>> data = Lists.newArrayList();
for (Map<String, Object> map : dataTransformed) {
List<String> row = Lists.newArrayList();
@@ -246,27 +227,20 @@ public class DownloadServiceImpl implements DownloadService {
return data;
}
private DataDownload buildDataDownload(
SemanticQueryResp queryResult, QueryStructReq queryStructReq, boolean isTransform) {
private DataDownload buildDataDownload(SemanticQueryResp queryResult,
QueryStructReq queryStructReq, boolean isTransform) {
List<QueryColumn> metricColumns = queryResult.getMetricColumns();
List<QueryColumn> dimensionColumns = queryResult.getDimensionColumns();
if (isTransform && !CollectionUtils.isEmpty(metricColumns)) {
QueryColumn metric = metricColumns.get(0);
List<String> groups = queryStructReq.getGroups();
List<Map<String, Object>> dataTransformed =
DataTransformUtils.transform(
queryResult.getResultList(),
metric.getNameEn(),
groups,
queryStructReq.getDateInfo());
DataTransformUtils.transform(queryResult.getResultList(), metric.getNameEn(),
groups, queryStructReq.getDateInfo());
List<List<String>> headers =
buildHeader(dimensionColumns, queryStructReq.getDateInfo().getDateList());
List<List<String>> data =
buildData(
headers,
getDimensionNameMap(dimensionColumns),
dataTransformed,
metric.getName());
List<List<String>> data = buildData(headers, getDimensionNameMap(dimensionColumns),
dataTransformed, metric.getName());
return DataDownload.builder().headers(headers).data(data).build();
} else {
List<List<String>> data = buildData(queryResult);
@@ -275,19 +249,15 @@ public class DownloadServiceImpl implements DownloadService {
}
}
private QueryStructReq buildDownloadReq(
List<DimensionResp> dimensionResps,
MetricResp metricResp,
BatchDownloadReq batchDownloadReq) {
private QueryStructReq buildDownloadReq(List<DimensionResp> dimensionResps,
MetricResp metricResp, BatchDownloadReq batchDownloadReq) {
DateConf dateConf = batchDownloadReq.getDateInfo();
Set<Long> modelIds =
dimensionResps.stream().map(DimensionResp::getModelId).collect(Collectors.toSet());
modelIds.add(metricResp.getModelId());
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setGroups(
dimensionResps.stream()
.map(DimensionResp::getBizName)
.collect(Collectors.toList()));
queryStructReq.setGroups(dimensionResps.stream().map(DimensionResp::getBizName)
.collect(Collectors.toList()));
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricResp.getBizName());
@@ -325,19 +295,15 @@ public class DownloadServiceImpl implements DownloadService {
.collect(Collectors.toMap(QueryColumn::getName, QueryColumn::getNameEn));
}
private List<DimensionResp> getMetricRelaDimensions(
MetricResp metricResp, Map<Long, DimensionResp> dimensionRespMap) {
if (metricResp.getRelateDimension() == null
|| CollectionUtils.isEmpty(
metricResp.getRelateDimension().getDrillDownDimensions())) {
private List<DimensionResp> getMetricRelaDimensions(MetricResp metricResp,
Map<Long, DimensionResp> dimensionRespMap) {
if (metricResp.getRelateDimension() == null || CollectionUtils
.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) {
return Lists.newArrayList();
}
return metricResp.getRelateDimension().getDrillDownDimensions().stream()
.map(
drillDownDimension ->
dimensionRespMap.get(drillDownDimension.getDimensionId()))
.filter(Objects::nonNull)
.collect(Collectors.toList());
return metricResp.getRelateDimension().getDrillDownDimensions().stream().map(
drillDownDimension -> dimensionRespMap.get(drillDownDimension.getDimensionId()))
.filter(Objects::nonNull).collect(Collectors.toList());
}
private void downloadFile(HttpServletResponse response, File file, String filename) {
@@ -345,8 +311,7 @@ public class DownloadServiceImpl implements DownloadService {
byte[] buffer = readFileToByteArray(file);
response.reset();
response.setCharacterEncoding("UTF-8");
response.addHeader(
"Content-Disposition",
response.addHeader("Content-Disposition",
"attachment;filename=" + URLEncoder.encode(filename, "UTF-8"));
response.addHeader("Content-Length", "" + file.length());
try (OutputStream outputStream = new BufferedOutputStream(response.getOutputStream())) {

View File

@@ -88,10 +88,8 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
private final AuthenticationConfig authenticationConfig;
private final UserService userService;
public FlightServiceImpl(
SemanticLayerService queryService,
AuthenticationConfig authenticationConfig,
UserService userService) {
public FlightServiceImpl(SemanticLayerService queryService,
AuthenticationConfig authenticationConfig, UserService userService) {
this.queryService = queryService;
this.authenticationConfig = authenticationConfig;
@@ -104,14 +102,11 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
}
@Override
public void setExecutorService(
ExecutorService executorService, Integer queue, Integer expireMinute) {
public void setExecutorService(ExecutorService executorService, Integer queue,
Integer expireMinute) {
this.executorService = executorService;
this.preparedStatementCache =
CacheBuilder.newBuilder()
.maximumSize(queue)
.expireAfterWrite(expireMinute, TimeUnit.MINUTES)
.build();
this.preparedStatementCache = CacheBuilder.newBuilder().maximumSize(queue)
.expireAfterWrite(expireMinute, TimeUnit.MINUTES).build();
}
@Override
@@ -120,26 +115,20 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
}
@Override
public void getStreamStatement(
final TicketStatementQuery ticketStatementQuery,
final CallContext context,
final ServerStreamListener listener) {
public void getStreamStatement(final TicketStatementQuery ticketStatementQuery,
final CallContext context, final ServerStreamListener listener) {
final ByteString handle = ticketStatementQuery.getStatementHandle();
log.info("getStreamStatement {} ", handle);
executeQuery(handle, listener);
}
@Override
public FlightInfo getFlightInfoStatement(
final CommandStatementQuery request,
final CallContext context,
final FlightDescriptor descriptor) {
public FlightInfo getFlightInfoStatement(final CommandStatementQuery request,
final CallContext context, final FlightDescriptor descriptor) {
try {
ByteString preparedStatementHandle = addPrepared(context, request.getQuery());
TicketStatementQuery ticket =
TicketStatementQuery.newBuilder()
.setStatementHandle(preparedStatementHandle)
.build();
TicketStatementQuery ticket = TicketStatementQuery.newBuilder()
.setStatementHandle(preparedStatementHandle).build();
return getFlightInfoForSchema(ticket, descriptor, null);
} catch (Exception e) {
log.error("getFlightInfoStatement error {}", e);
@@ -148,10 +137,8 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
}
@Override
public void getStreamPreparedStatement(
final CommandPreparedStatementQuery command,
final CallContext context,
final ServerStreamListener listener) {
public void getStreamPreparedStatement(final CommandPreparedStatementQuery command,
final CallContext context, final ServerStreamListener listener) {
log.info("getStreamPreparedStatement {}", command.getPreparedStatementHandle());
executeQuery(command.getPreparedStatementHandle(), listener);
}
@@ -160,117 +147,88 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
SemanticQueryReq semanticQueryReq = preparedStatementCache.getIfPresent(hander);
if (Objects.isNull(semanticQueryReq)) {
listener.error(
CallStatus.INTERNAL
.withDescription("Failed to get prepared statement: empty")
CallStatus.INTERNAL.withDescription("Failed to get prepared statement: empty")
.toRuntimeException());
log.error("getStreamPreparedStatement error {}", hander);
listener.completed();
return;
}
executorService.submit(
() -> {
BufferAllocator rootAllocator = new RootAllocator();
try {
Optional<Param> authOpt =
semanticQueryReq.getParams().stream()
.filter(
p ->
p.getName()
.equals(
authenticationConfig
.getTokenHttpHeaderKey()))
.findFirst();
if (authOpt.isPresent()) {
User user =
UserHolder.findUser(
authOpt.get().getValue(),
authenticationConfig.getTokenHttpHeaderAppKey());
SemanticQueryResp resp =
queryService.queryByReq(semanticQueryReq, user);
ResultSet resultSet =
semanticQueryRespToResultSet(
resp, semanticQueryReq.getDataSetId());
final Schema schema =
jdbcToArrowSchema(resultSet.getMetaData(), defaultCalendar);
try (final VectorSchemaRoot vectorSchemaRoot =
VectorSchemaRoot.create(schema, rootAllocator)) {
final VectorLoader loader = new VectorLoader(vectorSchemaRoot);
listener.start(vectorSchemaRoot);
final ArrowVectorIterator iterator =
sqlToArrowVectorIterator(resultSet, rootAllocator);
while (iterator.hasNext()) {
final VectorSchemaRoot batch = iterator.next();
if (batch.getRowCount() == 0) {
break;
}
final VectorUnloader unloader = new VectorUnloader(batch);
loader.load(unloader.getRecordBatch());
listener.putNext();
vectorSchemaRoot.clear();
}
listener.putNext();
executorService.submit(() -> {
BufferAllocator rootAllocator = new RootAllocator();
try {
Optional<Param> authOpt = semanticQueryReq.getParams().stream().filter(
p -> p.getName().equals(authenticationConfig.getTokenHttpHeaderKey()))
.findFirst();
if (authOpt.isPresent()) {
User user = UserHolder.findUser(authOpt.get().getValue(),
authenticationConfig.getTokenHttpHeaderAppKey());
SemanticQueryResp resp = queryService.queryByReq(semanticQueryReq, user);
ResultSet resultSet =
semanticQueryRespToResultSet(resp, semanticQueryReq.getDataSetId());
final Schema schema =
jdbcToArrowSchema(resultSet.getMetaData(), defaultCalendar);
try (final VectorSchemaRoot vectorSchemaRoot =
VectorSchemaRoot.create(schema, rootAllocator)) {
final VectorLoader loader = new VectorLoader(vectorSchemaRoot);
listener.start(vectorSchemaRoot);
final ArrowVectorIterator iterator =
sqlToArrowVectorIterator(resultSet, rootAllocator);
while (iterator.hasNext()) {
final VectorSchemaRoot batch = iterator.next();
if (batch.getRowCount() == 0) {
break;
}
final VectorUnloader unloader = new VectorUnloader(batch);
loader.load(unloader.getRecordBatch());
listener.putNext();
vectorSchemaRoot.clear();
}
} catch (Exception e) {
listener.error(
CallStatus.INTERNAL
.withDescription(
String.format(
"Failed to get exec statement %s",
e.getMessage()))
.toRuntimeException());
log.error("getStreamPreparedStatement error {}", hander);
} finally {
preparedStatementCache.invalidate(hander);
listener.completed();
rootAllocator.close();
listener.putNext();
}
});
}
} catch (Exception e) {
listener.error(CallStatus.INTERNAL
.withDescription(
String.format("Failed to get exec statement %s", e.getMessage()))
.toRuntimeException());
log.error("getStreamPreparedStatement error {}", hander);
} finally {
preparedStatementCache.invalidate(hander);
listener.completed();
rootAllocator.close();
}
});
}
@Override
public void closePreparedStatement(
final ActionClosePreparedStatementRequest request,
final CallContext context,
final StreamListener<Result> listener) {
public void closePreparedStatement(final ActionClosePreparedStatementRequest request,
final CallContext context, final StreamListener<Result> listener) {
log.info("closePreparedStatement {}", request.getPreparedStatementHandle());
listener.onCompleted();
}
@Override
public FlightInfo getFlightInfoPreparedStatement(
final CommandPreparedStatementQuery command,
final CallContext context,
final FlightDescriptor descriptor) {
public FlightInfo getFlightInfoPreparedStatement(final CommandPreparedStatementQuery command,
final CallContext context, final FlightDescriptor descriptor) {
return getFlightInfoForSchema(command, descriptor, null);
}
@Override
public void createPreparedStatement(
final ActionCreatePreparedStatementRequest request,
final CallContext context,
final StreamListener<Result> listener) {
public void createPreparedStatement(final ActionCreatePreparedStatementRequest request,
final CallContext context, final StreamListener<Result> listener) {
prepared(request, context, listener);
}
private ByteString addPrepared(final CallContext context, String query) throws Exception {
if (Arrays.asList(dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey).stream()
.anyMatch(
h ->
!context.getMiddleware(FlightConstants.HEADER_KEY)
.headers()
.containsKey(h))) {
throw new Exception(
String.format(
"Failed to create prepared statement: HeaderCallOption miss %s %s %s",
dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey));
if (Arrays.asList(dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey).stream().anyMatch(
h -> !context.getMiddleware(FlightConstants.HEADER_KEY).headers().containsKey(h))) {
throw new Exception(String.format(
"Failed to create prepared statement: HeaderCallOption miss %s %s %s",
dataSetIdHeaderKey, nameHeaderKey, passwordHeaderKey));
}
Long dataSetId =
Long.valueOf(
context.getMiddleware(FlightConstants.HEADER_KEY)
.headers()
.get(dataSetIdHeaderKey));
Long dataSetId = Long.valueOf(context.getMiddleware(FlightConstants.HEADER_KEY).headers()
.get(dataSetIdHeaderKey));
if (StringUtils.isBlank(query)) {
throw new Exception("Failed to create prepared statement: query is empty");
}
@@ -287,34 +245,28 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
querySqlReq.setParams(
Arrays.asList(new Param(authenticationConfig.getTokenHttpHeaderKey(), auth)));
preparedStatementCache.put(preparedStatementHandle, querySqlReq);
log.info(
"createPreparedStatement {} {} {} ", preparedStatementHandle, dataSetId, query);
log.info("createPreparedStatement {} {} {} ", preparedStatementHandle, dataSetId,
query);
return preparedStatementHandle;
} catch (Exception e) {
throw e;
}
}
private void prepared(
final ActionCreatePreparedStatementRequest request,
final CallContext context,
final StreamListener<Result> listener) {
private void prepared(final ActionCreatePreparedStatementRequest request,
final CallContext context, final StreamListener<Result> listener) {
try {
ByteString preparedStatementHandle = addPrepared(context, request.getQuery());
final ActionCreatePreparedStatementResult result =
ActionCreatePreparedStatementResult.newBuilder()
.setDatasetSchema(ByteString.EMPTY)
.setParameterSchema(ByteString.empty())
.setPreparedStatementHandle(preparedStatementHandle)
.build();
final ActionCreatePreparedStatementResult result = ActionCreatePreparedStatementResult
.newBuilder().setDatasetSchema(ByteString.EMPTY)
.setParameterSchema(ByteString.empty())
.setPreparedStatementHandle(preparedStatementHandle).build();
listener.onNext(new Result(pack(result).toByteArray()));
} catch (Exception e) {
listener.onError(
CallStatus.INTERNAL
.withDescription(
String.format(
"Failed to create prepared statement: %s",
e.getMessage()))
.withDescription(String.format(
"Failed to create prepared statement: %s", e.getMessage()))
.toRuntimeException());
} finally {
listener.onCompleted();
@@ -322,13 +274,13 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
}
@Override
protected <T extends Message> List<FlightEndpoint> determineEndpoints(
T t, FlightDescriptor flightDescriptor, Schema schema) {
protected <T extends Message> List<FlightEndpoint> determineEndpoints(T t,
FlightDescriptor flightDescriptor, Schema schema) {
throw CallStatus.UNIMPLEMENTED.withDescription("Not implemented.").toRuntimeException();
}
private <T extends Message> FlightInfo getFlightInfoForSchema(
final T request, final FlightDescriptor descriptor, final Schema schema) {
private <T extends Message> FlightInfo getFlightInfoForSchema(final T request,
final FlightDescriptor descriptor, final Schema schema) {
final Ticket ticket = new Ticket(pack(request).toByteArray());
Location listenLocation = Location.forGrpcInsecure(host, port);
final List<FlightEndpoint> endpoints =
@@ -359,13 +311,9 @@ public class FlightServiceImpl extends BasicFlightSqlProducer implements FlightS
for (int i = 1; i <= columnNum; i++) {
String columnName = resp.getColumns().get(i - 1).getNameEn();
rowSetMetaData.setColumnName(i, columnName);
Optional<Map<String, Object>> valOpt =
resp.getResultList().stream()
.filter(
r ->
r.containsKey(columnName)
&& Objects.nonNull(r.get(columnName)))
.findFirst();
Optional<Map<String, Object>> valOpt = resp.getResultList().stream()
.filter(r -> r.containsKey(columnName) && Objects.nonNull(r.get(columnName)))
.findFirst();
if (valOpt.isPresent()) {
int type = FlightUtils.resolveType(valOpt.get());
rowSetMetaData.setColumnType(i, type);

View File

@@ -110,15 +110,10 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
private ChatLayerService chatLayerService;
public MetricServiceImpl(
MetricRepository metricRepository,
ModelService modelService,
AliasGenerateHelper aliasGenerateHelper,
CollectService collectService,
DataSetService dataSetService,
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService,
TagMetaService tagMetaService,
public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService,
AliasGenerateHelper aliasGenerateHelper, CollectService collectService,
DataSetService dataSetService, ApplicationEventPublisher eventPublisher,
DimensionService dimensionService, TagMetaService tagMetaService,
@Lazy ChatLayerService chatLayerService) {
this.metricRepository = metricRepository;
this.modelService = modelService;
@@ -149,27 +144,21 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}
Long modelId = metricReqs.get(0).getModelId();
List<MetricResp> metricResps = getMetrics(new MetaFilter(Lists.newArrayList(modelId)));
Map<String, MetricResp> bizNameMap =
metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap =
metricResps.stream()
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> bizNameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
List<MetricReq> metricToInsert =
metricReqs.stream()
.filter(
metric ->
!bizNameMap.containsKey(metric.getBizName())
&& !nameMap.containsKey(metric.getName()))
.filter(metric -> !bizNameMap.containsKey(metric.getBizName())
&& !nameMap.containsKey(metric.getName()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(metricToInsert)) {
return;
}
List<MetricDO> metricDOS =
metricToInsert.stream()
.peek(metric -> metric.createdBy(user.getName()))
.map(MetricConverter::convert2MetricDO)
.collect(Collectors.toList());
metricToInsert.stream().peek(metric -> metric.createdBy(user.getName()))
.map(MetricConverter::convert2MetricDO).collect(Collectors.toList());
metricRepository.createMetricBatch(metricDOS);
sendEventBatch(metricDOS, EventType.ADD);
}
@@ -201,15 +190,11 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
if (CollectionUtils.isEmpty(metricDOS)) {
return;
}
metricDOS =
metricDOS.stream()
.peek(
metricDO -> {
metricDO.setStatus(metaBatchReq.getStatus());
metricDO.setUpdatedAt(new Date());
metricDO.setUpdatedBy(user.getName());
})
.collect(Collectors.toList());
metricDOS = metricDOS.stream().peek(metricDO -> {
metricDO.setStatus(metaBatchReq.getStatus());
metricDO.setUpdatedAt(new Date());
metricDO.setUpdatedBy(user.getName());
}).collect(Collectors.toList());
metricRepository.batchUpdateStatus(metricDOS);
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
@@ -314,20 +299,14 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
return metricRespPageInfo;
}
Map<Long, Double> result =
dataSetMapInfoMap.values().stream()
.map(DataSetMapInfo::getMapFields)
.filter(Objects::nonNull)
.flatMap(Collection::stream)
.filter(
schemaElementMatch ->
SchemaElementType.METRIC.equals(
schemaElementMatch.getElement().getType()))
.collect(
Collectors.toMap(
schemaElementMatch ->
schemaElementMatch.getElement().getId(),
SchemaElementMatch::getSimilarity,
(existingValue, newValue) -> existingValue));
dataSetMapInfoMap.values().stream().map(DataSetMapInfo::getMapFields)
.filter(Objects::nonNull).flatMap(Collection::stream)
.filter(schemaElementMatch -> SchemaElementType.METRIC
.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toMap(
schemaElementMatch -> schemaElementMatch.getElement().getId(),
SchemaElementMatch::getSimilarity,
(existingValue, newValue) -> existingValue));
List<Long> metricIds = new ArrayList<>(result.keySet());
if (CollectionUtils.isEmpty(result.keySet())) {
return metricRespPageInfo;
@@ -434,20 +413,13 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
return new ArrayList<>(metricRespFiltered);
}
private boolean filterByField(
List<MetricResp> metricResps,
MetricResp metricResp,
List<String> fields,
Set<MetricResp> metricRespFiltered) {
private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
List<String> fields, Set<MetricResp> metricRespFiltered) {
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
List<Long> ids =
metricResp.getMetricDefineByMetricParams().getMetrics().stream()
.map(MetricParam::getId)
.collect(Collectors.toList());
List<MetricResp> metricById =
metricResps.stream()
.filter(metric -> ids.contains(metric.getId()))
.collect(Collectors.toList());
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics().stream()
.map(MetricParam::getId).collect(Collectors.toList());
List<MetricResp> metricById = metricResps.stream()
.filter(metric -> ids.contains(metric.getId())).collect(Collectors.toList());
for (MetricResp metric : metricById) {
if (filterByField(metricResps, metric, fields, metricRespFiltered)) {
metricRespFiltered.add(metricResp);
@@ -461,12 +433,10 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}
} else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
List<MeasureParam> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
List<String> fieldNameDepended =
measures.stream()
.map(MeasureParam::getBizName)
// measure bizName = model bizName_fieldName
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
.collect(Collectors.toList());
List<String> fieldNameDepended = measures.stream().map(MeasureParam::getBizName)
// measure bizName = model bizName_fieldName
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
.collect(Collectors.toList());
if (fields.stream().anyMatch(fieldNameDepended::contains)) {
metricRespFiltered.add(metricResp);
return true;
@@ -480,12 +450,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
MetricFilter metricFilter = new MetricFilter();
metricFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metricFilter);
return metricResps.stream()
.filter(
metricResp ->
MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(
metricResp.getMetricDefineType()))
return metricResps.stream().filter(
metricResp -> MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
.collect(Collectors.toList());
}
@@ -546,19 +513,10 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
@Override
public List<String> mockAlias(MetricBaseReq metricReq, String mockType, User user) {
String mockAlias =
aliasGenerateHelper.generateAlias(
mockType,
metricReq.getName(),
metricReq.getBizName(),
"",
metricReq.getDescription());
String ret =
mockAlias
.replaceAll("`", "")
.replace("json", "")
.replace("\n", "")
.replace(" ", "");
String mockAlias = aliasGenerateHelper.generateAlias(mockType, metricReq.getName(),
metricReq.getBizName(), "", metricReq.getDescription());
String ret = mockAlias.replaceAll("`", "").replace("json", "").replace("\n", "")
.replace(" ", "");
return JSONObject.parseObject(ret, new TypeReference<List<String>>() {});
}
@@ -568,8 +526,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
if (CollectionUtils.isEmpty(metricResps)) {
return new HashSet<>();
}
return metricResps.stream()
.flatMap(metricResp -> metricResp.getClassifications().stream())
return metricResps.stream().flatMap(metricResp -> metricResp.getClassifications().stream())
.collect(Collectors.toSet());
}
@@ -580,11 +537,10 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
if (metricResp == null) {
return drillDownDimensions;
}
if (metricResp.getRelateDimension() != null
&& !CollectionUtils.isEmpty(
metricResp.getRelateDimension().getDrillDownDimensions())) {
for (DrillDownDimension drillDownDimension :
metricResp.getRelateDimension().getDrillDownDimensions()) {
if (metricResp.getRelateDimension() != null && !CollectionUtils
.isEmpty(metricResp.getRelateDimension().getDrillDownDimensions())) {
for (DrillDownDimension drillDownDimension : metricResp.getRelateDimension()
.getDrillDownDimensions()) {
if (drillDownDimension.isInheritedFromModel()
&& !drillDownDimension.isNecessary()) {
continue;
@@ -597,10 +553,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
return drillDownDimensions;
}
for (DrillDownDimension drillDownDimension : modelResp.getDrillDownDimensions()) {
if (!drillDownDimensions.stream()
.map(DrillDownDimension::getDimensionId)
.collect(Collectors.toList())
.contains(drillDownDimension.getDimensionId())) {
if (!drillDownDimensions.stream().map(DrillDownDimension::getDimensionId)
.collect(Collectors.toList()).contains(drillDownDimension.getDimensionId())) {
drillDownDimension.setInheritedFromModel(true);
drillDownDimensions.add(drillDownDimension);
}
@@ -639,29 +593,23 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
MetaFilter metaFilter = new MetaFilter();
metaFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metaFilter);
Map<String, MetricResp> bizNameMap =
metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap =
metricResps.stream()
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> bizNameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String, MetricResp> nameMap = metricResps.stream()
.collect(Collectors.toMap(MetricResp::getName, a -> a, (k1, k2) -> k1));
for (MetricBaseReq metricReq : metricReqs) {
if (bizNameMap.containsKey(metricReq.getBizName())) {
MetricResp metricResp = bizNameMap.get(metricReq.getBizName());
if (!metricResp.getId().equals(metricReq.getId())) {
throw new RuntimeException(
String.format(
"该模型下存在相同的指标字段名:%s 创建人:%s",
metricReq.getBizName(), metricResp.getCreatedBy()));
throw new RuntimeException(String.format("该模型下存在相同的指标字段名:%s 创建人:%s",
metricReq.getBizName(), metricResp.getCreatedBy()));
}
}
if (nameMap.containsKey(metricReq.getName())) {
MetricResp metricResp = nameMap.get(metricReq.getName());
if (!metricResp.getId().equals(metricReq.getId())) {
throw new RuntimeException(
String.format(
"该模型下存在相同的指标名:%s 创建人:%s",
metricReq.getName(), metricResp.getCreatedBy()));
throw new RuntimeException(String.format("该模型下存在相同的指标名:%s 创建人:%s",
metricReq.getName(), metricResp.getCreatedBy()));
}
}
}
@@ -678,13 +626,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
ModelFilter modelFilter = new ModelFilter(false, modelIds);
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
if (!CollectionUtils.isEmpty(metricDOS)) {
metricResps =
metricDOS.stream()
.map(
metricDO ->
MetricConverter.convert2MetricResp(
metricDO, modelMap, collect))
.collect(Collectors.toList());
metricResps = metricDOS.stream().map(
metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect))
.collect(Collectors.toList());
}
return metricResps;
}
@@ -729,19 +673,15 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
MetricResp metricResp =
MetricConverter.convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
fillDefaultAgg(metricResp);
return DataItem.builder()
.id(metricDO.getId() + Constants.UNDERLINE)
.name(metricDO.getName())
.bizName(metricDO.getBizName())
.modelId(metricDO.getModelId() + Constants.UNDERLINE)
.type(TypeEnums.METRIC)
.defaultAgg(metricResp.getDefaultAgg())
.build();
return DataItem.builder().id(metricDO.getId() + Constants.UNDERLINE)
.name(metricDO.getName()).bizName(metricDO.getBizName())
.modelId(metricDO.getModelId() + Constants.UNDERLINE).type(TypeEnums.METRIC)
.defaultAgg(metricResp.getDefaultAgg()).build();
}
@Override
public void batchFillMetricDefaultAgg(
List<MetricResp> metricResps, List<ModelResp> modelResps) {
public void batchFillMetricDefaultAgg(List<MetricResp> metricResps,
List<ModelResp> modelResps) {
Map<Long, ModelResp> modelRespMap =
modelResps.stream().collect(Collectors.toMap(ModelResp::getId, m -> m));
for (MetricResp metricResp : metricResps) {
@@ -762,9 +702,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}
private String getDefaultAgg(MetricResp metricResp, ModelResp modelResp) {
if (modelResp == null
|| (Objects.nonNull(metricResp.getDefaultAgg())
&& !metricResp.getDefaultAgg().isEmpty())) {
if (modelResp == null || (Objects.nonNull(metricResp.getDefaultAgg())
&& !metricResp.getDefaultAgg().isEmpty())) {
return metricResp.getDefaultAgg();
}
// FIELD define will get from expr
@@ -824,20 +763,12 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
queryMetricReq.setDateInfo(null);
}
// 4. set groups
List<String> dimensionBizNames =
dimensionResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.filter(
entry ->
queryMetricReq.getDimensionNames().contains(entry.getName())
|| queryMetricReq
.getDimensionNames()
.contains(entry.getBizName())
|| queryMetricReq
.getDimensionIds()
.contains(entry.getId()))
.map(SchemaItem::getBizName)
.collect(Collectors.toList());
List<String> dimensionBizNames = dimensionResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.filter(entry -> queryMetricReq.getDimensionNames().contains(entry.getName())
|| queryMetricReq.getDimensionNames().contains(entry.getBizName())
|| queryMetricReq.getDimensionIds().contains(entry.getId()))
.map(SchemaItem::getBizName).collect(Collectors.toList());
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateInfo = queryMetricReq.getDateInfo();
@@ -848,11 +779,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
queryStructReq.getGroups().addAll(dimensionBizNames);
}
// 5. set aggregators
List<String> metricBizNames =
metricResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.map(SchemaItem::getBizName)
.collect(Collectors.toList());
List<String> metricBizNames = metricResps.stream()
.filter(entry -> modelCluster.getModelIds().contains(entry.getModelId()))
.map(SchemaItem::getBizName).collect(Collectors.toList());
if (CollectionUtils.isEmpty(metricBizNames)) {
throw new IllegalArgumentException(
"Invalid input parameters, unable to obtain valid metrics");
@@ -898,18 +827,14 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}
}
}
String keyWithMaxSize =
modelClusterToMatchCount.entrySet().stream()
.max(Comparator.comparingInt(entry -> entry.getValue().size()))
.map(Map.Entry::getKey)
.orElse(null);
String keyWithMaxSize = modelClusterToMatchCount.entrySet().stream()
.max(Comparator.comparingInt(entry -> entry.getValue().size()))
.map(Map.Entry::getKey).orElse(null);
return modelClusterMap.get(keyWithMaxSize);
}
private Set<Long> getModelIds(
Set<Long> modelIdsByDomainId,
List<MetricResp> metricResps,
private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
List<DimensionResp> dimensionResps) {
Set<Long> result = new HashSet<>();
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
@@ -920,10 +845,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
metricResps.stream().map(entry -> entry.getModelId()).collect(Collectors.toSet());
result.addAll(metricModelIds);
Set<Long> dimensionModelIds =
dimensionResps.stream()
.map(entry -> entry.getModelId())
.collect(Collectors.toSet());
Set<Long> dimensionModelIds = dimensionResps.stream().map(entry -> entry.getModelId())
.collect(Collectors.toSet());
result.addAll(dimensionModelIds);
return result;
}
@@ -942,9 +865,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}
private Set<Long> getModelIdsByDomainId(QueryMetricReq queryMetricReq) {
List<ModelResp> modelResps =
modelService.getAllModelByDomainIds(
Collections.singletonList(queryMetricReq.getDomainId()));
List<ModelResp> modelResps = modelService
.getAllModelByDomainIds(Collections.singletonList(queryMetricReq.getDomainId()));
return modelResps.stream().map(ModelResp::getId).collect(Collectors.toSet());
}
}

View File

@@ -48,9 +48,7 @@ public class ModelRelaServiceImpl extends ServiceImpl<ModelRelaDOMapper, ModelRe
if (CollectionUtils.isEmpty(modelIds)) {
return Lists.newArrayList();
}
wrapper.lambda()
.in(ModelRelaDO::getFromModelId, modelIds)
.or()
wrapper.lambda().in(ModelRelaDO::getFromModelId, modelIds).or()
.in(ModelRelaDO::getToModelId, modelIds);
return list(wrapper).stream().map(this::convert).collect(Collectors.toList());
}

View File

@@ -78,14 +78,9 @@ public class ModelServiceImpl implements ModelService {
private DateInfoRepository dateInfoRepository;
public ModelServiceImpl(
ModelRepository modelRepository,
DatabaseService databaseService,
@Lazy DimensionService dimensionService,
@Lazy MetricService metricService,
DomainService domainService,
UserService userService,
DataSetService dataSetService,
public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
@Lazy DimensionService dimensionService, @Lazy MetricService metricService,
DomainService domainService, UserService userService, DataSetService dataSetService,
DateInfoRepository dateInfoRepository) {
this.modelRepository = modelRepository;
this.databaseService = databaseService;
@@ -185,9 +180,7 @@ public class ModelServiceImpl implements ModelService {
metaFilter.setFieldsDepend(fieldRemovedReq.getFields());
List<MetricResp> metricResps = metricService.getMetrics(metaFilter);
List<DimensionResp> dimensionResps = dimensionService.getDimensions(metaFilter);
return UnAvailableItemResp.builder()
.dimensionResps(dimensionResps)
.metricResps(metricResps)
return UnAvailableItemResp.builder().dimensionResps(dimensionResps).metricResps(metricResps)
.build();
}
@@ -204,9 +197,8 @@ public class ModelServiceImpl implements ModelService {
private void checkParams(ModelReq modelReq) {
String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(modelReq.getName());
if (StringUtils.isNotBlank(forbiddenCharacters)) {
String message =
String.format(
"模型名称[%s]包含特殊字符(%s), 请修改", modelReq.getName(), forbiddenCharacters);
String message = String.format("模型名称[%s]包含特殊字符(%s), 请修改", modelReq.getName(),
forbiddenCharacters);
throw new InvalidArgumentException(message);
}
@@ -228,10 +220,8 @@ public class ModelServiceImpl implements ModelService {
NameCheckUtils.findForbiddenCharacters(measure.getName());
if (StringUtils.isNotBlank(measure.getName())
&& StringUtils.isNotBlank(measureForbiddenCharacters)) {
String message =
String.format(
"度量[%s]包含特殊字符(%s), 请修改",
measure.getName(), measureForbiddenCharacters);
String message = String.format("度量[%s]包含特殊字符(%s), 请修改", measure.getName(),
measureForbiddenCharacters);
throw new InvalidArgumentException(message);
}
}
@@ -239,9 +229,8 @@ public class ModelServiceImpl implements ModelService {
String dimForbiddenCharacters = NameCheckUtils.findForbiddenCharacters(dim.getName());
if (StringUtils.isNotBlank(dim.getName())
&& StringUtils.isNotBlank(dimForbiddenCharacters)) {
String message =
String.format(
"维度[%s]包含特殊字符(%s), 请修改", dim.getName(), dimForbiddenCharacters);
String message = String.format("维度[%s]包含特殊字符(%s), 请修改", dim.getName(),
dimForbiddenCharacters);
throw new InvalidArgumentException(message);
}
}
@@ -250,10 +239,8 @@ public class ModelServiceImpl implements ModelService {
NameCheckUtils.findForbiddenCharacters(identify.getName());
if (StringUtils.isNotBlank(identify.getName())
&& StringUtils.isNotBlank(identifyForbiddenCharacters)) {
String message =
String.format(
"主键/外键[%s]包含特殊字符(%s), 请修改",
identify.getName(), identifyForbiddenCharacters);
String message = String.format("主键/外键[%s]包含特殊字符(%s), 请修改", identify.getName(),
identifyForbiddenCharacters);
throw new InvalidArgumentException(message);
}
}
@@ -304,26 +291,21 @@ public class ModelServiceImpl implements ModelService {
List<ModelResp> modelRespsAuthInheritDomain =
getModelRespAuthInheritDomain(user, domainId, authType);
modelRespSet.addAll(modelRespsAuthInheritDomain);
return modelRespSet.stream()
.sorted(Comparator.comparingLong(ModelResp::getId))
return modelRespSet.stream().sorted(Comparator.comparingLong(ModelResp::getId))
.collect(Collectors.toList());
}
public List<ModelResp> getModelRespAuthInheritDomain(
User user, Long domainId, AuthType authType) {
public List<ModelResp> getModelRespAuthInheritDomain(User user, Long domainId,
AuthType authType) {
List<Long> domainIds =
domainService.getDomainAuthSet(user, authType).stream()
.filter(
domainResp -> {
if (domainId == null) {
return true;
} else {
return domainId.equals(domainResp.getId())
|| domainId.equals(domainResp.getParentId());
}
})
.map(DomainResp::getId)
.collect(Collectors.toList());
domainService.getDomainAuthSet(user, authType).stream().filter(domainResp -> {
if (domainId == null) {
return true;
} else {
return domainId.equals(domainResp.getId())
|| domainId.equals(domainResp.getParentId());
}
}).map(DomainResp::getId).collect(Collectors.toList());
if (CollectionUtils.isEmpty(domainIds)) {
return Lists.newArrayList();
}
@@ -342,16 +324,14 @@ public class ModelServiceImpl implements ModelService {
Set<String> orgIds = userService.getUserAllOrgId(user.getName());
List<ModelResp> modelWithAuth = Lists.newArrayList();
if (authTypeEnum.equals(AuthType.ADMIN)) {
modelWithAuth =
modelResps.stream()
.filter(modelResp -> checkAdminPermission(orgIds, user, modelResp))
.collect(Collectors.toList());
modelWithAuth = modelResps.stream()
.filter(modelResp -> checkAdminPermission(orgIds, user, modelResp))
.collect(Collectors.toList());
}
if (authTypeEnum.equals(AuthType.VISIBLE)) {
modelWithAuth =
modelResps.stream()
.filter(domainResp -> checkDataSetPermission(orgIds, user, domainResp))
.collect(Collectors.toList());
modelWithAuth = modelResps.stream()
.filter(domainResp -> checkDataSetPermission(orgIds, user, domainResp))
.collect(Collectors.toList());
}
return modelWithAuth;
}
@@ -368,8 +348,7 @@ public class ModelServiceImpl implements ModelService {
if (CollectionUtils.isEmpty(modelResps)) {
return modelResps;
}
return modelResps.stream()
.filter(modelResp -> domainIds.contains(modelResp.getDomainId()))
return modelResps.stream().filter(modelResp -> domainIds.contains(modelResp.getDomainId()))
.collect(Collectors.toList());
}
@@ -434,35 +413,23 @@ public class ModelServiceImpl implements ModelService {
if (CollectionUtils.isEmpty(modelDOS)) {
return;
}
modelDOS =
modelDOS.stream()
.peek(
modelDO -> {
modelDO.setStatus(metaBatchReq.getStatus());
modelDO.setUpdatedAt(new Date());
modelDO.setUpdatedBy(user.getName());
if (StatusEnum.OFFLINE
.getCode()
.equals(metaBatchReq.getStatus())
|| StatusEnum.DELETED
.getCode()
.equals(metaBatchReq.getStatus())) {
metricService.sendMetricEventBatch(
Lists.newArrayList(modelDO.getId()),
EventType.DELETE);
dimensionService.sendDimensionEventBatch(
Lists.newArrayList(modelDO.getId()),
EventType.DELETE);
} else if (StatusEnum.ONLINE
.getCode()
.equals(metaBatchReq.getStatus())) {
metricService.sendMetricEventBatch(
Lists.newArrayList(modelDO.getId()), EventType.ADD);
dimensionService.sendDimensionEventBatch(
Lists.newArrayList(modelDO.getId()), EventType.ADD);
}
})
.collect(Collectors.toList());
modelDOS = modelDOS.stream().peek(modelDO -> {
modelDO.setStatus(metaBatchReq.getStatus());
modelDO.setUpdatedAt(new Date());
modelDO.setUpdatedBy(user.getName());
if (StatusEnum.OFFLINE.getCode().equals(metaBatchReq.getStatus())
|| StatusEnum.DELETED.getCode().equals(metaBatchReq.getStatus())) {
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()),
EventType.DELETE);
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()),
EventType.DELETE);
} else if (StatusEnum.ONLINE.getCode().equals(metaBatchReq.getStatus())) {
metricService.sendMetricEventBatch(Lists.newArrayList(modelDO.getId()),
EventType.ADD);
dimensionService.sendDimensionEventBatch(Lists.newArrayList(modelDO.getId()),
EventType.ADD);
}
}).collect(Collectors.toList());
modelRepository.batchUpdate(modelDOS);
}
@@ -472,14 +439,13 @@ public class ModelServiceImpl implements ModelService {
private List<DateInfoReq> convert(List<DateInfoDO> dateInfoDOList) {
List<DateInfoReq> dateInfoCommendList = new ArrayList<>();
dateInfoDOList.forEach(
dateInfoDO -> {
DateInfoReq dateInfoCommend = new DateInfoReq();
BeanUtils.copyProperties(dateInfoDO, dateInfoCommend);
dateInfoCommend.setUnavailableDateList(
JsonUtil.toList(dateInfoDO.getUnavailableDateList(), String.class));
dateInfoCommendList.add(dateInfoCommend);
});
dateInfoDOList.forEach(dateInfoDO -> {
DateInfoReq dateInfoCommend = new DateInfoReq();
BeanUtils.copyProperties(dateInfoDO, dateInfoCommend);
dateInfoCommend.setUnavailableDateList(
JsonUtil.toList(dateInfoDO.getUnavailableDateList(), String.class));
dateInfoCommendList.add(dateInfoCommend);
});
return dateInfoCommendList;
}
@@ -504,8 +470,8 @@ public class ModelServiceImpl implements ModelService {
return false;
}
public static boolean checkDataSetPermission(
Set<String> orgIds, User user, ModelResp modelResp) {
public static boolean checkDataSetPermission(Set<String> orgIds, User user,
ModelResp modelResp) {
if (checkAdminPermission(orgIds, user, modelResp)) {
return true;
}

View File

@@ -26,8 +26,8 @@ public class QueryRuleServiceImpl implements QueryRuleService {
private final QueryRuleRepository queryRuleRepository;
private final DataSetService dataSetService;
public QueryRuleServiceImpl(
QueryRuleRepository queryRuleRepository, DataSetService dataSetService) {
public QueryRuleServiceImpl(QueryRuleRepository queryRuleRepository,
DataSetService dataSetService) {
this.queryRuleRepository = queryRuleRepository;
this.dataSetService = dataSetService;
}

View File

@@ -48,13 +48,17 @@ public class RetrieveServiceImpl implements RetrieveService {
private static final int RESULT_SIZE = 10;
@Autowired private DataSetService dataSetService;
@Autowired
private DataSetService dataSetService;
@Autowired private SchemaService schemaService;
@Autowired
private SchemaService schemaService;
@Autowired private KnowledgeBaseService knowledgeBaseService;
@Autowired
private KnowledgeBaseService knowledgeBaseService;
@Autowired private SearchMatchStrategy searchMatchStrategy;
@Autowired
private SearchMatchStrategy searchMatchStrategy;
@Override
public List<SearchResult> retrieve(QueryNLReq queryNLReq) {
@@ -65,9 +69,8 @@ public class RetrieveServiceImpl implements RetrieveService {
schemaService.getSemanticSchema(queryNLReq.getDataSetIds());
List<SchemaElement> metricsDb = semanticSchemaDb.getMetrics();
final Map<Long, String> dataSetIdToName = semanticSchemaDb.getDataSetIdToName();
Map<Long, List<Long>> modelIdToDataSetIds =
dataSetService.getModelIdToDataSetIds(
new ArrayList<>(dataSetIdToName.keySet()), User.getFakeUser());
Map<Long, List<Long>> modelIdToDataSetIds = dataSetService.getModelIdToDataSetIds(
new ArrayList<>(dataSetIdToName.keySet()), User.getFakeUser());
// 2.detect by segment
List<S2Term> originals = knowledgeBaseService.getTerms(queryText, modelIdToDataSetIds);
log.debug("hanlp parse result: {}", originals);
@@ -86,14 +89,9 @@ public class RetrieveServiceImpl implements RetrieveService {
Optional<Map.Entry<MatchText, List<HanlpMapResult>>> mostSimilarSearchResult =
regTextMap.entrySet().stream()
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
.reduce(
(entry1, entry2) ->
entry1.getKey().getDetectSegment().length()
>= entry2.getKey()
.getDetectSegment()
.length()
? entry1
: entry2);
.reduce((entry1, entry2) -> entry1.getKey().getDetectSegment()
.length() >= entry2.getKey().getDetectSegment().length() ? entry1
: entry2);
// 4.optimize the results after the query
if (!mostSimilarSearchResult.isPresent()) {
@@ -108,12 +106,8 @@ public class RetrieveServiceImpl implements RetrieveService {
List<Long> possibleDataSets = getPossibleDataSets(queryNLReq, originals, dataSetIds);
// 5.1 priority dimension metric
boolean existMetricAndDimension =
searchMetricAndDimension(
new HashSet<>(possibleDataSets),
dataSetIdToName,
searchTextEntry,
searchResults);
boolean existMetricAndDimension = searchMetricAndDimension(new HashSet<>(possibleDataSets),
dataSetIdToName, searchTextEntry, searchResults);
// 5.2 process based on dimension values
MatchText matchText = searchTextEntry.getKey();
@@ -123,24 +117,17 @@ public class RetrieveServiceImpl implements RetrieveService {
for (Map.Entry<String, String> natureToNameEntry : natureToNameMap.entrySet()) {
Set<SearchResult> searchResultSet =
searchDimensionValue(
metricsDb,
dataSetIdToName,
dataSetInfoStat.getMetricDataSetCount(),
existMetricAndDimension,
matchText,
natureToNameMap,
natureToNameEntry,
queryNLReq.getQueryFilters());
Set<SearchResult> searchResultSet = searchDimensionValue(metricsDb, dataSetIdToName,
dataSetInfoStat.getMetricDataSetCount(), existMetricAndDimension, matchText,
natureToNameMap, natureToNameEntry, queryNLReq.getQueryFilters());
searchResults.addAll(searchResultSet);
}
return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList());
}
private List<Long> getPossibleDataSets(
QueryNLReq queryCtx, List<S2Term> originals, Set<Long> dataSetIds) {
private List<Long> getPossibleDataSets(QueryNLReq queryCtx, List<S2Term> originals,
Set<Long> dataSetIds) {
if (CollectionUtils.isNotEmpty(dataSetIds)) {
return new ArrayList<>(dataSetIds);
}
@@ -155,15 +142,10 @@ public class RetrieveServiceImpl implements RetrieveService {
return possibleDataSets;
}
private Set<SearchResult> searchDimensionValue(
List<SchemaElement> metricsDb,
Map<Long, String> modelToName,
long metricModelCount,
boolean existMetricAndDimension,
MatchText matchText,
Map<String, String> natureToNameMap,
Map.Entry<String, String> natureToNameEntry,
QueryFilters queryFilters) {
private Set<SearchResult> searchDimensionValue(List<SchemaElement> metricsDb,
Map<Long, String> modelToName, long metricModelCount, boolean existMetricAndDimension,
MatchText matchText, Map<String, String> natureToNameMap,
Map.Entry<String, String> natureToNameEntry, QueryFilters queryFilters) {
Set<SearchResult> searchResults = new LinkedHashSet();
String nature = natureToNameEntry.getKey();
@@ -175,15 +157,10 @@ public class RetrieveServiceImpl implements RetrieveService {
if (SchemaElementType.ENTITY.equals(schemaElementType)) {
return searchResults;
}
// If there are no metric/dimension, complete the metric information
SearchResult searchResult =
SearchResult.builder()
.modelId(modelId)
.modelName(modelToName.get(modelId))
.recommend(matchText.getRegText() + wordName)
.schemaElementType(schemaElementType)
.subRecommend(wordName)
.build();
// If there are no metric/dimension, complete the metric information
SearchResult searchResult = SearchResult.builder().modelId(modelId)
.modelName(modelToName.get(modelId)).recommend(matchText.getRegText() + wordName)
.schemaElementType(schemaElementType).subRecommend(wordName).build();
if (metricModelCount <= 0 && !existMetricAndDimension) {
if (filterByQueryFilter(wordName, queryFilters)) {
@@ -191,24 +168,15 @@ public class RetrieveServiceImpl implements RetrieveService {
}
searchResults.add(searchResult);
int metricSize = getMetricSize(natureToNameMap);
List<String> metrics =
filerMetricsByModel(metricsDb, modelId, metricSize * 3).stream()
.limit(metricSize)
.collect(Collectors.toList());
List<String> metrics = filerMetricsByModel(metricsDb, modelId, metricSize * 3).stream()
.limit(metricSize).collect(Collectors.toList());
for (String metric : metrics) {
SearchResult result =
SearchResult.builder()
.modelId(modelId)
.modelName(modelToName.get(modelId))
.recommend(
matchText.getRegText()
+ wordName
+ DictWordType.SPACE
+ metric)
.subRecommend(wordName + DictWordType.SPACE + metric)
.isComplete(false)
.build();
SearchResult result = SearchResult.builder().modelId(modelId)
.modelName(modelToName.get(modelId))
.recommend(matchText.getRegText() + wordName + DictWordType.SPACE + metric)
.subRecommend(wordName + DictWordType.SPACE + metric).isComplete(false)
.build();
searchResults.add(result);
}
} else {
@@ -238,22 +206,19 @@ public class RetrieveServiceImpl implements RetrieveService {
return true;
}
protected List<String> filerMetricsByModel(
List<SchemaElement> metricsDb, Long model, int metricSize) {
protected List<String> filerMetricsByModel(List<SchemaElement> metricsDb, Long model,
int metricSize) {
if (CollectionUtils.isEmpty(metricsDb)) {
return Lists.newArrayList();
}
return metricsDb.stream()
.filter(mapDO -> Objects.nonNull(mapDO) && model.equals(mapDO.getDataSetId()))
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.flatMap(
entry -> {
List<String> result = new ArrayList<>();
result.add(entry.getName());
return result.stream();
})
.limit(metricSize)
.collect(Collectors.toList());
.flatMap(entry -> {
List<String> result = new ArrayList<>();
result.add(entry.getName());
return result.stream();
}).limit(metricSize).collect(Collectors.toList());
}
/**
@@ -267,35 +232,23 @@ public class RetrieveServiceImpl implements RetrieveService {
Set<Long> possibleModels) {
List<HanlpMapResult> recommendValues = recommendTextListEntry.getValue();
return recommendValues.stream()
.flatMap(
entry ->
entry.getNatures().stream()
.filter(
nature -> {
if (CollectionUtils.isEmpty(possibleModels)) {
return true;
}
Long model = NatureHelper.getDataSetId(nature);
return possibleModels.contains(model);
})
.map(
nature -> {
DictWord posDO = new DictWord();
posDO.setWord(entry.getName());
posDO.setNature(nature);
return posDO;
}))
.sorted(Comparator.comparingInt(a -> a.getWord().length()))
.collect(
Collectors.toMap(
DictWord::getNature,
DictWord::getWord,
(value1, value2) -> value1,
LinkedHashMap::new));
.flatMap(entry -> entry.getNatures().stream().filter(nature -> {
if (CollectionUtils.isEmpty(possibleModels)) {
return true;
}
Long model = NatureHelper.getDataSetId(nature);
return possibleModels.contains(model);
}).map(nature -> {
DictWord posDO = new DictWord();
posDO.setWord(entry.getName());
posDO.setNature(nature);
return posDO;
})).sorted(Comparator.comparingInt(a -> a.getWord().length()))
.collect(Collectors.toMap(DictWord::getNature, DictWord::getWord,
(value1, value2) -> value1, LinkedHashMap::new));
}
private boolean searchMetricAndDimension(
Set<Long> possibleDataSets,
private boolean searchMetricAndDimension(Set<Long> possibleDataSets,
Map<Long, String> modelToName,
Map.Entry<MatchText, List<HanlpMapResult>> searchTextEntry,
Set<SearchResult> searchResults) {
@@ -306,15 +259,12 @@ public class RetrieveServiceImpl implements RetrieveService {
for (HanlpMapResult hanlpMapResult : hanlpMapResults) {
List<ModelWithSemanticType> dimensionMetricClassIds =
hanlpMapResult.getNatures().stream()
.map(
nature ->
new ModelWithSemanticType(
NatureHelper.getDataSetId(nature),
NatureHelper.convertToElementType(nature)))
.filter(entry -> matchCondition(entry, possibleDataSets))
.collect(Collectors.toList());
List<ModelWithSemanticType> dimensionMetricClassIds = hanlpMapResult.getNatures()
.stream()
.map(nature -> new ModelWithSemanticType(NatureHelper.getDataSetId(nature),
NatureHelper.convertToElementType(nature)))
.filter(entry -> matchCondition(entry, possibleDataSets))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(dimensionMetricClassIds)) {
continue;
@@ -324,21 +274,15 @@ public class RetrieveServiceImpl implements RetrieveService {
Long modelId = modelWithSemanticType.getModel();
SchemaElementType schemaElementType = modelWithSemanticType.getSchemaElementType();
SearchResult searchResult =
SearchResult.builder()
.modelId(modelId)
.modelName(modelToName.get(modelId))
SearchResult.builder().modelId(modelId).modelName(modelToName.get(modelId))
.recommend(matchText.getRegText() + hanlpMapResult.getName())
.subRecommend(hanlpMapResult.getName())
.schemaElementType(schemaElementType)
.build();
// visibility to filter metrics
.schemaElementType(schemaElementType).build();
// visibility to filter metrics
searchResults.add(searchResult);
}
log.debug(
"parseResult:{},dimensionMetricClassIds:{},possibleDataSets:{}",
hanlpMapResult,
dimensionMetricClassIds,
possibleDataSets);
log.debug("parseResult:{},dimensionMetricClassIds:{},possibleDataSets:{}",
hanlpMapResult, dimensionMetricClassIds, possibleDataSets);
}
log.info("searchMetricAndDimension searchResults:{}", searchResults);
return existMetric;

View File

@@ -103,17 +103,10 @@ public class SchemaServiceImpl implements SchemaService {
@Value("${s2.schema.cache.enable:true}")
private boolean schemaCacheEnable;
public SchemaServiceImpl(
ModelService modelService,
DimensionService dimensionService,
MetricService metricService,
DomainService domainService,
DataSetService dataSetService,
ModelRelaService modelRelaService,
StatUtils statUtils,
TagMetaService tagService,
TermService termService,
DatabaseService databaseService) {
public SchemaServiceImpl(ModelService modelService, DimensionService dimensionService,
MetricService metricService, DomainService domainService, DataSetService dataSetService,
ModelRelaService modelRelaService, StatUtils statUtils, TagMetaService tagService,
TermService termService, DatabaseService databaseService) {
this.modelService = modelService;
this.dimensionService = dimensionService;
this.metricService = metricService;
@@ -142,8 +135,7 @@ public class SchemaServiceImpl implements SchemaService {
if (dataSetId == null) {
return null;
}
return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream()
.findFirst()
return fetchDataSetSchema(new DataSetFilterReq(dataSetId)).stream().findFirst()
.orElse(null);
}
@@ -159,10 +151,8 @@ public class SchemaServiceImpl implements SchemaService {
ids.add(dataSetId);
List<DataSetSchemaResp> dataSetSchemaResps = fetchDataSetSchema(ids);
if (!CollectionUtils.isEmpty(dataSetSchemaResps)) {
Optional<DataSetSchemaResp> dataSetSchemaResp =
dataSetSchemaResps.stream()
.filter(d -> d.getId().equals(dataSetId))
.findFirst();
Optional<DataSetSchemaResp> dataSetSchemaResp = dataSetSchemaResps.stream()
.filter(d -> d.getId().equals(dataSetId)).findFirst();
if (dataSetSchemaResp.isPresent()) {
DataSetSchemaResp dataSetSchema = dataSetSchemaResp.get();
return DataSetSchemaBuilder.build(dataSetSchema);
@@ -199,11 +189,8 @@ public class SchemaServiceImpl implements SchemaService {
Map<Long, DataSetResp> dataSetRespMap = getDataSetMap(dataSetResps);
Set<Long> domainIds =
dataSetResps.stream().map(DataSetResp::getDomainId).collect(Collectors.toSet());
List<Long> modelIds =
dataSetRespMap.values().stream()
.map(DataSetResp::getAllModels)
.flatMap(Collection::stream)
.collect(Collectors.toList());
List<Long> modelIds = dataSetRespMap.values().stream().map(DataSetResp::getAllModels)
.flatMap(Collection::stream).collect(Collectors.toList());
Map<Long, List<TermResp>> termMaps = termService.getTermSets(domainIds);
metaFilter.setModelIds(modelIds);
@@ -228,28 +215,22 @@ public class SchemaServiceImpl implements SchemaService {
}
List<MetricSchemaResp> metricSchemaResps =
MetricConverter.filterByDataSet(metricResps, dataSetResp).stream()
.map(this::convert)
.collect(Collectors.toList());
.map(this::convert).collect(Collectors.toList());
List<DimSchemaResp> dimSchemaResps =
DimensionConverter.filterByDataSet(dimensionResps, dataSetResp).stream()
.map(this::convert)
.collect(Collectors.toList());
.map(this::convert).collect(Collectors.toList());
DataSetSchemaResp dataSetSchemaResp = new DataSetSchemaResp();
BeanUtils.copyProperties(dataSetResp, dataSetSchemaResp);
dataSetSchemaResp.setDimensions(dimSchemaResps);
dataSetSchemaResp.setMetrics(metricSchemaResps);
dataSetSchemaResp.setModelResps(
modelResps.stream()
.filter(
modelResp ->
dataSetResp.getAllModels().contains(modelResp.getId()))
.collect(Collectors.toList()));
dataSetSchemaResp.setModelResps(modelResps.stream()
.filter(modelResp -> dataSetResp.getAllModels().contains(modelResp.getId()))
.collect(Collectors.toList()));
dataSetSchemaResp.setTermResps(
termMaps.getOrDefault(dataSetResp.getDomainId(), Lists.newArrayList()));
if (!CollectionUtils.isEmpty(dataSetSchemaResp.getModelResps())) {
DatabaseResp databaseResp =
databaseService.getDatabase(
dataSetSchemaResp.getModelResps().get(0).getDatabaseId());
DatabaseResp databaseResp = databaseService
.getDatabase(dataSetSchemaResp.getModelResps().get(0).getDatabaseId());
dataSetSchemaResp.setDatabaseType(databaseResp.getType());
}
dataSetSchemaResps.add(dataSetSchemaResp);
@@ -265,9 +246,8 @@ public class SchemaServiceImpl implements SchemaService {
}
MetaFilter metaFilter = new MetaFilter(modelIds);
metaFilter.setStatus(StatusEnum.ONLINE.getCode());
Map<Long, List<MetricResp>> metricRespMap =
metricService.getMetrics(metaFilter).stream()
.collect(Collectors.groupingBy(MetricResp::getModelId));
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetrics(metaFilter).stream()
.collect(Collectors.groupingBy(MetricResp::getModelId));
Map<Long, List<DimensionResp>> dimensionRespsMap =
dimensionService.getDimensions(metaFilter).stream()
.collect(Collectors.groupingBy(DimensionResp::getModelId));
@@ -285,19 +265,15 @@ public class SchemaServiceImpl implements SchemaService {
metricResps.stream().map(this::convert).collect(Collectors.toList());
List<DimSchemaResp> dimensionResps =
dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList()).stream()
.map(this::convert)
.collect(Collectors.toList());
.map(this::convert).collect(Collectors.toList());
ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
BeanUtils.copyProperties(modelResp, modelSchemaResp);
modelSchemaResp.setDimensions(dimensionResps);
modelSchemaResp.setMetrics(metricSchemaResps);
modelSchemaResp.setModelRelas(
modelRelas.stream()
.filter(
modelRela ->
modelRela.getFromModelId().equals(modelId)
|| modelRela.getToModelId().equals(modelId))
.collect(Collectors.toList()));
modelSchemaResp.setModelRelas(modelRelas.stream()
.filter(modelRela -> modelRela.getFromModelId().equals(modelId)
|| modelRela.getToModelId().equals(modelId))
.collect(Collectors.toList()));
modelSchemaResps.add(modelSchemaResp);
}
return modelSchemaResps;
@@ -305,17 +281,11 @@ public class SchemaServiceImpl implements SchemaService {
private void fillCnt(List<DataSetSchemaResp> dataSetSchemaResps, List<ItemUseResp> statInfos) {
Map<String, ItemUseResp> typeIdAndStatPair =
statInfos.stream()
.collect(
Collectors.toMap(
itemUseInfo ->
itemUseInfo.getType()
+ AT_SYMBOL
+ AT_SYMBOL
+ itemUseInfo.getBizName(),
itemUseInfo -> itemUseInfo,
(item1, item2) -> item1));
Map<String, ItemUseResp> typeIdAndStatPair = statInfos.stream()
.collect(Collectors.toMap(
itemUseInfo -> itemUseInfo.getType() + AT_SYMBOL + AT_SYMBOL
+ itemUseInfo.getBizName(),
itemUseInfo -> itemUseInfo, (item1, item2) -> item1));
log.debug("typeIdAndStatPair:{}", typeIdAndStatPair);
for (DataSetSchemaResp dataSetSchemaResp : dataSetSchemaResps) {
fillDimCnt(dataSetSchemaResp, typeIdAndStatPair);
@@ -323,49 +293,39 @@ public class SchemaServiceImpl implements SchemaService {
}
}
private void fillMetricCnt(
DataSetSchemaResp dataSetSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
private void fillMetricCnt(DataSetSchemaResp dataSetSchemaResp,
Map<String, ItemUseResp> typeIdAndStatPair) {
List<MetricSchemaResp> metrics = dataSetSchemaResp.getMetrics();
if (CollectionUtils.isEmpty(dataSetSchemaResp.getMetrics())) {
return;
}
if (!CollectionUtils.isEmpty(metrics)) {
metrics.stream()
.forEach(
metric -> {
String key =
TypeEnums.METRIC.name().toLowerCase()
+ AT_SYMBOL
+ AT_SYMBOL
+ metric.getBizName();
if (typeIdAndStatPair.containsKey(key)) {
metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
}
});
metrics.stream().forEach(metric -> {
String key = TypeEnums.METRIC.name().toLowerCase() + AT_SYMBOL + AT_SYMBOL
+ metric.getBizName();
if (typeIdAndStatPair.containsKey(key)) {
metric.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
}
});
}
dataSetSchemaResp.setMetrics(metrics);
}
private void fillDimCnt(
DataSetSchemaResp dataSetSchemaResp, Map<String, ItemUseResp> typeIdAndStatPair) {
private void fillDimCnt(DataSetSchemaResp dataSetSchemaResp,
Map<String, ItemUseResp> typeIdAndStatPair) {
List<DimSchemaResp> dimensions = dataSetSchemaResp.getDimensions();
if (CollectionUtils.isEmpty(dataSetSchemaResp.getDimensions())) {
return;
}
if (!CollectionUtils.isEmpty(dimensions)) {
dimensions.stream()
.forEach(
dim -> {
String key =
TypeEnums.DIMENSION.name().toLowerCase()
+ AT_SYMBOL
+ AT_SYMBOL
+ dim.getBizName();
if (typeIdAndStatPair.containsKey(key)) {
dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
}
});
dimensions.stream().forEach(dim -> {
String key = TypeEnums.DIMENSION.name().toLowerCase() + AT_SYMBOL + AT_SYMBOL
+ dim.getBizName();
if (typeIdAndStatPair.containsKey(key)) {
dim.setUseCnt(typeIdAndStatPair.get(key).getUseCnt());
}
});
}
dataSetSchemaResp.setDimensions(dimensions);
}
@@ -404,11 +364,9 @@ public class SchemaServiceImpl implements SchemaService {
public List<ModelResp> getModelList(List<Long> modelIds) {
List<ModelResp> modelRespList = new ArrayList<>();
if (!org.apache.commons.collections.CollectionUtils.isEmpty(modelIds)) {
modelIds.stream()
.forEach(
m -> {
modelRespList.add(modelService.getModel(m));
});
modelIds.stream().forEach(m -> {
modelRespList.add(modelService.getModel(m));
});
}
return modelRespList;
}
@@ -434,21 +392,14 @@ public class SchemaServiceImpl implements SchemaService {
} else if (!CollectionUtils.isEmpty(schemaFilterReq.getModelIds())) {
List<ModelSchemaResp> modelSchemaResps =
fetchModelSchemaResps(schemaFilterReq.getModelIds());
semanticSchemaResp.setMetrics(
modelSchemaResps.stream()
.map(ModelSchemaResp::getMetrics)
.flatMap(Collection::stream)
.collect(Collectors.toList()));
semanticSchemaResp.setDimensions(
modelSchemaResps.stream()
.map(ModelSchemaResp::getDimensions)
.flatMap(Collection::stream)
.collect(Collectors.toList()));
semanticSchemaResp.setModelRelas(
modelSchemaResps.stream()
.map(ModelSchemaResp::getModelRelas)
.flatMap(Collection::stream)
.collect(Collectors.toList()));
semanticSchemaResp.setMetrics(modelSchemaResps.stream().map(ModelSchemaResp::getMetrics)
.flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp
.setDimensions(modelSchemaResps.stream().map(ModelSchemaResp::getDimensions)
.flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp
.setModelRelas(modelSchemaResps.stream().map(ModelSchemaResp::getModelRelas)
.flatMap(Collection::stream).collect(Collectors.toList()));
semanticSchemaResp.setModelResps(
modelSchemaResps.stream().map(this::convert).collect(Collectors.toList()));
semanticSchemaResp.setSchemaType(SchemaType.MODEL);
@@ -485,13 +436,11 @@ public class SchemaServiceImpl implements SchemaService {
@Override
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
if (itemUseReq.getCacheEnable()) {
return itemUseCache.get(
JsonUtil.toString(itemUseReq),
() -> {
List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
itemUseCache.put(JsonUtil.toString(itemUseReq), data);
return data;
});
return itemUseCache.get(JsonUtil.toString(itemUseReq), () -> {
List<ItemUseResp> data = statUtils.getStatInfo(itemUseReq);
itemUseCache.put(JsonUtil.toString(itemUseReq), data);
return data;
});
}
return statUtils.getStatInfo(itemUseReq);
}
@@ -499,28 +448,17 @@ public class SchemaServiceImpl implements SchemaService {
@Override
public List<ItemResp> getDomainDataSetTree() {
List<DomainResp> domainResps = domainService.getDomainList();
List<ItemResp> itemResps =
domainResps.stream()
.map(
domain ->
new ItemResp(
domain.getId(),
domain.getParentId(),
domain.getName(),
TypeEnums.DOMAIN))
.collect(Collectors.toList());
List<ItemResp> itemResps = domainResps.stream().map(domain -> new ItemResp(domain.getId(),
domain.getParentId(), domain.getName(), TypeEnums.DOMAIN))
.collect(Collectors.toList());
Map<Long, ItemResp> itemRespMap =
itemResps.stream().collect(Collectors.toMap(ItemResp::getId, item -> item));
List<DataSetResp> dataSetResps = dataSetService.getDataSetList(new MetaFilter());
for (DataSetResp dataSetResp : dataSetResps) {
ItemResp itemResp = itemRespMap.get(dataSetResp.getDomainId());
if (itemResp != null) {
ItemResp dataSet =
new ItemResp(
dataSetResp.getId(),
dataSetResp.getDomainId(),
dataSetResp.getName(),
TypeEnums.DATASET);
ItemResp dataSet = new ItemResp(dataSetResp.getId(), dataSetResp.getDomainId(),
dataSetResp.getName(), TypeEnums.DATASET);
itemResp.getChildren().add(dataSet);
}
}
@@ -528,10 +466,8 @@ public class SchemaServiceImpl implements SchemaService {
}
private void fillStaticInfo(List<DataSetSchemaResp> dataSetSchemaResps) {
List<Long> dataSetIds =
dataSetSchemaResps.stream()
.map(DataSetSchemaResp::getId)
.collect(Collectors.toList());
List<Long> dataSetIds = dataSetSchemaResps.stream().map(DataSetSchemaResp::getId)
.collect(Collectors.toList());
ItemUseReq itemUseReq = new ItemUseReq();
itemUseReq.setModelIds(dataSetIds);
@@ -567,11 +503,9 @@ public class SchemaServiceImpl implements SchemaService {
}
@Override
public void getSchemaYamlTpl(
SemanticSchemaResp semanticSchemaResp,
public void getSchemaYamlTpl(SemanticSchemaResp semanticSchemaResp,
Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DataModelYamlTpl> dataModelYamlTplList,
List<MetricYamlTpl> metricYamlTplList,
List<DataModelYamlTpl> dataModelYamlTplList, List<MetricYamlTpl> metricYamlTplList,
Map<Long, String> modelIdName) {
List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
@@ -587,15 +521,10 @@ public class SchemaServiceImpl implements SchemaService {
if (!dimensionYamlMap.containsKey(modelResp.getBizName())) {
dimensionYamlMap.put(modelResp.getBizName(), new ArrayList<>());
}
List<DimensionResp> dimensionRespList =
dimensionResps.stream()
.filter(
d ->
d.getModelBizName()
.equalsIgnoreCase(modelResp.getBizName()))
.collect(Collectors.toList());
dimensionYamlMap
.get(modelResp.getBizName())
List<DimensionResp> dimensionRespList = dimensionResps.stream()
.filter(d -> d.getModelBizName().equalsIgnoreCase(modelResp.getBizName()))
.collect(Collectors.toList());
dimensionYamlMap.get(modelResp.getBizName())
.addAll(DimensionYamlManager.convert2DimensionYaml(dimensionRespList));
}
List<MetricResp> metricResps = new ArrayList<>(semanticSchemaResp.getMetrics());

View File

@@ -58,13 +58,9 @@ public class TagMetaServiceImpl implements TagMetaService {
private final TagObjectService tagObjectService;
private final DomainService domainService;
public TagMetaServiceImpl(
TagRepository tagRepository,
ModelService modelService,
CollectService collectService,
@Lazy DimensionService dimensionService,
@Lazy MetricService metricService,
TagObjectService tagObjectService,
public TagMetaServiceImpl(TagRepository tagRepository, ModelService modelService,
CollectService collectService, @Lazy DimensionService dimensionService,
@Lazy MetricService metricService, TagObjectService tagObjectService,
DomainService domainService) {
this.tagRepository = tagRepository;
this.modelService = modelService;
@@ -162,11 +158,8 @@ public class TagMetaServiceImpl implements TagMetaService {
if (Objects.nonNull(tagMarketPageReq.getTagObjectId())) {
modelRespList =
modelRespList.stream()
.filter(
modelResp ->
tagMarketPageReq
.getTagObjectId()
.equals(modelResp.getTagObjectId()))
.filter(modelResp -> tagMarketPageReq.getTagObjectId()
.equals(modelResp.getTagObjectId()))
.collect(Collectors.toList());
}
if (CollectionUtils.isEmpty(modelRespList)) {
@@ -179,15 +172,9 @@ public class TagMetaServiceImpl implements TagMetaService {
BeanUtils.copyProperties(tagMarketPageReq, tagFilter);
List<CollectDO> collectList = collectService.getCollectionList(user.getName());
if (tagMarketPageReq.isHasCollect()) {
List<Long> collectIds =
collectList.stream()
.filter(
collectDO ->
SchemaElementType.TAG
.name()
.equalsIgnoreCase(collectDO.getType()))
.map(CollectDO::getCollectId)
.collect(Collectors.toList());
List<Long> collectIds = collectList.stream().filter(
collectDO -> SchemaElementType.TAG.name().equalsIgnoreCase(collectDO.getType()))
.map(CollectDO::getCollectId).collect(Collectors.toList());
if (CollectionUtils.isEmpty(collectIds)) {
tagFilter.setIds(Lists.newArrayList(-1L));
} else {
@@ -217,22 +204,14 @@ public class TagMetaServiceImpl implements TagMetaService {
if (CollectionUtils.isEmpty(tagObjects)) {
return;
}
Map<Long, TagObjectResp> tagObjectMap =
tagObjects.stream()
.collect(
Collectors.toMap(
TagObjectResp::getId,
tagObject -> tagObject,
(v1, v2) -> v2));
Map<Long, TagObjectResp> tagObjectMap = tagObjects.stream().collect(
Collectors.toMap(TagObjectResp::getId, tagObject -> tagObject, (v1, v2) -> v2));
if (CollectionUtils.isNotEmpty(tagRespList)) {
tagRespList.stream()
.forEach(
tagResp -> {
if (tagObjectMap.containsKey(tagResp.getTagObjectId())) {
tagResp.setTagObjectName(
tagObjectMap.get(tagResp.getTagObjectId()).getName());
}
});
tagRespList.stream().forEach(tagResp -> {
if (tagObjectMap.containsKey(tagResp.getTagObjectId())) {
tagResp.setTagObjectName(tagObjectMap.get(tagResp.getTagObjectId()).getName());
}
});
}
}
@@ -246,20 +225,14 @@ public class TagMetaServiceImpl implements TagMetaService {
}
private void fillDomainInfo(List<TagResp> tagRespList) {
Map<Long, DomainResp> domainMap =
domainService.getDomainList().stream()
.collect(
Collectors.toMap(
DomainResp::getId, domain -> domain, (v1, v2) -> v2));
Map<Long, DomainResp> domainMap = domainService.getDomainList().stream()
.collect(Collectors.toMap(DomainResp::getId, domain -> domain, (v1, v2) -> v2));
if (CollectionUtils.isNotEmpty(tagRespList) && Objects.nonNull(domainMap)) {
tagRespList.stream()
.forEach(
tagResp -> {
if (domainMap.containsKey(tagResp.getDomainId())) {
tagResp.setDomainName(
domainMap.get(tagResp.getDomainId()).getName());
}
});
tagRespList.stream().forEach(tagResp -> {
if (domainMap.containsKey(tagResp.getDomainId())) {
tagResp.setDomainName(domainMap.get(tagResp.getDomainId()).getName());
}
});
}
}
@@ -321,31 +294,21 @@ public class TagMetaServiceImpl implements TagMetaService {
tagRespList.stream().map(TagResp::getModelId).collect(Collectors.toList());
ModelFilter modelFilter = new ModelFilter(false, modelIds);
Map<Long, ModelResp> modelIdAndRespMap = modelService.getModelMap(modelFilter);
tagRespList.stream()
.forEach(
tagResp -> {
if (Objects.nonNull(modelIdAndRespMap)
&& modelIdAndRespMap.containsKey(tagResp.getModelId())) {
tagResp.setModelName(
modelIdAndRespMap.get(tagResp.getModelId()).getName());
tagResp.setDomainId(
modelIdAndRespMap.get(tagResp.getModelId()).getDomainId());
tagResp.setTagObjectId(
modelIdAndRespMap
.get(tagResp.getModelId())
.getTagObjectId());
}
});
tagRespList.stream().forEach(tagResp -> {
if (Objects.nonNull(modelIdAndRespMap)
&& modelIdAndRespMap.containsKey(tagResp.getModelId())) {
tagResp.setModelName(modelIdAndRespMap.get(tagResp.getModelId()).getName());
tagResp.setDomainId(modelIdAndRespMap.get(tagResp.getModelId()).getDomainId());
tagResp.setTagObjectId(
modelIdAndRespMap.get(tagResp.getModelId()).getTagObjectId());
}
});
}
private TagResp fillCollectAndAdminInfo(TagResp tagResp, User user) {
List<Long> collectIds =
collectService.getCollectionList(user.getName()).stream()
.filter(
collectDO ->
TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
.map(CollectDO::getCollectId)
.collect(Collectors.toList());
List<Long> collectIds = collectService.getCollectionList(user.getName()).stream()
.filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
.map(CollectDO::getCollectId).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagResp.getId())) {
tagResp.setIsCollect(true);
} else {
@@ -357,24 +320,17 @@ public class TagMetaServiceImpl implements TagMetaService {
}
private TagResp fillCollectAndAdminInfo(List<TagResp> tagRespList, User user) {
List<Long> collectIds =
collectService.getCollectionList(user.getName()).stream()
.filter(
collectDO ->
TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
.map(CollectDO::getCollectId)
.collect(Collectors.toList());
List<Long> collectIds = collectService.getCollectionList(user.getName()).stream()
.filter(collectDO -> TypeEnums.TAG.name().equalsIgnoreCase(collectDO.getType()))
.map(CollectDO::getCollectId).collect(Collectors.toList());
tagRespList.stream()
.forEach(
tagResp -> {
if (CollectionUtils.isNotEmpty(collectIds)
&& collectIds.contains(tagResp.getId())) {
tagResp.setIsCollect(true);
} else {
tagResp.setIsCollect(false);
}
});
tagRespList.stream().forEach(tagResp -> {
if (CollectionUtils.isNotEmpty(collectIds) && collectIds.contains(tagResp.getId())) {
tagResp.setIsCollect(true);
} else {
tagResp.setIsCollect(false);
}
});
fillAdminRes(tagRespList, user);
return tagRespList.get(0);
@@ -418,21 +374,17 @@ public class TagMetaServiceImpl implements TagMetaService {
ModelResp model = modelService.getModel(dimension.getModelId());
if (Objects.isNull(model.getTagObjectId())) {
throw new RuntimeException(
String.format(
"this dimension:%s is not supported to create tag,"
+ " no related tag object",
tagReq.getItemId()));
String.format("this dimension:%s is not supported to create tag,"
+ " no related tag object", tagReq.getItemId()));
}
}
if (TagDefineType.METRIC.equals(tagReq.getTagDefineType())) {
MetricResp metric = metricService.getMetric(tagReq.getItemId());
ModelResp model = modelService.getModel(metric.getModelId());
if (Objects.isNull(model.getTagObjectId())) {
throw new RuntimeException(
String.format(
"this metric:%s is not supported to create tag,"
+ " no related tag object",
tagReq.getItemId()));
throw new RuntimeException(String.format(
"this metric:%s is not supported to create tag," + " no related tag object",
tagReq.getItemId()));
}
}
}
@@ -451,15 +403,11 @@ public class TagMetaServiceImpl implements TagMetaService {
tagFilter.setItemIds(itemIds);
Set<Long> dimensionItemSet =
getTagDOList(tagFilter).stream().map(TagDO::getItemId).collect(Collectors.toSet());
return itemIds.stream()
.map(
entry -> {
TagItem tagItem = new TagItem();
tagItem.setIsTag(
Boolean.compare(dimensionItemSet.contains(entry), false));
tagItem.setItemId(entry);
return tagItem;
})
.collect(Collectors.toList());
return itemIds.stream().map(entry -> {
TagItem tagItem = new TagItem();
tagItem.setIsTag(Boolean.compare(dimensionItemSet.contains(entry), false));
tagItem.setItemId(entry);
return tagItem;
}).collect(Collectors.toList());
}
}

View File

@@ -34,9 +34,7 @@ public class TagObjectServiceImpl implements TagObjectService {
private final ModelService modelService;
private final TagMetaService tagMetaService;
public TagObjectServiceImpl(
TagObjectRepository tagObjectRepository,
ModelService modelService,
public TagObjectServiceImpl(TagObjectRepository tagObjectRepository, ModelService modelService,
@Lazy TagMetaService tagMetaService) {
this.tagObjectRepository = tagObjectRepository;
this.modelService = modelService;
@@ -66,14 +64,9 @@ public class TagObjectServiceImpl implements TagObjectService {
if (CollectionUtils.isEmpty(tagObjectRespList)) {
return;
}
tagObjectRespList =
tagObjectRespList.stream()
.filter(
tagObjectResp ->
StatusEnum.ONLINE
.getCode()
.equals(tagObjectResp.getStatus()))
.collect(Collectors.toList());
tagObjectRespList = tagObjectRespList.stream().filter(
tagObjectResp -> StatusEnum.ONLINE.getCode().equals(tagObjectResp.getStatus()))
.collect(Collectors.toList());
for (TagObjectResp tagObject : tagObjectRespList) {
if (tagObject.getBizName().equalsIgnoreCase(tagObjectReq.getBizName())) {
throw new Exception(
@@ -128,9 +121,8 @@ public class TagObjectServiceImpl implements TagObjectService {
if (!CollectionUtils.isEmpty(allModelByDomainIds)) {
List<Long> modelIds =
allModelByDomainIds.stream().map(ModelResp::getId).collect(Collectors.toList());
throw new Exception(
"delete operation is not supported at the moment. related modelIds:"
+ modelIds);
throw new Exception("delete operation is not supported at the moment. related modelIds:"
+ modelIds);
}
TagFilterPageReq tagMarketPageReq = new TagFilterPageReq();
tagMarketPageReq.setTagObjectId(tagObjectDO.getId());
@@ -173,9 +165,8 @@ public class TagObjectServiceImpl implements TagObjectService {
List<TagObjectDO> tagObjectDOList = tagObjectRepository.query(filter);
List<TagObjectResp> tagObjectRespList =
TagObjectConverter.convert2RespList(tagObjectDOList);
Map<Long, TagObjectResp> map =
tagObjectRespList.stream()
.collect(Collectors.toMap(TagObjectResp::getId, a -> a, (k1, k2) -> k1));
Map<Long, TagObjectResp> map = tagObjectRespList.stream()
.collect(Collectors.toMap(TagObjectResp::getId, a -> a, (k1, k2) -> k1));
return map;
}
}

View File

@@ -52,11 +52,8 @@ public class TagQueryServiceImpl implements TagQueryService {
private final ModelService modelService;
private final SqlGenerateUtils sqlGenerateUtils;
public TagQueryServiceImpl(
TagMetaService tagMetaService,
SemanticLayerService queryService,
ModelService modelService,
SqlGenerateUtils sqlGenerateUtils) {
public TagQueryServiceImpl(TagMetaService tagMetaService, SemanticLayerService queryService,
ModelService modelService, SqlGenerateUtils sqlGenerateUtils) {
this.tagMetaService = tagMetaService;
this.queryService = queryService;
this.modelService = modelService;
@@ -88,9 +85,8 @@ public class TagQueryServiceImpl implements TagQueryService {
private void checkTag(TagResp tag) throws Exception {
if (Objects.nonNull(tag)
&& TagDefineType.METRIC.name().equalsIgnoreCase(tag.getTagDefineType())) {
throw new Exception(
"do not support value distribution query for tag (from metric): "
+ tag.getBizName());
throw new Exception("do not support value distribution query for tag (from metric): "
+ tag.getBizName());
}
}
@@ -121,16 +117,12 @@ public class TagQueryServiceImpl implements TagQueryService {
return LocalDate.now().plusDays(-dayBefore).format(formatter);
}
private String queryTagDateFromDbBySql(
Dim dim, TagResp tag, ItemValueReq itemValueReq, User user) {
private String queryTagDateFromDbBySql(Dim dim, TagResp tag, ItemValueReq itemValueReq,
User user) {
String sqlPattern = "select max(%s) as %s from tbl where %s is not null";
String sql =
String.format(
sqlPattern,
TimeDimensionEnum.DAY.getName(),
maxDateAlias,
tag.getBizName());
String sql = String.format(sqlPattern, TimeDimensionEnum.DAY.getName(), maxDateAlias,
tag.getBizName());
// 添加时间过滤信息
log.info("[queryTagDateFromDbBySql] calculate the maximum time start");
@@ -143,22 +135,13 @@ public class TagQueryServiceImpl implements TagQueryService {
if (StringUtils.isEmpty(dateFormat)) {
dateFormat = itemValueDateFormat;
}
String start =
LocalDate.now()
.minusDays(itemValueReq.getDateConf().getUnit())
.format(DateTimeFormatter.ofPattern(dateFormat));
String end =
LocalDate.now()
.minusDays(0)
.format(DateTimeFormatter.ofPattern(dateFormat));
sql =
sql
+ String.format(
" and ( %s > '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(),
start,
TimeDimensionEnum.DAY.getName(),
end);
String start = LocalDate.now().minusDays(itemValueReq.getDateConf().getUnit())
.format(DateTimeFormatter.ofPattern(dateFormat));
String end = LocalDate.now().minusDays(0)
.format(DateTimeFormatter.ofPattern(dateFormat));
sql = sql + String.format(" and ( %s > '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(),
end);
}
}
}
@@ -219,42 +202,27 @@ public class TagQueryServiceImpl implements TagQueryService {
return " and " + dateWhereClause;
}
private void fillTagValueInfo(
ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp, Long totalCount) {
private void fillTagValueInfo(ItemValueResp itemValueResp, SemanticQueryResp semanticQueryResp,
Long totalCount) {
List<ValueDistribution> valueDistributionList = new ArrayList<>();
List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
if (!CollectionUtils.isEmpty(resultList)) {
resultList.stream()
.forEach(
line -> {
Object tagValue = line.get(itemValueResp.getBizName());
Long tagValueCount =
Long.parseLong(line.get(tagValueAlias).toString());
valueDistributionList.add(
ValueDistribution.builder()
.totalCount(totalCount)
.valueMap(tagValue)
.valueCount(tagValueCount)
.ratio(1.0 * tagValueCount / totalCount)
.build());
});
resultList.stream().forEach(line -> {
Object tagValue = line.get(itemValueResp.getBizName());
Long tagValueCount = Long.parseLong(line.get(tagValueAlias).toString());
valueDistributionList.add(ValueDistribution.builder().totalCount(totalCount)
.valueMap(tagValue).valueCount(tagValueCount)
.ratio(1.0 * tagValueCount / totalCount).build());
});
}
itemValueResp.setValueDistributionList(valueDistributionList);
}
private QuerySqlReq generateReq(TagResp tag, ItemValueReq itemValueReq) {
String sqlPattern =
"select %s, count(1) as %s from tbl where %s is not null %s "
+ "group by %s order by %s desc";
String sql =
String.format(
sqlPattern,
tag.getBizName(),
tagValueAlias,
tag.getBizName(),
getDateFilter(itemValueReq),
tag.getBizName(),
tag.getBizName());
String sqlPattern = "select %s, count(1) as %s from tbl where %s is not null %s "
+ "group by %s order by %s desc";
String sql = String.format(sqlPattern, tag.getBizName(), tagValueAlias, tag.getBizName(),
getDateFilter(itemValueReq), tag.getBizName(), tag.getBizName());
Set<Long> modelIds = new HashSet<>();
modelIds.add(tag.getModelId());

View File

@@ -56,15 +56,8 @@ public class TermServiceImpl extends ServiceImpl<TermMapper, TermDO> implements
QueryWrapper<TermDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(TermDO::getDomainId, domainId);
if (StringUtils.isNotBlank(queryKey)) {
queryWrapper
.lambda()
.and(
i ->
i.like(TermDO::getName, queryKey)
.or()
.like(TermDO::getDescription, queryKey)
.or()
.like(TermDO::getAlias, queryKey));
queryWrapper.lambda().and(i -> i.like(TermDO::getName, queryKey).or()
.like(TermDO::getDescription, queryKey).or().like(TermDO::getAlias, queryKey));
}
List<TermDO> termDOS = list(queryWrapper);
return termDOS.stream().map(this::convert).collect(Collectors.toList());
@@ -78,8 +71,7 @@ public class TermServiceImpl extends ServiceImpl<TermMapper, TermDO> implements
QueryWrapper<TermDO> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().in(TermDO::getDomainId, domainIds);
List<TermDO> list = list(queryWrapper);
return list.stream()
.map(this::convert)
return list.stream().map(this::convert)
.collect(Collectors.groupingBy(TermResp::getDomainId));
}

View File

@@ -13,7 +13,8 @@ import org.springframework.stereotype.Component;
@Order(2)
public class DictionaryReloadTask implements CommandLineRunner {
@Autowired private DictWordService dictWordService;
@Autowired
private DictWordService dictWordService;
@Override
public void run(String... args) {

View File

@@ -25,13 +25,17 @@ import java.util.List;
@Order(2)
public class MetaEmbeddingTask implements CommandLineRunner {
@Autowired private EmbeddingService embeddingService;
@Autowired
private EmbeddingService embeddingService;
@Autowired private EmbeddingConfig embeddingConfig;
@Autowired
private EmbeddingConfig embeddingConfig;
@Autowired private MetricService metricService;
@Autowired
private MetricService metricService;
@Autowired private DimensionService dimensionService;
@Autowired
private DimensionService dimensionService;
@PreDestroy
public void onShutdown() {
@@ -62,13 +66,11 @@ public class MetaEmbeddingTask implements CommandLineRunner {
try {
List<DataItem> metricDataItems = metricService.getDataEvent().getDataItems();
embeddingService.addQuery(
embeddingConfig.getMetaCollectionName(),
embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
TextSegmentConvert.convertToEmbedding(metricDataItems));
List<DataItem> dimensionDataItems = dimensionService.getDataEvent().getDataItems();
embeddingService.addQuery(
embeddingConfig.getMetaCollectionName(),
embeddingService.addQuery(embeddingConfig.getMetaCollectionName(),
TextSegmentConvert.convertToEmbedding(dimensionDataItems));
} catch (Exception e) {
log.error("Failed to reload meta embedding.", e);

View File

@@ -23,38 +23,32 @@ public class AliasGenerateHelper {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
private static final String NAME_ALIAS_INSTRUCTION =
""
+ "\n#Role: You are a professional data analyst specializing in metrics and dimensions."
+ "\n#Task: You will be provided with metadata about a metric or dimension, please help "
+ "generate a few aliases in the same language as its `fieldName`."
+ "\n#Rules:"
+ "1. Please do not generate aliases like xxx1, xxx2, xxx3."
+ "2. Please do not generate aliases that are the same as the original names of metrics/dimensions."
+ "3. Please pay attention to the quality of the generated aliases and "
+ "avoid creating aliases that look like test data."
+ "4. Please output as a json string array."
+ "\n#Metadata: {'table':'{{table}}', 'name':'{{name}}', 'type':'{{type}}', "
+ "'field':'field', 'description':'{{desc}}'}"
+ "\n#Output:";
private static final String NAME_ALIAS_INSTRUCTION = ""
+ "\n#Role: You are a professional data analyst specializing in metrics and dimensions."
+ "\n#Task: You will be provided with metadata about a metric or dimension, please help "
+ "generate a few aliases in the same language as its `fieldName`." + "\n#Rules:"
+ "1. Please do not generate aliases like xxx1, xxx2, xxx3."
+ "2. Please do not generate aliases that are the same as the original names of metrics/dimensions."
+ "3. Please pay attention to the quality of the generated aliases and "
+ "avoid creating aliases that look like test data."
+ "4. Please output as a json string array."
+ "\n#Metadata: {'table':'{{table}}', 'name':'{{name}}', 'type':'{{type}}', "
+ "'field':'field', 'description':'{{desc}}'}" + "\n#Output:";
private static final String VALUE_ALIAS_INSTRUCTION =
""
+ "\n#Role: You are a professional data analyst."
"" + "\n#Role: You are a professional data analyst."
+ "\n#Task: You will be provided with a json array of dimension values,"
+ "please help generate a few aliases for each value."
+ "\n#Rule:"
+ "please help generate a few aliases for each value." + "\n#Rule:"
+ "1. ALWAYS output json array for each value."
+ "2. The aliases should be in the same language as its original value."
+ "\n#Exemplar:"
+ "Values: [\\\"qq_music\\\",\\\"kugou_music\\\"], "
+ "\n#Exemplar:" + "Values: [\\\"qq_music\\\",\\\"kugou_music\\\"], "
+ "Output: {\\\"tran\\\":[\\\"qq音乐\\\",\\\"酷狗音乐\\\"],"
+ " \\\"alias\\\":{\\\"qq_music\\\":[\\\"q音\\\",\\\"qq音乐\\\"],"
+ " \\\"kugou_music\\\":[\\\"kugou\\\",\\\"酷狗\\\"]}}"
+ "\nValues: {{values}}, Output:";
public String generateAlias(
String mockType, String name, String bizName, String table, String desc) {
public String generateAlias(String mockType, String name, String bizName, String table,
String desc) {
Map<String, Object> variable = new HashMap<>();
variable.put("table", table);
variable.put("name", name);
@@ -88,8 +82,8 @@ public class AliasGenerateHelper {
return response.content().text();
}
private static String extractString(
String targetString, String left, String right, Boolean exclusionFlag) {
private static String extractString(String targetString, String left, String right,
Boolean exclusionFlag) {
if (targetString == null || left == null || right == null || exclusionFlag == null) {
return targetString;
}
@@ -139,19 +133,18 @@ public class AliasGenerateHelper {
}
}
BoundaryPattern[] patterns = {
// 不做任何匹配
new BoundaryPattern(null, null, null),
// ```{"name":"Alice","age":25,"city":"NewYork"}```
new BoundaryPattern("```", "```", true),
// ```json {"name":"Alice","age":25,"city":"NewYork"}```
new BoundaryPattern("```json", "```", true),
// ```JSON {"name":"Alice","age":25,"city":"NewYork"}```
new BoundaryPattern("```JSON", "```", true),
// {"name":"Alice","age":25,"city":"NewYork"}
new BoundaryPattern("{", "}", false),
// ["Alice", "Bob"]
new BoundaryPattern("[", "]", false)
};
// 不做任何匹配
new BoundaryPattern(null, null, null),
// ```{"name":"Alice","age":25,"city":"NewYork"}```
new BoundaryPattern("```", "```", true),
// ```json {"name":"Alice","age":25,"city":"NewYork"}```
new BoundaryPattern("```json", "```", true),
// ```JSON {"name":"Alice","age":25,"city":"NewYork"}```
new BoundaryPattern("```JSON", "```", true),
// {"name":"Alice","age":25,"city":"NewYork"}
new BoundaryPattern("{", "}", false),
// ["Alice", "Bob"]
new BoundaryPattern("[", "]", false)};
for (BoundaryPattern pattern : patterns) {
String extracted =
extractString(aiMessage, pattern.left, pattern.right, pattern.exclusionFlag);

View File

@@ -62,10 +62,8 @@ public class ChatWorkflowEngine {
parseResult.setErrorMsg("No semantic queries can be parsed out.");
queryCtx.setChatWorkflowState(ChatWorkflowState.FINISHED);
} else {
List<SemanticParseInfo> parseInfos =
queryCtx.getCandidateQueries().stream()
.map(SemanticQuery::getParseInfo)
.collect(Collectors.toList());
List<SemanticParseInfo> parseInfos = queryCtx.getCandidateQueries().stream()
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
parseResult.setSelectedParses(parseInfos);
queryCtx.setChatWorkflowState(ChatWorkflowState.CORRECTING);
}
@@ -101,14 +99,11 @@ public class ChatWorkflowEngine {
}
private void performParsing(ChatQueryContext queryCtx) {
semanticParsers.forEach(
parser -> {
parser.parse(queryCtx);
log.debug(
"{} result:{}",
parser.getClass().getSimpleName(),
JsonUtil.toString(queryCtx));
});
semanticParsers.forEach(parser -> {
parser.parse(queryCtx);
log.debug("{} result:{}", parser.getClass().getSimpleName(),
JsonUtil.toString(queryCtx));
});
}
private void performCorrecting(ChatQueryContext queryCtx) {
@@ -126,45 +121,38 @@ public class ChatWorkflowEngine {
}
private void performProcessing(ChatQueryContext queryCtx, ParseResp parseResult) {
resultProcessors.forEach(
processor -> {
processor.process(parseResult, queryCtx);
});
resultProcessors.forEach(processor -> {
processor.process(parseResult, queryCtx);
});
}
private void performTranslating(ChatQueryContext chatQueryContext) {
List<SemanticParseInfo> semanticParseInfos =
chatQueryContext.getCandidateQueries().stream()
.map(SemanticQuery::getParseInfo)
.collect(Collectors.toList());
List<SemanticParseInfo> semanticParseInfos = chatQueryContext.getCandidateQueries().stream()
.map(SemanticQuery::getParseInfo).collect(Collectors.toList());
semanticParseInfos.forEach(
parseInfo -> {
try {
SemanticQuery semanticQuery =
QueryManager.createQuery(parseInfo.getQueryMode());
if (Objects.isNull(semanticQuery)) {
return;
}
semanticQuery.setParseInfo(parseInfo);
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
SemanticLayerService queryService =
ContextUtils.getBean(SemanticLayerService.class);
SemanticTranslateResp explain =
queryService.translate(
semanticQueryReq, chatQueryContext.getUser());
parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL());
semanticParseInfos.forEach(parseInfo -> {
try {
SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
if (Objects.isNull(semanticQuery)) {
return;
}
semanticQuery.setParseInfo(parseInfo);
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
SemanticLayerService queryService =
ContextUtils.getBean(SemanticLayerService.class);
SemanticTranslateResp explain =
queryService.translate(semanticQueryReq, chatQueryContext.getUser());
parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL());
keyPipelineLog.info(
"SqlInfoProcessor results:\n"
+ "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getParsedS2SQL()),
StringUtils.normalizeSpace(
parseInfo.getSqlInfo().getCorrectedS2SQL()),
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getQuerySQL()));
} catch (Exception e) {
log.warn("get sql info failed:{}", parseInfo, e);
}
});
keyPipelineLog.info(
"SqlInfoProcessor results:\n"
+ "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getParsedS2SQL()),
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getCorrectedS2SQL()),
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getQuerySQL()));
} catch (Exception e) {
log.warn("get sql info failed:{}", parseInfo, e);
}
});
}
}

View File

@@ -29,9 +29,7 @@ public class ClassConverter {
private final DomainService domainService;
private final TagObjectService tagObjectService;
public ClassConverter(
ClassRepository classRepository,
DomainService domainService,
public ClassConverter(ClassRepository classRepository, DomainService domainService,
TagObjectService tagObjectService) {
this.classRepository = classRepository;
this.domainService = domainService;
@@ -57,16 +55,13 @@ public class ClassConverter {
return convert2RespInternal(classDO, idAndDomain, classFullPathMap);
}
private ClassResp convert2RespInternal(
ClassDO classDO,
Map<Long, DomainResp> idAndDomain,
private ClassResp convert2RespInternal(ClassDO classDO, Map<Long, DomainResp> idAndDomain,
Map<Long, String> classFullPathMap) {
ClassResp classResp = new ClassResp();
BeanUtils.copyProperties(classDO, classResp);
Long domainId = classResp.getDomainId();
if (Objects.nonNull(idAndDomain)
&& idAndDomain.containsKey(domainId)
if (Objects.nonNull(idAndDomain) && idAndDomain.containsKey(domainId)
&& Objects.nonNull(idAndDomain.get(domainId))) {
classResp.setDomainName(idAndDomain.get(domainId).getName());
}
@@ -94,9 +89,8 @@ public class ClassConverter {
public Map<Long, String> getClassFullPathMap() {
Map<Long, String> classFullPathMap = new HashMap<>();
List<ClassDO> classDOList = classRepository.getAllClassDOList();
Map<Long, ClassDO> classDOMap =
classDOList.stream()
.collect(Collectors.toMap(ClassDO::getId, a -> a, (k1, k2) -> k1));
Map<Long, ClassDO> classDOMap = classDOList.stream()
.collect(Collectors.toMap(ClassDO::getId, a -> a, (k1, k2) -> k1));
for (ClassDO classDO : classDOList) {
final Long domainId = classDO.getId();
StringBuilder fullPath = new StringBuilder(classDO.getBizName() + "/");

View File

@@ -27,8 +27,7 @@ public class ComponentFactory {
}
public static List<SchemaMapper> getSchemaMappers() {
return CollectionUtils.isEmpty(schemaMappers)
? init(SchemaMapper.class, schemaMappers)
return CollectionUtils.isEmpty(schemaMappers) ? init(SchemaMapper.class, schemaMappers)
: schemaMappers;
}
@@ -49,15 +48,13 @@ public class ComponentFactory {
}
private static <T> List<T> init(Class<T> factoryType, List list) {
list.addAll(
SpringFactoriesLoader.loadFactories(
factoryType, Thread.currentThread().getContextClassLoader()));
list.addAll(SpringFactoriesLoader.loadFactories(factoryType,
Thread.currentThread().getContextClassLoader()));
return list;
}
private static <T> T init(Class<T> factoryType) {
return SpringFactoriesLoader.loadFactories(
factoryType, Thread.currentThread().getContextClassLoader())
.get(0);
return SpringFactoriesLoader
.loadFactories(factoryType, Thread.currentThread().getContextClassLoader()).get(0);
}
}

View File

@@ -26,15 +26,9 @@ public class DataSetSchemaBuilder {
public static DataSetSchema build(DataSetSchemaResp resp) {
DataSetSchema dataSetSchema = new DataSetSchema();
dataSetSchema.setQueryConfig(resp.getQueryConfig());
SchemaElement dataSet =
SchemaElement.builder()
.dataSetId(resp.getId())
.dataSetName(resp.getName())
.id(resp.getId())
.name(resp.getName())
.bizName(resp.getBizName())
.type(SchemaElementType.DATASET)
.build();
SchemaElement dataSet = SchemaElement.builder().dataSetId(resp.getId())
.dataSetName(resp.getName()).id(resp.getId()).name(resp.getName())
.bizName(resp.getBizName()).type(SchemaElementType.DATASET).build();
dataSetSchema.setDataSet(dataSet);
dataSetSchema.setDatabaseType(resp.getDatabaseType());
@@ -68,21 +62,12 @@ public class DataSetSchemaBuilder {
for (MetricSchemaResp metric : resp.getMetrics()) {
List<String> alias = SchemaItem.getAliasList(metric.getAlias());
if (metric.getIsTag() == 1) {
SchemaElement tagToAdd =
SchemaElement.builder()
.dataSetId(resp.getId())
.dataSetName(resp.getName())
.model(metric.getModelId())
.id(metric.getId())
.name(metric.getName())
.bizName(metric.getBizName())
.type(SchemaElementType.TAG)
.useCnt(metric.getUseCnt())
.alias(alias)
.defaultAgg(metric.getDefaultAgg())
.isTag(metric.getIsTag())
.description(metric.getDescription())
.build();
SchemaElement tagToAdd = SchemaElement.builder().dataSetId(resp.getId())
.dataSetName(resp.getName()).model(metric.getModelId()).id(metric.getId())
.name(metric.getName()).bizName(metric.getBizName())
.type(SchemaElementType.TAG).useCnt(metric.getUseCnt()).alias(alias)
.defaultAgg(metric.getDefaultAgg()).isTag(metric.getIsTag())
.description(metric.getDescription()).build();
tags.add(tagToAdd);
}
}
@@ -103,21 +88,11 @@ public class DataSetSchemaBuilder {
}
}
if (dim.getIsTag() == 1) {
SchemaElement tagToAdd =
SchemaElement.builder()
.dataSetId(resp.getId())
.dataSetName(resp.getName())
.model(dim.getModelId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())
.type(SchemaElementType.TAG)
.useCnt(dim.getUseCnt())
.alias(alias)
.schemaValueMaps(schemaValueMaps)
.isTag(dim.getIsTag())
.description(dim.getDescription())
.build();
SchemaElement tagToAdd = SchemaElement.builder().dataSetId(resp.getId())
.dataSetName(resp.getName()).model(dim.getModelId()).id(dim.getId())
.name(dim.getName()).bizName(dim.getBizName()).type(SchemaElementType.TAG)
.useCnt(dim.getUseCnt()).alias(alias).schemaValueMaps(schemaValueMaps)
.isTag(dim.getIsTag()).description(dim.getDescription()).build();
tags.add(tagToAdd);
}
}
@@ -129,15 +104,9 @@ public class DataSetSchemaBuilder {
if (Objects.isNull(dim)) {
return null;
}
return SchemaElement.builder()
.dataSetId(resp.getId())
.model(dim.getModelId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())
.type(SchemaElementType.ENTITY)
.useCnt(dim.getUseCnt())
.alias(dim.getEntityAlias())
return SchemaElement.builder().dataSetId(resp.getId()).model(dim.getModelId())
.id(dim.getId()).name(dim.getName()).bizName(dim.getBizName())
.type(SchemaElementType.ENTITY).useCnt(dim.getUseCnt()).alias(dim.getEntityAlias())
.build();
}
@@ -154,21 +123,11 @@ public class DataSetSchemaBuilder {
schemaValueMaps.add(schemaValueMap);
}
}
SchemaElement dimToAdd =
SchemaElement.builder()
.dataSetId(resp.getId())
.dataSetName(resp.getName())
.model(dim.getModelId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())
.useCnt(dim.getUseCnt())
.alias(alias)
.schemaValueMaps(schemaValueMaps)
.isTag(dim.getIsTag())
.description(dim.getDescription())
.type(SchemaElementType.DIMENSION)
.build();
SchemaElement dimToAdd = SchemaElement.builder().dataSetId(resp.getId())
.dataSetName(resp.getName()).model(dim.getModelId()).id(dim.getId())
.name(dim.getName()).bizName(dim.getBizName()).useCnt(dim.getUseCnt())
.alias(alias).schemaValueMaps(schemaValueMaps).isTag(dim.getIsTag())
.description(dim.getDescription()).type(SchemaElementType.DIMENSION).build();
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TYPE, dim.getType());
if (dim.isTimeDimension()) {
@@ -196,22 +155,12 @@ public class DataSetSchemaBuilder {
}
}
}
SchemaElement dimValueToAdd =
SchemaElement.builder()
.dataSetId(resp.getId())
.dataSetName(resp.getName())
.model(dim.getModelId())
.id(dim.getId())
.name(dim.getName())
.bizName(dim.getBizName())
.type(SchemaElementType.VALUE)
.useCnt(dim.getUseCnt())
.alias(
new ArrayList<>(
Arrays.asList(dimValueAlias.toArray(new String[0]))))
.isTag(dim.getIsTag())
.description(dim.getDescription())
.build();
SchemaElement dimValueToAdd = SchemaElement.builder().dataSetId(resp.getId())
.dataSetName(resp.getName()).model(dim.getModelId()).id(dim.getId())
.name(dim.getName()).bizName(dim.getBizName()).type(SchemaElementType.VALUE)
.useCnt(dim.getUseCnt())
.alias(new ArrayList<>(Arrays.asList(dimValueAlias.toArray(new String[0]))))
.isTag(dim.getIsTag()).description(dim.getDescription()).build();
dimensionValues.add(dimValueToAdd);
}
return dimensionValues;
@@ -223,23 +172,13 @@ public class DataSetSchemaBuilder {
List<String> alias = SchemaItem.getAliasList(metric.getAlias());
SchemaElement metricToAdd =
SchemaElement.builder()
.dataSetId(resp.getId())
.dataSetName(resp.getName())
.model(metric.getModelId())
.id(metric.getId())
.name(metric.getName())
.bizName(metric.getBizName())
.type(SchemaElementType.METRIC)
.useCnt(metric.getUseCnt())
.alias(alias)
.relatedSchemaElements(getRelateSchemaElement(metric))
.defaultAgg(metric.getDefaultAgg())
.dataFormatType(metric.getDataFormatType())
.isTag(metric.getIsTag())
.description(metric.getDescription())
.build();
SchemaElement metricToAdd = SchemaElement.builder().dataSetId(resp.getId())
.dataSetName(resp.getName()).model(metric.getModelId()).id(metric.getId())
.name(metric.getName()).bizName(metric.getBizName())
.type(SchemaElementType.METRIC).useCnt(metric.getUseCnt()).alias(alias)
.relatedSchemaElements(getRelateSchemaElement(metric))
.defaultAgg(metric.getDefaultAgg()).dataFormatType(metric.getDataFormatType())
.isTag(metric.getIsTag()).description(metric.getDescription()).build();
metrics.add(metricToAdd);
}
return metrics;
@@ -250,18 +189,10 @@ public class DataSetSchemaBuilder {
for (TermResp termResp : resp.getTermResps()) {
List<String> alias = termResp.getAlias();
SchemaElement metricToAdd =
SchemaElement.builder()
.dataSetId(resp.getId())
.dataSetName(resp.getName())
.model(-1L)
.id(termResp.getId())
.name(termResp.getName())
.bizName(termResp.getName())
.type(SchemaElementType.TERM)
.useCnt(0L)
.alias(alias)
.description(termResp.getDescription())
.build();
SchemaElement.builder().dataSetId(resp.getId()).dataSetName(resp.getName())
.model(-1L).id(termResp.getId()).name(termResp.getName())
.bizName(termResp.getName()).type(SchemaElementType.TERM).useCnt(0L)
.alias(alias).description(termResp.getDescription()).build();
terms.add(metricToAdd);
}
return terms;
@@ -274,26 +205,19 @@ public class DataSetSchemaBuilder {
|| CollectionUtils.isEmpty(relateDimension.getDrillDownDimensions())) {
return Lists.newArrayList();
}
return relateDimension.getDrillDownDimensions().stream()
.map(
dimension -> {
RelatedSchemaElement relateSchemaElement = new RelatedSchemaElement();
BeanUtils.copyProperties(dimension, relateSchemaElement);
return relateSchemaElement;
})
.collect(Collectors.toList());
return relateDimension.getDrillDownDimensions().stream().map(dimension -> {
RelatedSchemaElement relateSchemaElement = new RelatedSchemaElement();
BeanUtils.copyProperties(dimension, relateSchemaElement);
return relateSchemaElement;
}).collect(Collectors.toList());
}
private static void setDefaultTimeFormat(
SchemaElement dimToAdd,
DimensionTimeTypeParams dimensionTimeTypeParams,
String timeFormat) {
if (null != dimensionTimeTypeParams
&& TimeDimensionEnum.DAY
.name()
.equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
dimToAdd.getExtInfo()
.put(DimensionConstants.DIMENSION_TIME_FORMAT, DateUtils.DEFAULT_DATE_FORMAT);
private static void setDefaultTimeFormat(SchemaElement dimToAdd,
DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) {
if (null != dimensionTimeTypeParams && TimeDimensionEnum.DAY.name()
.equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT,
DateUtils.DEFAULT_DATE_FORMAT);
} else {
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
}

View File

@@ -90,11 +90,8 @@ public class DictUtils {
private final ModelService modelService;
private final TagMetaService tagMetaService;
public DictUtils(
DimensionService dimensionService,
MetricService metricService,
SemanticLayerService queryService,
ModelService modelService,
public DictUtils(DimensionService dimensionService, MetricService metricService,
SemanticLayerService queryService, ModelService modelService,
@Lazy TagMetaService tagMetaService) {
this.dimensionService = dimensionService;
this.metricService = metricService;
@@ -104,13 +101,12 @@ public class DictUtils {
}
public String fetchDictFileName(DictItemResp dictItemResp) {
return String.format(
"dic_value_%d_%s_%s",
dictItemResp.getModelId(), dictItemResp.getType().name(), dictItemResp.getItemId());
return String.format("dic_value_%d_%s_%s", dictItemResp.getModelId(),
dictItemResp.getType().name(), dictItemResp.getItemId());
}
public DictTaskDO generateDictTaskDO(
DictItemResp dictItemResp, User user, TaskStatusEnum status) {
public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user,
TaskStatusEnum status) {
DictTaskDO taskDO = new DictTaskDO();
Date createAt = new Date();
String name = dictItemResp.fetchDictFileName();
@@ -141,14 +137,12 @@ public class DictUtils {
public List<DictItemResp> dictDOList2Req(List<DictConfDO> dictConfDOList) {
List<DictItemResp> dictItemReqList = new ArrayList<>();
dictConfDOList.stream()
.forEach(
conf -> {
DictItemResp dictItemResp = dictDO2Req(conf);
if (Objects.nonNull(dictItemResp)) {
dictItemReqList.add(dictDO2Req(conf));
}
});
dictConfDOList.stream().forEach(conf -> {
DictItemResp dictItemResp = dictDO2Req(conf);
if (Objects.nonNull(dictItemResp)) {
dictItemReqList.add(dictDO2Req(conf));
}
});
return dictItemReqList;
}
@@ -190,10 +184,8 @@ public class DictUtils {
Map<String, Long> valueAndFrequencyPair = new HashMap<>(2000);
for (Map<String, Object> line : semanticQueryResp.getResultList()) {
if (CollectionUtils.isEmpty(line)
|| !line.containsKey(bizName)
|| line.get(bizName) == null
|| line.size() != 2) {
if (CollectionUtils.isEmpty(line) || !line.containsKey(bizName)
|| line.get(bizName) == null || line.size() != 2) {
continue;
}
String dimValue = line.get(bizName).toString();
@@ -218,38 +210,35 @@ public class DictUtils {
}
private void addWhiteValueLines(DictItemResp dictItemResp, List<String> lines, String nature) {
if (Objects.isNull(dictItemResp)
|| Objects.isNull(dictItemResp.getConfig())
if (Objects.isNull(dictItemResp) || Objects.isNull(dictItemResp.getConfig())
|| CollectionUtils.isEmpty(dictItemResp.getConfig().getWhiteList())) {
return;
}
List<String> whiteList = dictItemResp.getConfig().getWhiteList();
whiteList.forEach(
white -> {
if (!StringUtils.isEmpty(white)) {
white = white.replace(SPACE, POUND);
}
lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency));
});
whiteList.forEach(white -> {
if (!StringUtils.isEmpty(white)) {
white = white.replace(SPACE, POUND);
}
lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency));
});
}
private void constructDictLines(
Map<String, Long> valueAndFrequencyPair, List<String> lines, String nature) {
private void constructDictLines(Map<String, Long> valueAndFrequencyPair, List<String> lines,
String nature) {
if (CollectionUtils.isEmpty(valueAndFrequencyPair)) {
return;
}
valueAndFrequencyPair.forEach(
(value, frequency) -> {
if (!StringUtils.isEmpty(value)) {
value = value.replace(SPACE, POUND);
}
lines.add(String.format("%s %s %s", value, nature, frequency));
});
valueAndFrequencyPair.forEach((value, frequency) -> {
if (!StringUtils.isEmpty(value)) {
value = value.replace(SPACE, POUND);
}
lines.add(String.format("%s %s %s", value, nature, frequency));
});
}
private void mergeMultivaluedValue(
Map<String, Long> valueAndFrequencyPair, String dimValue, Long metric) {
private void mergeMultivaluedValue(Map<String, Long> valueAndFrequencyPair, String dimValue,
Long metric) {
if (StringUtils.isEmpty(dimValue)) {
return;
}
@@ -263,8 +252,7 @@ public class DictUtils {
for (String value : tmp.keySet()) {
long metricOld =
valueAndFrequencyPair.containsKey(value)
? valueAndFrequencyPair.get(value)
valueAndFrequencyPair.containsKey(value) ? valueAndFrequencyPair.get(value)
: 0L;
valueAndFrequencyPair.put(value, metric + metricOld);
}
@@ -286,8 +274,7 @@ public class DictUtils {
String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr;
ItemValueConfig config = dictItemResp.getConfig();
int limit =
(Objects.isNull(config) || Objects.isNull(config.getLimit()))
? itemValueMaxCount
(Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount
: dictItemResp.getConfig().getLimit();
// todo 自定义指标
@@ -312,8 +299,7 @@ public class DictUtils {
}
private QuerySqlReq constructDimQueryReq(DictItemResp dictItemResp) {
if (Objects.nonNull(dictItemResp)
&& Objects.nonNull(dictItemResp.getConfig())
if (Objects.nonNull(dictItemResp) && Objects.nonNull(dictItemResp.getConfig())
&& Objects.nonNull(dictItemResp.getConfig().getMetricId())) {
// 查询默认指标
QueryStructReq queryStructReq = generateQueryStruct(dictItemResp);
@@ -332,8 +318,7 @@ public class DictUtils {
String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr;
ItemValueConfig config = dictItemResp.getConfig();
long limit =
(Objects.isNull(config) || Objects.isNull(config.getLimit()))
? itemValueMaxCount
(Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount
: dictItemResp.getConfig().getLimit();
String sql = String.format(sqlPattern, bizName, where, bizName, limit);
Set<Long> modelIds = new HashSet<>();
@@ -371,10 +356,8 @@ public class DictUtils {
fillStructDateInfo(queryStructReq, dictItemResp);
int limit =
Objects.isNull(dictItemResp.getConfig().getLimit())
? itemValueMaxCount
: dictItemResp.getConfig().getLimit();
int limit = Objects.isNull(dictItemResp.getConfig().getLimit()) ? itemValueMaxCount
: dictItemResp.getConfig().getLimit();
queryStructReq.setLimit(limit);
queryStructReq.setNeedAuth(false);
return queryStructReq;
@@ -410,25 +393,18 @@ public class DictUtils {
}
}
private void fillStructDateBetween(
QueryStructReq queryStructReq,
ModelResp model,
Integer itemValueDateStart,
Integer itemValueDateEnd) {
private void fillStructDateBetween(QueryStructReq queryStructReq, ModelResp model,
Integer itemValueDateStart, Integer itemValueDateEnd) {
if (Objects.nonNull(model)) {
List<Dim> timeDims = model.getTimeDimension();
if (!CollectionUtils.isEmpty(timeDims)) {
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateConf.DateMode.BETWEEN);
String format = timeDims.get(0).getDateFormat();
String start =
LocalDate.now()
.minusDays(itemValueDateStart)
.format(DateTimeFormatter.ofPattern(format));
String end =
LocalDate.now()
.minusDays(itemValueDateEnd)
.format(DateTimeFormatter.ofPattern(format));
String start = LocalDate.now().minusDays(itemValueDateStart)
.format(DateTimeFormatter.ofPattern(format));
String end = LocalDate.now().minusDays(itemValueDateEnd)
.format(DateTimeFormatter.ofPattern(format));
dateConf.setStartDate(start);
dateConf.setEndDate(end);
queryStructReq.setDateInfo(dateConf);
@@ -477,17 +453,12 @@ public class DictUtils {
public String defaultDateFilter() {
String format = itemValueDateFormat;
String start =
LocalDate.now()
.minusDays(itemValueDateStart)
.format(DateTimeFormatter.ofPattern(format));
String end =
LocalDate.now()
.minusDays(itemValueDateEnd)
.format(DateTimeFormatter.ofPattern(format));
return String.format(
"( %s >= '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(), end);
String start = LocalDate.now().minusDays(itemValueDateStart)
.format(DateTimeFormatter.ofPattern(format));
String end = LocalDate.now().minusDays(itemValueDateEnd)
.format(DateTimeFormatter.ofPattern(format));
return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(),
start, TimeDimensionEnum.DAY.getName(), end);
}
private String generateDictDateFilter(DictItemResp dictItemResp) {
@@ -502,11 +473,8 @@ public class DictUtils {
}
// 静态日期
if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) {
return String.format(
"( %s >= '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(),
config.getDateConf().getStartDate(),
TimeDimensionEnum.DAY.getName(),
return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(),
config.getDateConf().getStartDate(), TimeDimensionEnum.DAY.getName(),
config.getDateConf().getEndDate());
}
// 动态日期
@@ -527,18 +495,12 @@ public class DictUtils {
dateFormat = itemValueDateFormat;
}
String start =
LocalDate.now()
.minusDays(dictItemResp.getConfig().getDateConf().getUnit())
LocalDate.now().minusDays(dictItemResp.getConfig().getDateConf().getUnit())
.format(DateTimeFormatter.ofPattern(dateFormat));
String end =
LocalDate.now()
.minusDays(0)
.format(DateTimeFormatter.ofPattern(dateFormat));
return String.format(
"( %s > '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(),
start,
TimeDimensionEnum.DAY.getName(),
String end = LocalDate.now().minusDays(0)
.format(DateTimeFormatter.ofPattern(dateFormat));
return String.format("( %s > '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(),
end);
}
}

View File

@@ -69,24 +69,20 @@ public class DimensionConverter {
return dimensionDO;
}
public static DimensionResp convert2DimensionResp(
DimensionDO dimensionDO, Map<Long, ModelResp> modelRespMap) {
public static DimensionResp convert2DimensionResp(DimensionDO dimensionDO,
Map<Long, ModelResp> modelRespMap) {
DimensionResp dimensionResp = new DimensionResp();
BeanUtils.copyProperties(dimensionDO, dimensionResp);
dimensionResp.setModelName(
modelRespMap.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getName());
dimensionResp.setModelBizName(
modelRespMap
.getOrDefault(dimensionResp.getModelId(), new ModelResp())
.getBizName());
dimensionResp.setModelBizName(modelRespMap
.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getBizName());
if (dimensionDO.getDefaultValues() != null) {
dimensionResp.setDefaultValues(
JSONObject.parseObject(dimensionDO.getDefaultValues(), List.class));
}
dimensionResp.setModelFilterSql(
modelRespMap
.getOrDefault(dimensionResp.getModelId(), new ModelResp())
.getFilterSql());
dimensionResp.setModelFilterSql(modelRespMap
.getOrDefault(dimensionResp.getModelId(), new ModelResp()).getFilterSql());
if (StringUtils.isNotEmpty(dimensionDO.getDimValueMaps())) {
dimensionResp.setDimValueMaps(
JsonUtil.toList(dimensionDO.getDimValueMaps(), DimValueMap.class));
@@ -98,9 +94,8 @@ public class DimensionConverter {
dimensionResp.setExt(JSONObject.parseObject(dimensionDO.getExt(), Map.class));
}
if (StringUtils.isNoneBlank(dimensionDO.getTypeParams())) {
dimensionResp.setTypeParams(
JSONObject.parseObject(
dimensionDO.getTypeParams(), DimensionTimeTypeParams.class));
dimensionResp.setTypeParams(JSONObject.parseObject(dimensionDO.getTypeParams(),
DimensionTimeTypeParams.class));
}
dimensionResp.setType(getType(dimensionDO.getType()));
dimensionResp.setTypeEnum(TypeEnums.DIMENSION);
@@ -122,15 +117,12 @@ public class DimensionConverter {
}
}
public static List<DimensionResp> filterByDataSet(
List<DimensionResp> dimensionResps, DataSetResp dataSetResp) {
public static List<DimensionResp> filterByDataSet(List<DimensionResp> dimensionResps,
DataSetResp dataSetResp) {
return dimensionResps.stream()
.filter(
dimensionResp ->
dataSetResp.dimensionIds().contains(dimensionResp.getId())
|| dataSetResp
.getAllIncludeAllModels()
.contains(dimensionResp.getModelId()))
.filter(dimensionResp -> dataSetResp.dimensionIds().contains(dimensionResp.getId())
|| dataSetResp.getAllIncludeAllModels()
.contains(dimensionResp.getModelId()))
.collect(Collectors.toList());
}
}

View File

@@ -29,22 +29,14 @@ public class DomainConvert {
DomainResp domainResp = new DomainResp();
BeanUtils.copyProperties(domainDO, domainResp);
domainResp.setFullPath(domainFullPathMap.get(domainDO.getId()));
domainResp.setAdmins(
StringUtils.isBlank(domainDO.getAdmin())
? Lists.newArrayList()
: Arrays.asList(domainDO.getAdmin().split(",")));
domainResp.setAdminOrgs(
StringUtils.isBlank(domainDO.getAdminOrg())
? Lists.newArrayList()
: Arrays.asList(domainDO.getAdminOrg().split(",")));
domainResp.setViewers(
StringUtils.isBlank(domainDO.getViewer())
? Lists.newArrayList()
: Arrays.asList(domainDO.getViewer().split(",")));
domainResp.setViewOrgs(
StringUtils.isBlank(domainDO.getViewOrg())
? Lists.newArrayList()
: Arrays.asList(domainDO.getViewOrg().split(",")));
domainResp.setAdmins(StringUtils.isBlank(domainDO.getAdmin()) ? Lists.newArrayList()
: Arrays.asList(domainDO.getAdmin().split(",")));
domainResp.setAdminOrgs(StringUtils.isBlank(domainDO.getAdminOrg()) ? Lists.newArrayList()
: Arrays.asList(domainDO.getAdminOrg().split(",")));
domainResp.setViewers(StringUtils.isBlank(domainDO.getViewer()) ? Lists.newArrayList()
: Arrays.asList(domainDO.getViewer().split(",")));
domainResp.setViewOrgs(StringUtils.isBlank(domainDO.getViewOrg()) ? Lists.newArrayList()
: Arrays.asList(domainDO.getViewOrg().split(",")));
return domainResp;
}

View File

@@ -70,13 +70,13 @@ public class MetricConverter {
return convert2MetricResp(metricDO, new HashMap<>(), Lists.newArrayList());
}
public static MetricResp convert2MetricResp(
MetricDO metricDO, Map<Long, ModelResp> modelMap, List<Long> collect) {
public static MetricResp convert2MetricResp(MetricDO metricDO, Map<Long, ModelResp> modelMap,
List<Long> collect) {
MetricResp metricResp = new MetricResp();
BeanUtils.copyProperties(metricDO, metricResp);
metricResp.setDataFormat(
JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class));
metricResp
.setDataFormat(JSONObject.parseObject(metricDO.getDataFormat(), DataFormat.class));
ModelResp modelResp = modelMap.get(metricDO.getModelId());
if (modelResp != null) {
metricResp.setModelName(modelResp.getName());
@@ -97,17 +97,14 @@ public class MetricConverter {
}
metricResp.setTypeEnum(TypeEnums.METRIC);
if (MetricDefineType.MEASURE.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setMetricDefineByMeasureParams(
JSONObject.parseObject(
metricDO.getTypeParams(), MetricDefineByMeasureParams.class));
metricResp.setMetricDefineByMeasureParams(JSONObject
.parseObject(metricDO.getTypeParams(), MetricDefineByMeasureParams.class));
} else if (MetricDefineType.METRIC.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setMetricDefineByMetricParams(
JSONObject.parseObject(
metricDO.getTypeParams(), MetricDefineByMetricParams.class));
metricResp.setMetricDefineByMetricParams(JSONObject
.parseObject(metricDO.getTypeParams(), MetricDefineByMetricParams.class));
} else if (MetricDefineType.FIELD.name().equalsIgnoreCase(metricDO.getDefineType())) {
metricResp.setMetricDefineByFieldParams(
JSONObject.parseObject(
metricDO.getTypeParams(), MetricDefineByFieldParams.class));
metricResp.setMetricDefineByFieldParams(JSONObject.parseObject(metricDO.getTypeParams(),
MetricDefineByFieldParams.class));
}
if (metricDO.getDefineType() != null) {
metricResp.setMetricDefineType(MetricDefineType.valueOf(metricDO.getDefineType()));
@@ -116,15 +113,11 @@ public class MetricConverter {
return metricResp;
}
public static List<MetricResp> filterByDataSet(
List<MetricResp> metricResps, DataSetResp dataSetResp) {
public static List<MetricResp> filterByDataSet(List<MetricResp> metricResps,
DataSetResp dataSetResp) {
return metricResps.stream()
.filter(
metricResp ->
dataSetResp.metricIds().contains(metricResp.getId())
|| dataSetResp
.getAllIncludeAllModels()
.contains(metricResp.getModelId()))
.filter(metricResp -> dataSetResp.metricIds().contains(metricResp.getId())
|| dataSetResp.getAllIncludeAllModels().contains(metricResp.getModelId()))
.collect(Collectors.toList());
}
}

View File

@@ -27,7 +27,8 @@ import java.util.stream.Collectors;
@Slf4j
public class MetricDrillDownChecker {
@Autowired private MetricService metricService;
@Autowired
private MetricService metricService;
public void checkQuery(QueryStatement queryStatement) {
SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
@@ -54,12 +55,8 @@ public class MetricDrillDownChecker {
getNecessaryDimensionMissing(necessaryDimensions, dimensionFields);
if (!CollectionUtils.isEmpty(dimensionsMissing)) {
String errMsg =
String.format(
"指标:%s 缺失必要下钻维度:%s",
metric.getName(),
dimensionsMissing.stream()
.map(DimensionResp::getName)
.collect(Collectors.toList()));
String.format("指标:%s 缺失必要下钻维度:%s", metric.getName(), dimensionsMissing
.stream().map(DimensionResp::getName).collect(Collectors.toList()));
throw new InvalidArgumentException(errMsg);
}
}
@@ -95,28 +92,18 @@ public class MetricDrillDownChecker {
* To check whether the dimension can drill down the metric, eg: some descriptive dimensions are
* not suitable as drill-down dimensions
*/
private boolean checkDrillDownDimension(
String dimensionName,
List<MetricResp> metricResps,
private boolean checkDrillDownDimension(String dimensionName, List<MetricResp> metricResps,
SemanticSchemaResp semanticSchemaResp) {
if (CollectionUtils.isEmpty(metricResps)) {
return true;
}
List<String> relateDimensions =
metricResps.stream()
.map(this::getDrillDownDimensions)
.filter(
drillDownDimensions ->
!CollectionUtils.isEmpty(drillDownDimensions))
.map(
drillDownDimensions ->
drillDownDimensions.stream()
.map(DrillDownDimension::getDimensionId)
.collect(Collectors.toList()))
.flatMap(Collection::stream)
.map(id -> convertDimensionIdToBizName(id, semanticSchemaResp))
.filter(Objects::nonNull)
.collect(Collectors.toList());
List<String> relateDimensions = metricResps.stream().map(this::getDrillDownDimensions)
.filter(drillDownDimensions -> !CollectionUtils.isEmpty(drillDownDimensions))
.map(drillDownDimensions -> drillDownDimensions.stream()
.map(DrillDownDimension::getDimensionId).collect(Collectors.toList()))
.flatMap(Collection::stream)
.map(id -> convertDimensionIdToBizName(id, semanticSchemaResp))
.filter(Objects::nonNull).collect(Collectors.toList());
// if no metric has drill down dimension, return true
if (CollectionUtils.isEmpty(relateDimensions)) {
return true;
@@ -125,8 +112,8 @@ public class MetricDrillDownChecker {
return relateDimensions.contains(dimensionName);
}
private List<DimensionResp> getNecessaryDimensions(
MetricSchemaResp metric, SemanticSchemaResp semanticSchemaResp) {
private List<DimensionResp> getNecessaryDimensions(MetricSchemaResp metric,
SemanticSchemaResp semanticSchemaResp) {
if (metric == null) {
return Lists.newArrayList();
}
@@ -134,12 +121,9 @@ public class MetricDrillDownChecker {
if (CollectionUtils.isEmpty(drillDownDimensions)) {
return Lists.newArrayList();
}
return drillDownDimensions.stream()
.filter(DrillDownDimension::isNecessary)
.map(DrillDownDimension::getDimensionId)
.map(semanticSchemaResp::getDimension)
.filter(Objects::nonNull)
.collect(Collectors.toList());
return drillDownDimensions.stream().filter(DrillDownDimension::isNecessary)
.map(DrillDownDimension::getDimensionId).map(semanticSchemaResp::getDimension)
.filter(Objects::nonNull).collect(Collectors.toList());
}
private List<String> getDimensionFields(List<String> groupByFields, List<String> whereFields) {
@@ -153,8 +137,8 @@ public class MetricDrillDownChecker {
return dimensionFields;
}
private List<MetricResp> getMetrics(
List<String> metricFields, SemanticSchemaResp semanticSchemaResp) {
private List<MetricResp> getMetrics(List<String> metricFields,
SemanticSchemaResp semanticSchemaResp) {
return semanticSchemaResp.getMetrics().stream()
.filter(metricSchemaResp -> metricFields.contains(metricSchemaResp.getBizName()))
.collect(Collectors.toList());

View File

@@ -19,11 +19,8 @@ public class ModelClusterBuilder {
public static Map<String, ModelCluster> buildModelClusters(List<Long> modelIds) {
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchemaResps(modelIds);
Map<Long, ModelSchemaResp> modelIdToModelSchema =
modelSchemaResps.stream()
.collect(
Collectors.toMap(
ModelSchemaResp::getId, value -> value, (k1, k2) -> k1));
Map<Long, ModelSchemaResp> modelIdToModelSchema = modelSchemaResps.stream()
.collect(Collectors.toMap(ModelSchemaResp::getId, value -> value, (k1, k2) -> k1));
Set<Long> visited = new HashSet<>();
List<Set<Long>> modelClusters = new ArrayList<>();
@@ -40,25 +37,17 @@ public class ModelClusterBuilder {
.collect(Collectors.toMap(ModelCluster::getKey, value -> value, (k1, k2) -> k1));
}
private static ModelCluster getModelCluster(
Map<Long, ModelSchemaResp> modelIdToModelSchema, Set<Long> modelIds) {
boolean containsPartitionDimensions =
modelIds.stream()
.map(modelIdToModelSchema::get)
.filter(Objects::nonNull)
.anyMatch(
modelSchemaResp ->
CollectionUtils.isNotEmpty(
modelSchemaResp.getTimeDimension()));
private static ModelCluster getModelCluster(Map<Long, ModelSchemaResp> modelIdToModelSchema,
Set<Long> modelIds) {
boolean containsPartitionDimensions = modelIds.stream().map(modelIdToModelSchema::get)
.filter(Objects::nonNull).anyMatch(modelSchemaResp -> CollectionUtils
.isNotEmpty(modelSchemaResp.getTimeDimension()));
return ModelCluster.build(modelIds, containsPartitionDimensions);
}
private static void dfs(
ModelSchemaResp model,
Map<Long, ModelSchemaResp> modelMap,
Set<Long> visited,
Set<Long> modelCluster) {
private static void dfs(ModelSchemaResp model, Map<Long, ModelSchemaResp> modelMap,
Set<Long> visited, Set<Long> modelCluster) {
visited.add(model.getId());
modelCluster.add(model.getId());
for (Long neighborId : model.getModelClusterSet()) {

Some files were not shown because too many files have changed in this diff Show More