[fix]Fix unit test cases.

This commit is contained in:
jerryjzhang
2025-08-05 17:22:10 +08:00
parent bf3213e8fb
commit 91e4b51ef8
22 changed files with 197 additions and 115 deletions

View File

@@ -19,7 +19,8 @@ public class ParseContext {
} }
public boolean enableNL2SQL() { public boolean enableNL2SQL() {
return Objects.nonNull(agent) && agent.containsDatasetTool()&&response.getSelectedParses().size() == 0; return Objects.nonNull(agent) && agent.containsDatasetTool()
&& response.getSelectedParses().size() == 0;
} }
public boolean enableLLM() { public boolean enableLLM() {

View File

@@ -727,7 +727,7 @@ public class SqlReplaceHelper {
List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList); List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
for (PlainSelect plainSelect : plainSelects) { for (PlainSelect plainSelect : plainSelects) {
if (Objects.nonNull(plainSelect.getFromItem())) { if (Objects.nonNull(plainSelect.getFromItem())) {
Table table = (Table) plainSelect.getFromItem(); Table table = SqlSelectHelper.getTable(plainSelect.getFromItem());
if (table.getName().equals(tableName)) { if (table.getName().equals(tableName)) {
replacePlainSelectByExpr(plainSelect, replace); replacePlainSelectByExpr(plainSelect, replace);
if (SqlSelectHelper.hasAggregateFunction(plainSelect)) { if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {

View File

@@ -723,6 +723,44 @@ public class SqlSelectHelper {
return null; return null;
} }
public static Table getTable(FromItem fromItem) {
Table table = null;
if (fromItem instanceof Table) {
table = (Table) fromItem;
} else if (fromItem instanceof ParenthesedSelect) {
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) fromItem;
if (parenthesedSelect.getSelect() instanceof PlainSelect) {
PlainSelect subSelect = (PlainSelect) parenthesedSelect.getSelect();
table = getTable(subSelect.getSelectBody());
} else if (parenthesedSelect.getSelect() instanceof SetOperationList) {
table = getTable(parenthesedSelect.getSelect());
}
}
return table;
}
public static Table getTable(Select select) {
if (select == null) {
return null;
}
List<PlainSelect> plainSelectList = getWithItem(select);
if (!CollectionUtils.isEmpty(plainSelectList)) {
List<PlainSelect> selectList = new ArrayList<>(plainSelectList);
Table table = getTable(selectList.get(0));
return table;
}
if (select instanceof PlainSelect) {
PlainSelect plainSelect = (PlainSelect) select;
return getTable(plainSelect.getFromItem());
} else if (select instanceof SetOperationList) {
SetOperationList setOperationList = (SetOperationList) select;
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
return getTable(setOperationList.getSelects().get(0));
}
}
return null;
}
public static String getDbTableName(String sql) { public static String getDbTableName(String sql) {
Table table = getTable(sql); Table table = getTable(sql);
return table.getFullyQualifiedName(); return table.getFullyQualifiedName();

View File

@@ -3,7 +3,11 @@ package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.pojo.*; import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
@@ -21,14 +25,22 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table; import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.select.*; import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.Limit;
import net.sf.jsqlparser.statement.select.Offset;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.ParenthesedSelect;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.SelectItem;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Data @Data
@@ -176,7 +188,7 @@ public class QueryStructReq extends SemanticQueryReq {
private List<SelectItem<?>> buildSelectItems(QueryStructReq queryStructReq) { private List<SelectItem<?>> buildSelectItems(QueryStructReq queryStructReq) {
List<SelectItem<?>> selectItems = new ArrayList<>(); List<SelectItem<?>> selectItems = new ArrayList<>();
List<String> groups = queryStructReq.getGroups(); Set<String> groups = new HashSet<>(queryStructReq.getGroups());
if (!CollectionUtils.isEmpty(groups)) { if (!CollectionUtils.isEmpty(groups)) {
for (String group : groups) { for (String group : groups) {
@@ -236,7 +248,7 @@ public class QueryStructReq extends SemanticQueryReq {
} }
private GroupByElement buildGroupByElement(QueryStructReq queryStructReq) { private GroupByElement buildGroupByElement(QueryStructReq queryStructReq) {
List<String> groups = queryStructReq.getGroups(); Set<String> groups = new HashSet<>(queryStructReq.getGroups());
if ((!CollectionUtils.isEmpty(groups) && !queryStructReq.getAggregators().isEmpty()) if ((!CollectionUtils.isEmpty(groups) && !queryStructReq.getAggregators().isEmpty())
|| !queryStructReq.getMetricFilters().isEmpty()) { || !queryStructReq.getMetricFilters().isEmpty()) {
GroupByElement groupByElement = new GroupByElement(); GroupByElement groupByElement = new GroupByElement();

View File

@@ -23,9 +23,9 @@ public class SqlExecuteReq {
private Integer limit = 1000; private Integer limit = 1000;
public String getSql() { public String getSql() {
if(StringUtils.isNotBlank(sql)){ if (StringUtils.isNotBlank(sql)) {
sql=sql.replaceAll("^[\\n]+|[\\n]+$", ""); sql = sql.replaceAll("^[\\n]+|[\\n]+$", "");
sql=StringUtils.removeEnd(sql,";"); sql = StringUtils.removeEnd(sql, ";");
} }
return String.format(LIMIT_WRAPPER, sql, limit); return String.format(LIMIT_WRAPPER, sql, limit);

View File

@@ -86,7 +86,7 @@ public class FileHandlerImpl implements FileHandler {
} }
private PageInfo<DictValueResp> getDictValueRespPagWithKey(String fileName, private PageInfo<DictValueResp> getDictValueRespPagWithKey(String fileName,
DictValueReq dictValueReq) { DictValueReq dictValueReq) {
PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>(); PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize()); dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize());
dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent()); dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
@@ -95,7 +95,7 @@ public class FileHandlerImpl implements FileHandler {
Integer startLine = 1; Integer startLine = 1;
List<DictValueResp> dictValueRespList = List<DictValueResp> dictValueRespList =
getFileData(filePath, startLine, fileLineNum.intValue()).stream().filter( getFileData(filePath, startLine, fileLineNum.intValue()).stream().filter(
dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue())) dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue()))
.collect(Collectors.toList()); .collect(Collectors.toList());
if (CollectionUtils.isEmpty(dictValueRespList)) { if (CollectionUtils.isEmpty(dictValueRespList)) {
dictValueRespPageInfo.setList(new ArrayList<>()); dictValueRespPageInfo.setList(new ArrayList<>());
@@ -118,7 +118,7 @@ public class FileHandlerImpl implements FileHandler {
} }
private PageInfo<DictValueResp> getDictValueRespPagWithoutKey(String fileName, private PageInfo<DictValueResp> getDictValueRespPagWithoutKey(String fileName,
DictValueReq dictValueReq) { DictValueReq dictValueReq) {
PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>(); PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName; String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
Long fileLineNum = getFileLineNum(filePath); Long fileLineNum = getFileLineNum(filePath);
@@ -175,7 +175,7 @@ public class FileHandlerImpl implements FileHandler {
private DictValueResp convert2Resp(String lineStr) { private DictValueResp convert2Resp(String lineStr) {
DictValueResp dictValueResp = new DictValueResp(); DictValueResp dictValueResp = new DictValueResp();
if (StringUtils.isNotEmpty(lineStr)) { if (StringUtils.isNotEmpty(lineStr)) {
lineStr=StringUtils.stripStart(lineStr,null); lineStr = StringUtils.stripStart(lineStr, null);
String[] itemArray = lineStr.split("\\s+"); String[] itemArray = lineStr.split("\\s+");
if (Objects.nonNull(itemArray) && itemArray.length >= 3) { if (Objects.nonNull(itemArray) && itemArray.length >= 3) {
dictValueResp.setValue(itemArray[0].replace("#", " ")); dictValueResp.setValue(itemArray[0].replace("#", " "));

View File

@@ -63,7 +63,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
@Override @Override
public List<EmbeddingResult> detect(ChatQueryContext chatQueryContext, List<S2Term> terms, public List<EmbeddingResult> detect(ChatQueryContext chatQueryContext, List<S2Term> terms,
Set<Long> detectDataSetIds) { Set<Long> detectDataSetIds) {
if (chatQueryContext == null || CollectionUtils.isEmpty(detectDataSetIds)) { if (chatQueryContext == null || CollectionUtils.isEmpty(detectDataSetIds)) {
log.warn("Invalid input parameters: context={}, dataSetIds={}", chatQueryContext, log.warn("Invalid input parameters: context={}, dataSetIds={}", chatQueryContext,
detectDataSetIds); detectDataSetIds);
@@ -92,7 +92,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
* Perform enhanced detection using LLM * Perform enhanced detection using LLM
*/ */
private List<EmbeddingResult> detectWithLLM(ChatQueryContext chatQueryContext, private List<EmbeddingResult> detectWithLLM(ChatQueryContext chatQueryContext,
Set<Long> detectDataSetIds) { Set<Long> detectDataSetIds) {
try { try {
String queryText = chatQueryContext.getRequest().getQueryText(); String queryText = chatQueryContext.getRequest().getQueryText();
if (StringUtils.isBlank(queryText)) { if (StringUtils.isBlank(queryText)) {
@@ -126,7 +126,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
@Override @Override
public List<EmbeddingResult> detectByBatch(ChatQueryContext chatQueryContext, public List<EmbeddingResult> detectByBatch(ChatQueryContext chatQueryContext,
Set<Long> detectDataSetIds, Set<String> detectSegments) { Set<Long> detectDataSetIds, Set<String> detectSegments) {
return detectByBatch(chatQueryContext, detectDataSetIds, detectSegments, false); return detectByBatch(chatQueryContext, detectDataSetIds, detectSegments, false);
} }
@@ -140,7 +140,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
* @return List of embedding results * @return List of embedding results
*/ */
public List<EmbeddingResult> detectByBatch(ChatQueryContext chatQueryContext, public List<EmbeddingResult> detectByBatch(ChatQueryContext chatQueryContext,
Set<Long> detectDataSetIds, Set<String> detectSegments, boolean useLlm) { Set<Long> detectDataSetIds, Set<String> detectSegments, boolean useLlm) {
Set<EmbeddingResult> results = ConcurrentHashMap.newKeySet(); Set<EmbeddingResult> results = ConcurrentHashMap.newKeySet();
int embeddingMapperBatch = Integer int embeddingMapperBatch = Integer
.valueOf(mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_BATCH)); .valueOf(mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_BATCH));
@@ -168,10 +168,11 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
variable.put("retrievedInfo", JSONObject.toJSONString(results)); variable.put("retrievedInfo", JSONObject.toJSONString(results));
Prompt prompt = PromptTemplate.from(LLM_FILTER_PROMPT).apply(variable); Prompt prompt = PromptTemplate.from(LLM_FILTER_PROMPT).apply(variable);
ChatModelConfig chatModelConfig=null; ChatModelConfig chatModelConfig = null;
if(chatQueryContext.getRequest().getChatAppConfig()!=null if (chatQueryContext.getRequest().getChatAppConfig() != null && chatQueryContext
&& chatQueryContext.getRequest().getChatAppConfig().containsKey("REWRITE_MULTI_TURN")){ .getRequest().getChatAppConfig().containsKey("REWRITE_MULTI_TURN")) {
chatModelConfig=chatQueryContext.getRequest().getChatAppConfig().get("REWRITE_MULTI_TURN").getChatModelConfig(); chatModelConfig = chatQueryContext.getRequest().getChatAppConfig()
.get("REWRITE_MULTI_TURN").getChatModelConfig();
} }
ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatModelConfig); ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatModelConfig);
String response = chatLanguageModel.generate(prompt.toUserMessage().singleText()); String response = chatLanguageModel.generate(prompt.toUserMessage().singleText());
@@ -200,7 +201,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
* @return Callable task * @return Callable task
*/ */
private Callable<Void> createTask(ChatQueryContext chatQueryContext, Set<Long> detectDataSetIds, private Callable<Void> createTask(ChatQueryContext chatQueryContext, Set<Long> detectDataSetIds,
List<String> queryTextsSub, Set<EmbeddingResult> results, boolean useLlm) { List<String> queryTextsSub, Set<EmbeddingResult> results, boolean useLlm) {
return () -> { return () -> {
List<EmbeddingResult> oneRoundResults = detectByQueryTextsSub(detectDataSetIds, List<EmbeddingResult> oneRoundResults = detectByQueryTextsSub(detectDataSetIds,
queryTextsSub, chatQueryContext, useLlm); queryTextsSub, chatQueryContext, useLlm);
@@ -221,7 +222,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
* @return List of embedding results for this batch * @return List of embedding results for this batch
*/ */
private List<EmbeddingResult> detectByQueryTextsSub(Set<Long> detectDataSetIds, private List<EmbeddingResult> detectByQueryTextsSub(Set<Long> detectDataSetIds,
List<String> queryTextsSub, ChatQueryContext chatQueryContext, boolean useLlm) { List<String> queryTextsSub, ChatQueryContext chatQueryContext, boolean useLlm) {
Map<Long, List<Long>> modelIdToDataSetIds = chatQueryContext.getModelIdToDataSetIds(); Map<Long, List<Long>> modelIdToDataSetIds = chatQueryContext.getModelIdToDataSetIds();
// Get configuration parameters // Get configuration parameters
@@ -243,12 +244,12 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
// Process results // Process results
List<EmbeddingResult> collect = retrieveQueryResults.stream().peek(result -> { List<EmbeddingResult> collect = retrieveQueryResults.stream().peek(result -> {
if (!useLlm && CollectionUtils.isNotEmpty(result.getRetrieval())) { if (!useLlm && CollectionUtils.isNotEmpty(result.getRetrieval())) {
result.getRetrieval() result.getRetrieval()
.removeIf(retrieval -> !result.getQuery().contains(retrieval.getQuery()) .removeIf(retrieval -> !result.getQuery().contains(retrieval.getQuery())
&& retrieval.getSimilarity() < threshold); && retrieval.getSimilarity() < threshold);
} }
}).filter(result -> CollectionUtils.isNotEmpty(result.getRetrieval())) }).filter(result -> CollectionUtils.isNotEmpty(result.getRetrieval()))
.flatMap(result -> result.getRetrieval().stream() .flatMap(result -> result.getRetrieval().stream()
.map(retrieval -> convertToEmbeddingResult(result, retrieval))) .map(retrieval -> convertToEmbeddingResult(result, retrieval)))
.collect(Collectors.toList()); .collect(Collectors.toList());
@@ -267,7 +268,7 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
* @return Converted EmbeddingResult * @return Converted EmbeddingResult
*/ */
private EmbeddingResult convertToEmbeddingResult(RetrieveQueryResult queryResult, private EmbeddingResult convertToEmbeddingResult(RetrieveQueryResult queryResult,
Retrieval retrieval) { Retrieval retrieval) {
EmbeddingResult embeddingResult = new EmbeddingResult(); EmbeddingResult embeddingResult = new EmbeddingResult();
BeanUtils.copyProperties(retrieval, embeddingResult); BeanUtils.copyProperties(retrieval, embeddingResult);
embeddingResult.setDetectWord(queryResult.getQuery()); embeddingResult.setDetectWord(queryResult.getQuery());

View File

@@ -51,7 +51,7 @@ public class KeywordMapper extends BaseMapper {
} }
private void convertMapResultToMapInfo(List<HanlpMapResult> mapResults, private void convertMapResultToMapInfo(List<HanlpMapResult> mapResults,
ChatQueryContext chatQueryContext, List<S2Term> terms) { ChatQueryContext chatQueryContext, List<S2Term> terms) {
if (CollectionUtils.isEmpty(mapResults)) { if (CollectionUtils.isEmpty(mapResults)) {
return; return;
} }
@@ -87,14 +87,15 @@ public class KeywordMapper extends BaseMapper {
.similarity(hanlpMapResult.getSimilarity()) .similarity(hanlpMapResult.getSimilarity())
.detectWord(hanlpMapResult.getDetectWord()).build(); .detectWord(hanlpMapResult.getDetectWord()).build();
// doDimValueAliasLogic 将维度值别名进行替换成真实维度值 // doDimValueAliasLogic 将维度值别名进行替换成真实维度值
doDimValueAliasLogic(schemaElementMatch,chatQueryContext.getSemanticSchema().getDimensionValues()); doDimValueAliasLogic(schemaElementMatch,
chatQueryContext.getSemanticSchema().getDimensionValues());
addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch); addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch);
} }
} }
} }
private void doDimValueAliasLogic(SchemaElementMatch schemaElementMatch, private void doDimValueAliasLogic(SchemaElementMatch schemaElementMatch,
List<SchemaElement> dimensionValues) { List<SchemaElement> dimensionValues) {
SchemaElement element = schemaElementMatch.getElement(); SchemaElement element = schemaElementMatch.getElement();
if (SchemaElementType.VALUE.equals(element.getType())) { if (SchemaElementType.VALUE.equals(element.getType())) {
Long dimId = element.getId(); Long dimId = element.getId();
@@ -126,7 +127,7 @@ public class KeywordMapper extends BaseMapper {
} }
private void convertMapResultToMapInfo(ChatQueryContext chatQueryContext, private void convertMapResultToMapInfo(ChatQueryContext chatQueryContext,
List<DatabaseMapResult> mapResults) { List<DatabaseMapResult> mapResults) {
for (DatabaseMapResult match : mapResults) { for (DatabaseMapResult match : mapResults) {
SchemaElement schemaElement = match.getSchemaElement(); SchemaElement schemaElement = match.getSchemaElement();
Set<Long> regElementSet = Set<Long> regElementSet =
@@ -153,8 +154,8 @@ public class KeywordMapper extends BaseMapper {
return new HashSet<>(); return new HashSet<>();
} }
return elements.stream().filter( return elements.stream().filter(
elementMatch -> SchemaElementType.METRIC.equals(elementMatch.getElement().getType()) elementMatch -> SchemaElementType.METRIC.equals(elementMatch.getElement().getType())
|| SchemaElementType.DIMENSION.equals(elementMatch.getElement().getType())) || SchemaElementType.DIMENSION.equals(elementMatch.getElement().getType()))
.map(elementMatch -> elementMatch.getElement().getId()).collect(Collectors.toSet()); .map(elementMatch -> elementMatch.getElement().getId()).collect(Collectors.toSet());
} }
} }

View File

@@ -124,7 +124,7 @@ public class DimValueAspect {
sql = SqlReplaceHelper.replaceValue(sql, filedNameToValueMap); sql = SqlReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.debug("correctorSql after replacing:{}", sql); log.debug("correctorSql after replacing:{}", sql);
querySqlReq.setSql(sql); querySqlReq.setSql(sql);
querySqlReq.getSqlInfo().setQuerySQL(sql); // querySqlReq.getSqlInfo().setQuerySQL(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions); Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();

View File

@@ -88,7 +88,7 @@ public class DimensionRepositoryImpl implements DimensionRepository {
} }
if (StringUtils.isNotBlank(dimensionFilter.getKey())) { if (StringUtils.isNotBlank(dimensionFilter.getKey())) {
String key = dimensionFilter.getKey(); String key = dimensionFilter.getKey();
queryWrapper.and(qw->qw.lambda().like(DimensionDO::getName, key).or() queryWrapper.and(qw -> qw.lambda().like(DimensionDO::getName, key).or()
.like(DimensionDO::getBizName, key).or().like(DimensionDO::getDescription, key) .like(DimensionDO::getBizName, key).or().like(DimensionDO::getDescription, key)
.or().like(DimensionDO::getAlias, key).or() .or().like(DimensionDO::getAlias, key).or()
.like(DimensionDO::getCreatedBy, key)); .like(DimensionDO::getCreatedBy, key));

View File

@@ -114,14 +114,9 @@ public class MetricRepositoryImpl implements MetricRepository {
} }
if (StringUtils.isNotBlank(metricFilter.getKey())) { if (StringUtils.isNotBlank(metricFilter.getKey())) {
String key = metricFilter.getKey(); String key = metricFilter.getKey();
queryWrapper.lambda() queryWrapper.lambda().and(wrapper -> wrapper.like(MetricDO::getName, key).or()
.and(wrapper -> wrapper .like(MetricDO::getBizName, key).or().like(MetricDO::getDescription, key).or()
.like(MetricDO::getName, key) .like(MetricDO::getAlias, key).or().like(MetricDO::getCreatedBy, key));
.or().like(MetricDO::getBizName, key)
.or().like(MetricDO::getDescription, key)
.or().like(MetricDO::getAlias, key)
.or().like(MetricDO::getCreatedBy, key)
);
} }
return metricDOMapper.selectList(queryWrapper); return metricDOMapper.selectList(queryWrapper);

View File

@@ -51,8 +51,9 @@ public class DataSetController {
@GetMapping("/getDataSetList") @GetMapping("/getDataSetList")
public List<DataSetResp> getDataSetList(@RequestParam("domainId") Long domainId) { public List<DataSetResp> getDataSetList(@RequestParam("domainId") Long domainId) {
List<Integer> statuCodeList = Arrays.asList(StatusEnum.ONLINE.getCode(),StatusEnum.OFFLINE.getCode()); List<Integer> statuCodeList =
return dataSetService.getDataSetList(domainId,statuCodeList); Arrays.asList(StatusEnum.ONLINE.getCode(), StatusEnum.OFFLINE.getCode());
return dataSetService.getDataSetList(domainId, statuCodeList);
} }
@DeleteMapping("/{id}") @DeleteMapping("/{id}")

View File

@@ -20,7 +20,7 @@ public interface DataSetService {
List<DataSetResp> getDataSetList(MetaFilter metaFilter); List<DataSetResp> getDataSetList(MetaFilter metaFilter);
List<DataSetResp> getDataSetList(Long domainId ,List<Integer> statuCodesList); List<DataSetResp> getDataSetList(Long domainId, List<Integer> statuCodesList);
void delete(Long id, User user); void delete(Long id, User user);

View File

@@ -27,7 +27,8 @@ public interface DimensionService {
DimensionResp createDimension(DimensionReq dimensionReq, User user) throws Exception; DimensionResp createDimension(DimensionReq dimensionReq, User user) throws Exception;
void alterDimensionBatch(List<DimensionReq> dimensionReqs, Long modelId, User user) throws Exception; void alterDimensionBatch(List<DimensionReq> dimensionReqs, Long modelId, User user)
throws Exception;
void createDimensionBatch(List<DimensionReq> dimensionReqs, User user) throws Exception; void createDimensionBatch(List<DimensionReq> dimensionReqs, User user) throws Exception;

View File

@@ -55,7 +55,9 @@ public interface ModelService {
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user); void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
void updateModelByDimAndMetric(Long modelId, List<DimensionReq> dimensionReqList, List<MetricReq> metricReqList, User user); void updateModelByDimAndMetric(Long modelId, List<DimensionReq> dimensionReqList,
List<MetricReq> metricReqList, User user);
void deleteModelDetailByDimAndMetric(Long modelId, List<DimensionDO> dimensionReqList, List<MetricDO> metricReqList); void deleteModelDetailByDimAndMetric(Long modelId, List<DimensionDO> dimensionReqList,
List<MetricDO> metricReqList);
} }

View File

@@ -104,7 +104,7 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
@Override @Override
public List<DataSetResp> getDataSetList(Long domainId, List<Integer> statuCodesList) { public List<DataSetResp> getDataSetList(Long domainId, List<Integer> statuCodesList) {
if(domainId==null || CollectionUtils.isEmpty(statuCodesList)){ if (domainId == null || CollectionUtils.isEmpty(statuCodesList)) {
return List.of(); return List.of();
} }
QueryWrapper<DataSetDO> wrapper = new QueryWrapper<>(); QueryWrapper<DataSetDO> wrapper = new QueryWrapper<>();

View File

@@ -67,8 +67,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
private ApplicationEventPublisher eventPublisher; private ApplicationEventPublisher eventPublisher;
public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService, public DimensionServiceImpl(DimensionRepository dimensionRepository, ModelService modelService,
AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService, AliasGenerateHelper aliasGenerateHelper, DatabaseService databaseService,
ModelRelaService modelRelaService, DataSetService dataSetService) { ModelRelaService modelRelaService, DataSetService dataSetService) {
this.modelService = modelService; this.modelService = modelService;
this.dimensionRepository = dimensionRepository; this.dimensionRepository = dimensionRepository;
this.aliasGenerateHelper = aliasGenerateHelper; this.aliasGenerateHelper = aliasGenerateHelper;
@@ -86,13 +86,15 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD); sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD);
// should update modelDetail // should update modelDetail
modelService.updateModelByDimAndMetric(dimensionReq.getModelId(), Lists.newArrayList(dimensionReq), null, user); modelService.updateModelByDimAndMetric(dimensionReq.getModelId(),
Lists.newArrayList(dimensionReq), null, user);
return DimensionConverter.convert2DimensionResp(dimensionDO); return DimensionConverter.convert2DimensionResp(dimensionDO);
} }
@Override @Override
public void alterDimensionBatch(List<DimensionReq> dimensionReqs, Long modelId, User user) throws Exception { public void alterDimensionBatch(List<DimensionReq> dimensionReqs, Long modelId, User user)
throws Exception {
List<DimensionResp> dimensionResps = getDimensions(modelId); List<DimensionResp> dimensionResps = getDimensions(modelId);
// get all dimension in model, only use bizname, because name can be changed to everything // get all dimension in model, only use bizname, because name can be changed to everything
Map<String, DimensionResp> bizNameMap = dimensionResps.stream() Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
@@ -117,7 +119,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
}); });
// the bizNames from alter dimensions // the bizNames from alter dimensions
List<String> bizNames = dimensionReqs.stream().map(DimensionReq::getBizName).collect(Collectors.toList()); List<String> bizNames =
dimensionReqs.stream().map(DimensionReq::getBizName).collect(Collectors.toList());
bizNameMap.keySet().forEach(bizNameInDb -> { bizNameMap.keySet().forEach(bizNameInDb -> {
if (!bizNames.contains(bizNameInDb)) { if (!bizNames.contains(bizNameInDb)) {
dimensionToDelete.add(bizNameMap.get(bizNameInDb).getId()); dimensionToDelete.add(bizNameMap.get(bizNameInDb).getId());
@@ -148,7 +151,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
.map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList()); .map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
dimensionRepository.createDimensionBatch(dimensionDOS); dimensionRepository.createDimensionBatch(dimensionDOS);
// should update modelDetail as well // should update modelDetail as well
modelService.updateModelByDimAndMetric(dimensionReqs.get(0).getModelId(), dimensionReqs, null, user); modelService.updateModelByDimAndMetric(dimensionReqs.get(0).getModelId(), dimensionReqs,
null, user);
sendEventBatch(dimensionDOS, EventType.ADD); sendEventBatch(dimensionDOS, EventType.ADD);
} }
@@ -162,7 +166,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
DimensionConverter.convert(dimensionDO, dimensionReq); DimensionConverter.convert(dimensionDO, dimensionReq);
dimensionRepository.updateDimension(dimensionDO); dimensionRepository.updateDimension(dimensionDO);
// should update modelDetail as well // should update modelDetail as well
modelService.updateModelByDimAndMetric(dimensionReq.getModelId(), Lists.newArrayList(dimensionReq), null, user); modelService.updateModelByDimAndMetric(dimensionReq.getModelId(),
Lists.newArrayList(dimensionReq), null, user);
if (!oldName.equals(dimensionDO.getName())) { if (!oldName.equals(dimensionDO.getName())) {
sendEvent(getDataItem(dimensionDO), EventType.UPDATE); sendEvent(getDataItem(dimensionDO), EventType.UPDATE);
@@ -172,10 +177,12 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
@Override @Override
public void updateDimensionBatch(List<DimensionReq> dimensionReqList, User user) { public void updateDimensionBatch(List<DimensionReq> dimensionReqList, User user) {
checkExist(dimensionReqList); checkExist(dimensionReqList);
List<DimensionDO> dimensionDOS = dimensionReqList.stream().map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList()); List<DimensionDO> dimensionDOS = dimensionReqList.stream()
.map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
dimensionRepository.batchUpdate(dimensionDOS); dimensionRepository.batchUpdate(dimensionDOS);
// should update modelDetail as well // should update modelDetail as well
modelService.updateModelByDimAndMetric(dimensionReqList.get(0).getModelId(),dimensionReqList, null, user); modelService.updateModelByDimAndMetric(dimensionReqList.get(0).getModelId(),
dimensionReqList, null, user);
sendEventBatch(dimensionDOS, EventType.UPDATE); sendEventBatch(dimensionDOS, EventType.UPDATE);
} }
@@ -231,7 +238,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
dimensionDO.setUpdatedBy(user.getName()); dimensionDO.setUpdatedBy(user.getName());
dimensionRepository.updateDimension(dimensionDO); dimensionRepository.updateDimension(dimensionDO);
// should update modelDetail // should update modelDetail
modelService.deleteModelDetailByDimAndMetric(dimensionDO.getModelId(), Lists.newArrayList(dimensionDO), null); modelService.deleteModelDetailByDimAndMetric(dimensionDO.getModelId(),
Lists.newArrayList(dimensionDO), null);
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.DELETE); sendEventBatch(Lists.newArrayList(dimensionDO), EventType.DELETE);
} }
@@ -241,7 +249,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
dimensionFilter.setDimensionIds(idList); dimensionFilter.setDimensionIds(idList);
List<DimensionDO> dimensionDOList = dimensionRepository.getDimensions(dimensionFilter); List<DimensionDO> dimensionDOList = dimensionRepository.getDimensions(dimensionFilter);
if (CollectionUtils.isEmpty(dimensionDOList)) { if (CollectionUtils.isEmpty(dimensionDOList)) {
throw new RuntimeException(String.format("the dimension %s not exist", StringUtils.join(",",idList))); throw new RuntimeException(
String.format("the dimension %s not exist", StringUtils.join(",", idList)));
} }
dimensionDOList.forEach(dimensionDO -> { dimensionDOList.forEach(dimensionDO -> {
dimensionDO.setStatus(StatusEnum.DELETED.getCode()); dimensionDO.setStatus(StatusEnum.DELETED.getCode());
@@ -250,7 +259,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
}); });
dimensionRepository.batchUpdateStatus(dimensionDOList); dimensionRepository.batchUpdateStatus(dimensionDOList);
// should update modelDetail // should update modelDetail
modelService.deleteModelDetailByDimAndMetric(dimensionDOList.get(0).getModelId(), dimensionDOList, null); modelService.deleteModelDetailByDimAndMetric(dimensionDOList.get(0).getModelId(),
dimensionDOList, null);
sendEventBatch(dimensionDOList, EventType.DELETE); sendEventBatch(dimensionDOList, EventType.DELETE);
} }
@@ -323,7 +333,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
} }
private List<DimensionResp> filterByField(List<DimensionResp> dimensionResps, private List<DimensionResp> filterByField(List<DimensionResp> dimensionResps,
List<String> fields) { List<String> fields) {
List<DimensionResp> dimensionFiltered = Lists.newArrayList(); List<DimensionResp> dimensionFiltered = Lists.newArrayList();
for (DimensionResp dimensionResp : dimensionResps) { for (DimensionResp dimensionResp : dimensionResps) {
for (String field : fields) { for (String field : fields) {
@@ -358,7 +368,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
List<DimensionResp> dimensionResps = Lists.newArrayList(); List<DimensionResp> dimensionResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(dimensionDOS)) { if (!CollectionUtils.isEmpty(dimensionDOS)) {
dimensionResps = dimensionDOS.stream().map( dimensionResps = dimensionDOS.stream().map(
dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelMap)) dimensionDO -> DimensionConverter.convert2DimensionResp(dimensionDO, modelMap))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
return dimensionResps; return dimensionResps;

View File

@@ -58,9 +58,9 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
private ChatLayerService chatLayerService; private ChatLayerService chatLayerService;
public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService, public MetricServiceImpl(MetricRepository metricRepository, ModelService modelService,
AliasGenerateHelper aliasGenerateHelper, CollectService collectService, AliasGenerateHelper aliasGenerateHelper, CollectService collectService,
DataSetService dataSetService, ApplicationEventPublisher eventPublisher, DataSetService dataSetService, ApplicationEventPublisher eventPublisher,
DimensionService dimensionService, @Lazy ChatLayerService chatLayerService) { DimensionService dimensionService, @Lazy ChatLayerService chatLayerService) {
this.metricRepository = metricRepository; this.metricRepository = metricRepository;
this.modelService = modelService; this.modelService = modelService;
this.aliasGenerateHelper = aliasGenerateHelper; this.aliasGenerateHelper = aliasGenerateHelper;
@@ -80,7 +80,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
metricRepository.createMetric(metricDO); metricRepository.createMetric(metricDO);
sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD); sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD);
// should update modelDetail as well // should update modelDetail as well
modelService.updateModelByDimAndMetric(metricReq.getModelId(),null, Lists.newArrayList(metricReq), user); modelService.updateModelByDimAndMetric(metricReq.getModelId(), null,
Lists.newArrayList(metricReq), user);
return MetricConverter.convert2MetricResp(metricDO); return MetricConverter.convert2MetricResp(metricDO);
@@ -93,7 +94,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
.map(MetricConverter::convert2MetricDO).collect(Collectors.toList()); .map(MetricConverter::convert2MetricDO).collect(Collectors.toList());
metricRepository.createMetricBatch(metricDOS); metricRepository.createMetricBatch(metricDOS);
// should update modelDetail as well // should update modelDetail as well
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs, user); modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs,
user);
sendEventBatch(metricDOS, EventType.ADD); sendEventBatch(metricDOS, EventType.ADD);
} }
@@ -123,7 +125,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}); });
// the bizNames from alter dimensions // the bizNames from alter dimensions
List<String> bizNames = metricReqs.stream().map(MetricReq::getBizName).collect(Collectors.toList()); List<String> bizNames =
metricReqs.stream().map(MetricReq::getBizName).collect(Collectors.toList());
bizNameMap.keySet().forEach(bizNameInDb -> { bizNameMap.keySet().forEach(bizNameInDb -> {
if (!bizNames.contains(bizNameInDb)) { if (!bizNames.contains(bizNameInDb)) {
metricToDelete.add(bizNameMap.get(bizNameInDb).getId()); metricToDelete.add(bizNameMap.get(bizNameInDb).getId());
@@ -163,7 +166,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
sendEvent(dataItem, EventType.UPDATE); sendEvent(dataItem, EventType.UPDATE);
} }
// should update modelDetail as well // should update modelDetail as well
modelService.updateModelByDimAndMetric(metricReq.getModelId(), null, Lists.newArrayList(metricReq), user); modelService.updateModelByDimAndMetric(metricReq.getModelId(), null,
Lists.newArrayList(metricReq), user);
return MetricConverter.convert2MetricResp(metricDO); return MetricConverter.convert2MetricResp(metricDO);
} }
@@ -171,10 +175,12 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
public void updateMetricBatch(List<MetricReq> metricReqs, User user) { public void updateMetricBatch(List<MetricReq> metricReqs, User user) {
MetricCheckUtils.checkParam(metricReqs); MetricCheckUtils.checkParam(metricReqs);
checkExist(metricReqs); checkExist(metricReqs);
List<MetricDO> metricDOS = metricReqs.stream().map(MetricConverter::convert2MetricDO).collect(Collectors.toList()); List<MetricDO> metricDOS = metricReqs.stream().map(MetricConverter::convert2MetricDO)
.collect(Collectors.toList());
metricRepository.batchUpdateMetric(metricDOS); metricRepository.batchUpdateMetric(metricDOS);
// should update modelDetail as well // should update modelDetail as well
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs, user); modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs,
user);
sendEventBatch(metricDOS, EventType.UPDATE); sendEventBatch(metricDOS, EventType.UPDATE);
} }
@@ -277,7 +283,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
metricDO.setUpdatedBy(user.getName()); metricDO.setUpdatedBy(user.getName());
metricRepository.updateMetric(metricDO); metricRepository.updateMetric(metricDO);
// should update modelDetail // should update modelDetail
modelService.deleteModelDetailByDimAndMetric(metricDO.getModelId(), null, Lists.newArrayList(metricDO)); modelService.deleteModelDetailByDimAndMetric(metricDO.getModelId(), null,
Lists.newArrayList(metricDO));
sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE); sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE);
} }
@@ -287,7 +294,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
metricsFilter.setMetricIds(idList); metricsFilter.setMetricIds(idList);
List<MetricDO> metricDOList = metricRepository.getMetrics(metricsFilter); List<MetricDO> metricDOList = metricRepository.getMetrics(metricsFilter);
if (CollectionUtils.isEmpty(metricDOList)) { if (CollectionUtils.isEmpty(metricDOList)) {
throw new RuntimeException(String.format("the metrics %s not exist", StringUtils.join(",",idList))); throw new RuntimeException(
String.format("the metrics %s not exist", StringUtils.join(",", idList)));
} }
metricDOList.forEach(metricDO -> { metricDOList.forEach(metricDO -> {
metricDO.setStatus(StatusEnum.DELETED.getCode()); metricDO.setStatus(StatusEnum.DELETED.getCode());
@@ -296,7 +304,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}); });
metricRepository.batchUpdateStatus(metricDOList); metricRepository.batchUpdateStatus(metricDOList);
// should update modelDetail // should update modelDetail
modelService.deleteModelDetailByDimAndMetric(metricDOList.get(0).getModelId(), null, metricDOList); modelService.deleteModelDetailByDimAndMetric(metricDOList.get(0).getModelId(), null,
metricDOList);
sendEventBatch(metricDOList, EventType.DELETE); sendEventBatch(metricDOList, EventType.DELETE);
} }
@@ -433,7 +442,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
} }
private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp, private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
List<String> fields, Set<MetricResp> metricRespFiltered) { List<String> fields, Set<MetricResp> metricRespFiltered) {
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) { if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics().stream() List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics().stream()
.map(MetricParam::getId).collect(Collectors.toList()); .map(MetricParam::getId).collect(Collectors.toList());
@@ -470,8 +479,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
metricFilter.setModelIds(Lists.newArrayList(modelId)); metricFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metricFilter); List<MetricResp> metricResps = getMetrics(metricFilter);
return metricResps.stream().filter( return metricResps.stream().filter(
metricResp -> MetricDefineType.FIELD.equals(metricResp.getMetricDefineType()) metricResp -> MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) || MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
@@ -645,7 +654,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter); Map<Long, ModelResp> modelMap = modelService.getModelMap(modelFilter);
if (!CollectionUtils.isEmpty(metricDOS)) { if (!CollectionUtils.isEmpty(metricDOS)) {
metricResps = metricDOS.stream().map( metricResps = metricDOS.stream().map(
metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect)) metricDO -> MetricConverter.convert2MetricResp(metricDO, modelMap, collect))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
return metricResps; return metricResps;
@@ -703,7 +712,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
@Override @Override
public void batchFillMetricDefaultAgg(List<MetricResp> metricResps, public void batchFillMetricDefaultAgg(List<MetricResp> metricResps,
List<ModelResp> modelResps) { List<ModelResp> modelResps) {
Map<Long, ModelResp> modelRespMap = Map<Long, ModelResp> modelRespMap =
modelResps.stream().collect(Collectors.toMap(ModelResp::getId, m -> m)); modelResps.stream().collect(Collectors.toMap(ModelResp::getId, m -> m));
for (MetricResp metricResp : metricResps) { for (MetricResp metricResp : metricResps) {
@@ -852,7 +861,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
} }
private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps, private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
List<DimensionResp> dimensionResps) { List<DimensionResp> dimensionResps) {
Set<Long> result = new HashSet<>(); Set<Long> result = new HashSet<>();
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) { if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
result.addAll(modelIdsByDomainId); result.addAll(modelIdsByDomainId);
@@ -889,7 +898,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
private boolean isChange(MetricReq metricReq, MetricResp metricResp) { private boolean isChange(MetricReq metricReq, MetricResp metricResp) {
boolean isNameChange = !metricReq.getName().equals(metricResp.getName()); boolean isNameChange = !metricReq.getName().equals(metricResp.getName());
boolean isBizNameChange = !Objects.equals(metricReq.getMetricDefineByMeasureParams(),metricResp.getMetricDefineByMeasureParams()); boolean isBizNameChange = !Objects.equals(metricReq.getMetricDefineByMeasureParams(),
metricResp.getMetricDefineByMeasureParams());
return isNameChange || isBizNameChange; return isNameChange || isBizNameChange;
} }
} }

View File

@@ -67,10 +67,10 @@ public class ModelServiceImpl implements ModelService {
new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService, public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
@Lazy DimensionService dimensionService, @Lazy MetricService metricService, @Lazy DimensionService dimensionService, @Lazy MetricService metricService,
DomainService domainService, UserService userService, DataSetService dataSetService, DomainService domainService, UserService userService, DataSetService dataSetService,
DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService, DateInfoRepository dateInfoRepository, ModelRelaService modelRelaService,
ApplicationEventPublisher eventPublisher) { ApplicationEventPublisher eventPublisher) {
this.modelRepository = modelRepository; this.modelRepository = modelRepository;
this.databaseService = databaseService; this.databaseService = databaseService;
this.dimensionService = dimensionService; this.dimensionService = dimensionService;
@@ -216,7 +216,7 @@ public class ModelServiceImpl implements ModelService {
} }
private void doBuild(ModelBuildReq modelBuildReq, DbSchema curSchema, List<DbSchema> dbSchemas, private void doBuild(ModelBuildReq modelBuildReq, DbSchema curSchema, List<DbSchema> dbSchemas,
Map<String, ModelSchema> modelSchemaMap) { Map<String, ModelSchema> modelSchemaMap) {
ModelSchema modelSchema = new ModelSchema(); ModelSchema modelSchema = new ModelSchema();
List<SemanticModeller> semanticModellers = CoreComponentFactory.getSemanticModellers(); List<SemanticModeller> semanticModellers = CoreComponentFactory.getSemanticModellers();
for (SemanticModeller semanticModeller : semanticModellers) { for (SemanticModeller semanticModeller : semanticModellers) {
@@ -234,7 +234,7 @@ public class ModelServiceImpl implements ModelService {
} }
private List<DbSchema> convert(Map<String, List<DBColumn>> dbColumnMap, private List<DbSchema> convert(Map<String, List<DBColumn>> dbColumnMap,
ModelBuildReq modelBuildReq) { ModelBuildReq modelBuildReq) {
return dbColumnMap.keySet().stream() return dbColumnMap.keySet().stream()
.map(key -> convert(modelBuildReq, key, dbColumnMap.get(key))) .map(key -> convert(modelBuildReq, key, dbColumnMap.get(key)))
.collect(Collectors.toList()); .collect(Collectors.toList());
@@ -383,7 +383,7 @@ public class ModelServiceImpl implements ModelService {
} }
public List<ModelResp> getModelRespAuthInheritDomain(User user, Long domainId, public List<ModelResp> getModelRespAuthInheritDomain(User user, Long domainId,
AuthType authType) { AuthType authType) {
List<Long> domainIds = List<Long> domainIds =
domainService.getDomainAuthSet(user, authType).stream().filter(domainResp -> { domainService.getDomainAuthSet(user, authType).stream().filter(domainResp -> {
if (domainId == null) { if (domainId == null) {
@@ -521,13 +521,14 @@ public class ModelServiceImpl implements ModelService {
} }
@Override @Override
public void updateModelByDimAndMetric(Long modelId, List<DimensionReq> dimensionReqList, List<MetricReq> metricReqList, User user) { public void updateModelByDimAndMetric(Long modelId, List<DimensionReq> dimensionReqList,
List<MetricReq> metricReqList, User user) {
ModelDO modelDO = getModelDO(modelId); ModelDO modelDO = getModelDO(modelId);
ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class); ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class);
if (!CollectionUtils.isEmpty(dimensionReqList)) { if (!CollectionUtils.isEmpty(dimensionReqList)) {
dimensionReqList.forEach(dimensionReq -> { dimensionReqList.forEach(dimensionReq -> {
Optional<Dimension> dimOptional = modelDetail.getDimensions().stream() Optional<Dimension> dimOptional = modelDetail.getDimensions().stream().filter(
.filter(dimension -> dimension.getBizName().equals(dimensionReq.getBizName())) dimension -> dimension.getBizName().equals(dimensionReq.getBizName()))
.findFirst(); .findFirst();
if (dimOptional.isPresent()) { if (dimOptional.isPresent()) {
Dimension dimension = dimOptional.get(); Dimension dimension = dimOptional.get();
@@ -547,10 +548,13 @@ public class ModelServiceImpl implements ModelService {
if (!CollectionUtils.isEmpty(metricReqList)) { if (!CollectionUtils.isEmpty(metricReqList)) {
// 目前modeltail中的measure // 目前modeltail中的measure
Map<String, Measure> mesureMap = modelDetail.getMeasures().stream().collect(Collectors.toMap(Measure::getBizName, a -> a, (k1, k2) -> k1)); Map<String, Measure> mesureMap = modelDetail.getMeasures().stream()
.collect(Collectors.toMap(Measure::getBizName, a -> a, (k1, k2) -> k1));
metricReqList.forEach(metricReq -> { metricReqList.forEach(metricReq -> {
if (null != metricReq.getMetricDefineByMeasureParams() && !CollectionUtils.isEmpty(metricReq.getMetricDefineByMeasureParams().getMeasures())) { if (null != metricReq.getMetricDefineByMeasureParams() && !CollectionUtils
for(Measure alterMeasure : metricReq.getMetricDefineByMeasureParams().getMeasures()) { .isEmpty(metricReq.getMetricDefineByMeasureParams().getMeasures())) {
for (Measure alterMeasure : metricReq.getMetricDefineByMeasureParams()
.getMeasures()) {
if (mesureMap.containsKey(alterMeasure.getBizName())) { if (mesureMap.containsKey(alterMeasure.getBizName())) {
Measure measure = mesureMap.get(alterMeasure.getBizName()); Measure measure = mesureMap.get(alterMeasure.getBizName());
BeanUtils.copyProperties(alterMeasure, measure); BeanUtils.copyProperties(alterMeasure, measure);
@@ -569,13 +573,14 @@ public class ModelServiceImpl implements ModelService {
} }
@Override @Override
public void deleteModelDetailByDimAndMetric(Long modelId, List<DimensionDO> dimensionList, List<MetricDO> metricReqList) { public void deleteModelDetailByDimAndMetric(Long modelId, List<DimensionDO> dimensionList,
List<MetricDO> metricReqList) {
ModelDO modelDO = getModelDO(modelId); ModelDO modelDO = getModelDO(modelId);
ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class); ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class);
if (!CollectionUtils.isEmpty(dimensionList)) { if (!CollectionUtils.isEmpty(dimensionList)) {
dimensionList.forEach(dimensionReq -> { dimensionList.forEach(dimensionReq -> {
Optional<Dimension> dimOptional = modelDetail.getDimensions().stream() Optional<Dimension> dimOptional = modelDetail.getDimensions().stream().filter(
.filter(dimension -> dimension.getBizName().equals(dimensionReq.getBizName())) dimension -> dimension.getBizName().equals(dimensionReq.getBizName()))
.findFirst(); .findFirst();
if (dimOptional.isPresent()) { if (dimOptional.isPresent()) {
Dimension dimension = dimOptional.get(); Dimension dimension = dimOptional.get();
@@ -638,7 +643,7 @@ public class ModelServiceImpl implements ModelService {
} }
public static boolean checkDataSetPermission(Set<String> orgIds, User user, public static boolean checkDataSetPermission(Set<String> orgIds, User user,
ModelResp modelResp) { ModelResp modelResp) {
if (checkAdminPermission(orgIds, user, modelResp)) { if (checkAdminPermission(orgIds, user, modelResp)) {
return true; return true;
} }

View File

@@ -198,8 +198,7 @@ public class DataSetSchemaBuilder {
}).collect(Collectors.toList()); }).collect(Collectors.toList());
} }
private static void setDefaultTimeFormat(SchemaElement dimToAdd, private static void setDefaultTimeFormat(SchemaElement dimToAdd, String timeFormat) {
String timeFormat) {
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat); dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
} }
} }

View File

@@ -56,7 +56,8 @@ public class MetricCheckUtils {
if (StringUtils.isBlank(expr)) { if (StringUtils.isBlank(expr)) {
throw new InvalidArgumentException("表达式不可为空"); throw new InvalidArgumentException("表达式不可为空");
} }
String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(metricReq.getName()); String forbiddenCharacters =
NameCheckUtils.findForbiddenCharacters(metricReq.getName());
if (StringUtils.isNotBlank(forbiddenCharacters)) { if (StringUtils.isNotBlank(forbiddenCharacters)) {
throw new InvalidArgumentException( throw new InvalidArgumentException(
String.format("名称包含特殊字符%s, 请修改", forbiddenCharacters)); String.format("名称包含特殊字符%s, 请修改", forbiddenCharacters));

View File

@@ -143,7 +143,7 @@ public class ModelConverter {
} }
public static ModelReq convert(ModelSchema modelSchema, ModelBuildReq modelBuildReq, public static ModelReq convert(ModelSchema modelSchema, ModelBuildReq modelBuildReq,
String tableName) { String tableName) {
ModelReq modelReq = new ModelReq(); ModelReq modelReq = new ModelReq();
modelReq.setName(modelBuildReq.getName() != null ? modelBuildReq.getName() : tableName); modelReq.setName(modelBuildReq.getName() != null ? modelBuildReq.getName() : tableName);
modelReq.setBizName( modelReq.setBizName(
@@ -169,7 +169,7 @@ public class ModelConverter {
if (getIdentifyType(fieldType) != null) { if (getIdentifyType(fieldType) != null) {
Optional<Identify> optional = modelDetail.getIdentifiers().stream().filter( Optional<Identify> optional = modelDetail.getIdentifiers().stream().filter(
identify -> identify.getBizName().equals(semanticColumn.getColumnName())) identify -> identify.getBizName().equals(semanticColumn.getColumnName()))
.findAny(); .findAny();
if (optional.isEmpty()) { if (optional.isEmpty()) {
Identify identify = new Identify(semanticColumn.getName(), Identify identify = new Identify(semanticColumn.getName(),
@@ -178,7 +178,7 @@ public class ModelConverter {
} }
} else if (FieldType.measure.equals(fieldType)) { } else if (FieldType.measure.equals(fieldType)) {
Optional<Measure> optional = modelDetail.getMeasures().stream().filter( Optional<Measure> optional = modelDetail.getMeasures().stream().filter(
measure -> measure.getBizName().equals(semanticColumn.getColumnName())) measure -> measure.getBizName().equals(semanticColumn.getColumnName()))
.findAny(); .findAny();
if (optional.isEmpty()) { if (optional.isEmpty()) {
Measure measure = new Measure(semanticColumn.getName(), Measure measure = new Measure(semanticColumn.getName(),
@@ -188,7 +188,7 @@ public class ModelConverter {
} }
} else { } else {
Optional<Dimension> optional = modelDetail.getDimensions().stream().filter( Optional<Dimension> optional = modelDetail.getDimensions().stream().filter(
dimension -> dimension.getBizName().equals(semanticColumn.getColumnName())) dimension -> dimension.getBizName().equals(semanticColumn.getColumnName()))
.findAny(); .findAny();
if (optional.isEmpty()) { if (optional.isEmpty()) {
Dimension dim = new Dimension(semanticColumn.getName(), Dimension dim = new Dimension(semanticColumn.getName(),
@@ -294,7 +294,8 @@ public class ModelConverter {
List<Dimension> dimensions = modelReq.getModelDetail().getDimensions(); List<Dimension> dimensions = modelReq.getModelDetail().getDimensions();
List<Identify> identifiers = modelReq.getModelDetail().getIdentifiers(); List<Identify> identifiers = modelReq.getModelDetail().getIdentifiers();
List<Field> fields = modelReq.getModelDetail().getFields(); List<Field> fields = modelReq.getModelDetail().getFields();
List<String> fieldNames = fields.stream().map(Field::getFieldName).collect(Collectors.toList()); List<String> fieldNames =
fields.stream().map(Field::getFieldName).collect(Collectors.toList());
if (measures != null) { if (measures != null) {
for (Measure measure : measures) { for (Measure measure : measures) {
@@ -302,7 +303,8 @@ public class ModelConverter {
&& StringUtils.isBlank(measure.getExpr())) { && StringUtils.isBlank(measure.getExpr())) {
measure.setExpr(measure.getBizName()); measure.setExpr(measure.getBizName());
} }
if (StringUtils.isNotBlank(measure.getBizName()) && !fieldNames.contains(measure.getBizName())) { if (StringUtils.isNotBlank(measure.getBizName())
&& !fieldNames.contains(measure.getBizName())) {
fields.add(new Field(measure.getBizName(), "")); fields.add(new Field(measure.getBizName(), ""));
} }
} }
@@ -313,18 +315,21 @@ public class ModelConverter {
&& StringUtils.isBlank(dimension.getExpr())) { && StringUtils.isBlank(dimension.getExpr())) {
dimension.setExpr(dimension.getBizName()); dimension.setExpr(dimension.getBizName());
} }
if (StringUtils.isNotBlank(dimension.getBizName()) && !fieldNames.contains(dimension.getBizName())) { if (StringUtils.isNotBlank(dimension.getBizName())
&& !fieldNames.contains(dimension.getBizName())) {
fields.add(new Field(dimension.getBizName(), "")); fields.add(new Field(dimension.getBizName(), ""));
} }
} }
} }
if (identifiers != null) { if (identifiers != null) {
for (Identify identify : identifiers) { for (Identify identify : identifiers) {
if (StringUtils.isNotBlank(identify.getBizName()) && StringUtils.isBlank(identify.getName())) { if (StringUtils.isNotBlank(identify.getBizName())
&& StringUtils.isBlank(identify.getName())) {
identify.setName(identify.getBizName()); identify.setName(identify.getBizName());
} }
identify.setIsCreateDimension(1); identify.setIsCreateDimension(1);
if (StringUtils.isNotBlank(identify.getBizName()) && !fieldNames.contains(identify.getBizName())) { if (StringUtils.isNotBlank(identify.getBizName())
&& !fieldNames.contains(identify.getBizName())) {
fields.add(new Field(identify.getBizName(), "")); fields.add(new Field(identify.getBizName(), ""));
} }
} }