mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-12 12:37:55 +00:00
[fix]Fix unit test cases.
This commit is contained in:
@@ -19,7 +19,8 @@ public class ParseContext {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public boolean enableNL2SQL() {
|
public boolean enableNL2SQL() {
|
||||||
return Objects.nonNull(agent) && agent.containsDatasetTool()&&response.getSelectedParses().size() == 0;
|
return Objects.nonNull(agent) && agent.containsDatasetTool()
|
||||||
|
&& response.getSelectedParses().size() == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean enableLLM() {
|
public boolean enableLLM() {
|
||||||
|
|||||||
@@ -727,7 +727,7 @@ public class SqlReplaceHelper {
|
|||||||
List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
|
List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
|
||||||
for (PlainSelect plainSelect : plainSelects) {
|
for (PlainSelect plainSelect : plainSelects) {
|
||||||
if (Objects.nonNull(plainSelect.getFromItem())) {
|
if (Objects.nonNull(plainSelect.getFromItem())) {
|
||||||
Table table = (Table) plainSelect.getFromItem();
|
Table table = SqlSelectHelper.getTable(plainSelect.getFromItem());
|
||||||
if (table.getName().equals(tableName)) {
|
if (table.getName().equals(tableName)) {
|
||||||
replacePlainSelectByExpr(plainSelect, replace);
|
replacePlainSelectByExpr(plainSelect, replace);
|
||||||
if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
|
if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
|
||||||
|
|||||||
@@ -723,6 +723,44 @@ public class SqlSelectHelper {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Table getTable(FromItem fromItem) {
|
||||||
|
Table table = null;
|
||||||
|
if (fromItem instanceof Table) {
|
||||||
|
table = (Table) fromItem;
|
||||||
|
} else if (fromItem instanceof ParenthesedSelect) {
|
||||||
|
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) fromItem;
|
||||||
|
if (parenthesedSelect.getSelect() instanceof PlainSelect) {
|
||||||
|
PlainSelect subSelect = (PlainSelect) parenthesedSelect.getSelect();
|
||||||
|
table = getTable(subSelect.getSelectBody());
|
||||||
|
} else if (parenthesedSelect.getSelect() instanceof SetOperationList) {
|
||||||
|
table = getTable(parenthesedSelect.getSelect());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Table getTable(Select select) {
|
||||||
|
if (select == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
List<PlainSelect> plainSelectList = getWithItem(select);
|
||||||
|
if (!CollectionUtils.isEmpty(plainSelectList)) {
|
||||||
|
List<PlainSelect> selectList = new ArrayList<>(plainSelectList);
|
||||||
|
Table table = getTable(selectList.get(0));
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
if (select instanceof PlainSelect) {
|
||||||
|
PlainSelect plainSelect = (PlainSelect) select;
|
||||||
|
return getTable(plainSelect.getFromItem());
|
||||||
|
} else if (select instanceof SetOperationList) {
|
||||||
|
SetOperationList setOperationList = (SetOperationList) select;
|
||||||
|
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
|
||||||
|
return getTable(setOperationList.getSelects().get(0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
public static String getDbTableName(String sql) {
|
public static String getDbTableName(String sql) {
|
||||||
Table table = getTable(sql);
|
Table table = getTable(sql);
|
||||||
return table.getFullyQualifiedName();
|
return table.getFullyQualifiedName();
|
||||||
|
|||||||
@@ -3,7 +3,11 @@ package com.tencent.supersonic.headless.api.pojo.request;
|
|||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
|
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
|
||||||
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
|
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
|
||||||
import com.tencent.supersonic.common.pojo.*;
|
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||||
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
|
import com.tencent.supersonic.common.pojo.Filter;
|
||||||
|
import com.tencent.supersonic.common.pojo.Order;
|
||||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
@@ -21,14 +25,22 @@ import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
|
|||||||
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
||||||
import net.sf.jsqlparser.schema.Column;
|
import net.sf.jsqlparser.schema.Column;
|
||||||
import net.sf.jsqlparser.schema.Table;
|
import net.sf.jsqlparser.schema.Table;
|
||||||
import net.sf.jsqlparser.statement.select.*;
|
import net.sf.jsqlparser.statement.select.GroupByElement;
|
||||||
|
import net.sf.jsqlparser.statement.select.Limit;
|
||||||
|
import net.sf.jsqlparser.statement.select.Offset;
|
||||||
|
import net.sf.jsqlparser.statement.select.OrderByElement;
|
||||||
|
import net.sf.jsqlparser.statement.select.ParenthesedSelect;
|
||||||
|
import net.sf.jsqlparser.statement.select.PlainSelect;
|
||||||
|
import net.sf.jsqlparser.statement.select.SelectItem;
|
||||||
import org.apache.commons.codec.digest.DigestUtils;
|
import org.apache.commons.codec.digest.DigestUtils;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@@ -176,7 +188,7 @@ public class QueryStructReq extends SemanticQueryReq {
|
|||||||
|
|
||||||
private List<SelectItem<?>> buildSelectItems(QueryStructReq queryStructReq) {
|
private List<SelectItem<?>> buildSelectItems(QueryStructReq queryStructReq) {
|
||||||
List<SelectItem<?>> selectItems = new ArrayList<>();
|
List<SelectItem<?>> selectItems = new ArrayList<>();
|
||||||
List<String> groups = queryStructReq.getGroups();
|
Set<String> groups = new HashSet<>(queryStructReq.getGroups());
|
||||||
|
|
||||||
if (!CollectionUtils.isEmpty(groups)) {
|
if (!CollectionUtils.isEmpty(groups)) {
|
||||||
for (String group : groups) {
|
for (String group : groups) {
|
||||||
@@ -236,7 +248,7 @@ public class QueryStructReq extends SemanticQueryReq {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private GroupByElement buildGroupByElement(QueryStructReq queryStructReq) {
|
private GroupByElement buildGroupByElement(QueryStructReq queryStructReq) {
|
||||||
List<String> groups = queryStructReq.getGroups();
|
Set<String> groups = new HashSet<>(queryStructReq.getGroups());
|
||||||
if ((!CollectionUtils.isEmpty(groups) && !queryStructReq.getAggregators().isEmpty())
|
if ((!CollectionUtils.isEmpty(groups) && !queryStructReq.getAggregators().isEmpty())
|
||||||
|| !queryStructReq.getMetricFilters().isEmpty()) {
|
|| !queryStructReq.getMetricFilters().isEmpty()) {
|
||||||
GroupByElement groupByElement = new GroupByElement();
|
GroupByElement groupByElement = new GroupByElement();
|
||||||
|
|||||||
@@ -169,9 +169,10 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
|
|||||||
|
|
||||||
Prompt prompt = PromptTemplate.from(LLM_FILTER_PROMPT).apply(variable);
|
Prompt prompt = PromptTemplate.from(LLM_FILTER_PROMPT).apply(variable);
|
||||||
ChatModelConfig chatModelConfig = null;
|
ChatModelConfig chatModelConfig = null;
|
||||||
if(chatQueryContext.getRequest().getChatAppConfig()!=null
|
if (chatQueryContext.getRequest().getChatAppConfig() != null && chatQueryContext
|
||||||
&& chatQueryContext.getRequest().getChatAppConfig().containsKey("REWRITE_MULTI_TURN")){
|
.getRequest().getChatAppConfig().containsKey("REWRITE_MULTI_TURN")) {
|
||||||
chatModelConfig=chatQueryContext.getRequest().getChatAppConfig().get("REWRITE_MULTI_TURN").getChatModelConfig();
|
chatModelConfig = chatQueryContext.getRequest().getChatAppConfig()
|
||||||
|
.get("REWRITE_MULTI_TURN").getChatModelConfig();
|
||||||
}
|
}
|
||||||
ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatModelConfig);
|
ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel(chatModelConfig);
|
||||||
String response = chatLanguageModel.generate(prompt.toUserMessage().singleText());
|
String response = chatLanguageModel.generate(prompt.toUserMessage().singleText());
|
||||||
|
|||||||
@@ -87,7 +87,8 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
.similarity(hanlpMapResult.getSimilarity())
|
.similarity(hanlpMapResult.getSimilarity())
|
||||||
.detectWord(hanlpMapResult.getDetectWord()).build();
|
.detectWord(hanlpMapResult.getDetectWord()).build();
|
||||||
// doDimValueAliasLogic 将维度值别名进行替换成真实维度值
|
// doDimValueAliasLogic 将维度值别名进行替换成真实维度值
|
||||||
doDimValueAliasLogic(schemaElementMatch,chatQueryContext.getSemanticSchema().getDimensionValues());
|
doDimValueAliasLogic(schemaElementMatch,
|
||||||
|
chatQueryContext.getSemanticSchema().getDimensionValues());
|
||||||
addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch);
|
addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -124,7 +124,7 @@ public class DimValueAspect {
|
|||||||
sql = SqlReplaceHelper.replaceValue(sql, filedNameToValueMap);
|
sql = SqlReplaceHelper.replaceValue(sql, filedNameToValueMap);
|
||||||
log.debug("correctorSql after replacing:{}", sql);
|
log.debug("correctorSql after replacing:{}", sql);
|
||||||
querySqlReq.setSql(sql);
|
querySqlReq.setSql(sql);
|
||||||
querySqlReq.getSqlInfo().setQuerySQL(sql);
|
// querySqlReq.getSqlInfo().setQuerySQL(sql);
|
||||||
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
|
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
|
||||||
|
|
||||||
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
|
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
|
||||||
|
|||||||
@@ -114,14 +114,9 @@ public class MetricRepositoryImpl implements MetricRepository {
|
|||||||
}
|
}
|
||||||
if (StringUtils.isNotBlank(metricFilter.getKey())) {
|
if (StringUtils.isNotBlank(metricFilter.getKey())) {
|
||||||
String key = metricFilter.getKey();
|
String key = metricFilter.getKey();
|
||||||
queryWrapper.lambda()
|
queryWrapper.lambda().and(wrapper -> wrapper.like(MetricDO::getName, key).or()
|
||||||
.and(wrapper -> wrapper
|
.like(MetricDO::getBizName, key).or().like(MetricDO::getDescription, key).or()
|
||||||
.like(MetricDO::getName, key)
|
.like(MetricDO::getAlias, key).or().like(MetricDO::getCreatedBy, key));
|
||||||
.or().like(MetricDO::getBizName, key)
|
|
||||||
.or().like(MetricDO::getDescription, key)
|
|
||||||
.or().like(MetricDO::getAlias, key)
|
|
||||||
.or().like(MetricDO::getCreatedBy, key)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return metricDOMapper.selectList(queryWrapper);
|
return metricDOMapper.selectList(queryWrapper);
|
||||||
|
|||||||
@@ -51,7 +51,8 @@ public class DataSetController {
|
|||||||
|
|
||||||
@GetMapping("/getDataSetList")
|
@GetMapping("/getDataSetList")
|
||||||
public List<DataSetResp> getDataSetList(@RequestParam("domainId") Long domainId) {
|
public List<DataSetResp> getDataSetList(@RequestParam("domainId") Long domainId) {
|
||||||
List<Integer> statuCodeList = Arrays.asList(StatusEnum.ONLINE.getCode(),StatusEnum.OFFLINE.getCode());
|
List<Integer> statuCodeList =
|
||||||
|
Arrays.asList(StatusEnum.ONLINE.getCode(), StatusEnum.OFFLINE.getCode());
|
||||||
return dataSetService.getDataSetList(domainId, statuCodeList);
|
return dataSetService.getDataSetList(domainId, statuCodeList);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,8 @@ public interface DimensionService {
|
|||||||
|
|
||||||
DimensionResp createDimension(DimensionReq dimensionReq, User user) throws Exception;
|
DimensionResp createDimension(DimensionReq dimensionReq, User user) throws Exception;
|
||||||
|
|
||||||
void alterDimensionBatch(List<DimensionReq> dimensionReqs, Long modelId, User user) throws Exception;
|
void alterDimensionBatch(List<DimensionReq> dimensionReqs, Long modelId, User user)
|
||||||
|
throws Exception;
|
||||||
|
|
||||||
void createDimensionBatch(List<DimensionReq> dimensionReqs, User user) throws Exception;
|
void createDimensionBatch(List<DimensionReq> dimensionReqs, User user) throws Exception;
|
||||||
|
|
||||||
|
|||||||
@@ -55,7 +55,9 @@ public interface ModelService {
|
|||||||
|
|
||||||
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
|
void batchUpdateStatus(MetaBatchReq metaBatchReq, User user);
|
||||||
|
|
||||||
void updateModelByDimAndMetric(Long modelId, List<DimensionReq> dimensionReqList, List<MetricReq> metricReqList, User user);
|
void updateModelByDimAndMetric(Long modelId, List<DimensionReq> dimensionReqList,
|
||||||
|
List<MetricReq> metricReqList, User user);
|
||||||
|
|
||||||
void deleteModelDetailByDimAndMetric(Long modelId, List<DimensionDO> dimensionReqList, List<MetricDO> metricReqList);
|
void deleteModelDetailByDimAndMetric(Long modelId, List<DimensionDO> dimensionReqList,
|
||||||
|
List<MetricDO> metricReqList);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,13 +86,15 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD);
|
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.ADD);
|
||||||
|
|
||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.updateModelByDimAndMetric(dimensionReq.getModelId(), Lists.newArrayList(dimensionReq), null, user);
|
modelService.updateModelByDimAndMetric(dimensionReq.getModelId(),
|
||||||
|
Lists.newArrayList(dimensionReq), null, user);
|
||||||
|
|
||||||
return DimensionConverter.convert2DimensionResp(dimensionDO);
|
return DimensionConverter.convert2DimensionResp(dimensionDO);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void alterDimensionBatch(List<DimensionReq> dimensionReqs, Long modelId, User user) throws Exception {
|
public void alterDimensionBatch(List<DimensionReq> dimensionReqs, Long modelId, User user)
|
||||||
|
throws Exception {
|
||||||
List<DimensionResp> dimensionResps = getDimensions(modelId);
|
List<DimensionResp> dimensionResps = getDimensions(modelId);
|
||||||
// get all dimension in model, only use bizname, because name can be changed to everything
|
// get all dimension in model, only use bizname, because name can be changed to everything
|
||||||
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
|
Map<String, DimensionResp> bizNameMap = dimensionResps.stream()
|
||||||
@@ -117,7 +119,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
});
|
});
|
||||||
|
|
||||||
// the bizNames from alter dimensions
|
// the bizNames from alter dimensions
|
||||||
List<String> bizNames = dimensionReqs.stream().map(DimensionReq::getBizName).collect(Collectors.toList());
|
List<String> bizNames =
|
||||||
|
dimensionReqs.stream().map(DimensionReq::getBizName).collect(Collectors.toList());
|
||||||
bizNameMap.keySet().forEach(bizNameInDb -> {
|
bizNameMap.keySet().forEach(bizNameInDb -> {
|
||||||
if (!bizNames.contains(bizNameInDb)) {
|
if (!bizNames.contains(bizNameInDb)) {
|
||||||
dimensionToDelete.add(bizNameMap.get(bizNameInDb).getId());
|
dimensionToDelete.add(bizNameMap.get(bizNameInDb).getId());
|
||||||
@@ -148,7 +151,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
.map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
|
.map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
|
||||||
dimensionRepository.createDimensionBatch(dimensionDOS);
|
dimensionRepository.createDimensionBatch(dimensionDOS);
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(dimensionReqs.get(0).getModelId(), dimensionReqs, null, user);
|
modelService.updateModelByDimAndMetric(dimensionReqs.get(0).getModelId(), dimensionReqs,
|
||||||
|
null, user);
|
||||||
|
|
||||||
sendEventBatch(dimensionDOS, EventType.ADD);
|
sendEventBatch(dimensionDOS, EventType.ADD);
|
||||||
}
|
}
|
||||||
@@ -162,7 +166,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
DimensionConverter.convert(dimensionDO, dimensionReq);
|
DimensionConverter.convert(dimensionDO, dimensionReq);
|
||||||
dimensionRepository.updateDimension(dimensionDO);
|
dimensionRepository.updateDimension(dimensionDO);
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(dimensionReq.getModelId(), Lists.newArrayList(dimensionReq), null, user);
|
modelService.updateModelByDimAndMetric(dimensionReq.getModelId(),
|
||||||
|
Lists.newArrayList(dimensionReq), null, user);
|
||||||
|
|
||||||
if (!oldName.equals(dimensionDO.getName())) {
|
if (!oldName.equals(dimensionDO.getName())) {
|
||||||
sendEvent(getDataItem(dimensionDO), EventType.UPDATE);
|
sendEvent(getDataItem(dimensionDO), EventType.UPDATE);
|
||||||
@@ -172,10 +177,12 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
@Override
|
@Override
|
||||||
public void updateDimensionBatch(List<DimensionReq> dimensionReqList, User user) {
|
public void updateDimensionBatch(List<DimensionReq> dimensionReqList, User user) {
|
||||||
checkExist(dimensionReqList);
|
checkExist(dimensionReqList);
|
||||||
List<DimensionDO> dimensionDOS = dimensionReqList.stream().map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
|
List<DimensionDO> dimensionDOS = dimensionReqList.stream()
|
||||||
|
.map(DimensionConverter::convert2DimensionDO).collect(Collectors.toList());
|
||||||
dimensionRepository.batchUpdate(dimensionDOS);
|
dimensionRepository.batchUpdate(dimensionDOS);
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(dimensionReqList.get(0).getModelId(),dimensionReqList, null, user);
|
modelService.updateModelByDimAndMetric(dimensionReqList.get(0).getModelId(),
|
||||||
|
dimensionReqList, null, user);
|
||||||
sendEventBatch(dimensionDOS, EventType.UPDATE);
|
sendEventBatch(dimensionDOS, EventType.UPDATE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -231,7 +238,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
dimensionDO.setUpdatedBy(user.getName());
|
dimensionDO.setUpdatedBy(user.getName());
|
||||||
dimensionRepository.updateDimension(dimensionDO);
|
dimensionRepository.updateDimension(dimensionDO);
|
||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.deleteModelDetailByDimAndMetric(dimensionDO.getModelId(), Lists.newArrayList(dimensionDO), null);
|
modelService.deleteModelDetailByDimAndMetric(dimensionDO.getModelId(),
|
||||||
|
Lists.newArrayList(dimensionDO), null);
|
||||||
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.DELETE);
|
sendEventBatch(Lists.newArrayList(dimensionDO), EventType.DELETE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -241,7 +249,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
dimensionFilter.setDimensionIds(idList);
|
dimensionFilter.setDimensionIds(idList);
|
||||||
List<DimensionDO> dimensionDOList = dimensionRepository.getDimensions(dimensionFilter);
|
List<DimensionDO> dimensionDOList = dimensionRepository.getDimensions(dimensionFilter);
|
||||||
if (CollectionUtils.isEmpty(dimensionDOList)) {
|
if (CollectionUtils.isEmpty(dimensionDOList)) {
|
||||||
throw new RuntimeException(String.format("the dimension %s not exist", StringUtils.join(",",idList)));
|
throw new RuntimeException(
|
||||||
|
String.format("the dimension %s not exist", StringUtils.join(",", idList)));
|
||||||
}
|
}
|
||||||
dimensionDOList.forEach(dimensionDO -> {
|
dimensionDOList.forEach(dimensionDO -> {
|
||||||
dimensionDO.setStatus(StatusEnum.DELETED.getCode());
|
dimensionDO.setStatus(StatusEnum.DELETED.getCode());
|
||||||
@@ -250,7 +259,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
|
|||||||
});
|
});
|
||||||
dimensionRepository.batchUpdateStatus(dimensionDOList);
|
dimensionRepository.batchUpdateStatus(dimensionDOList);
|
||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.deleteModelDetailByDimAndMetric(dimensionDOList.get(0).getModelId(), dimensionDOList, null);
|
modelService.deleteModelDetailByDimAndMetric(dimensionDOList.get(0).getModelId(),
|
||||||
|
dimensionDOList, null);
|
||||||
sendEventBatch(dimensionDOList, EventType.DELETE);
|
sendEventBatch(dimensionDOList, EventType.DELETE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -80,7 +80,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
metricRepository.createMetric(metricDO);
|
metricRepository.createMetric(metricDO);
|
||||||
sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD);
|
sendEventBatch(Lists.newArrayList(metricDO), EventType.ADD);
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(metricReq.getModelId(),null, Lists.newArrayList(metricReq), user);
|
modelService.updateModelByDimAndMetric(metricReq.getModelId(), null,
|
||||||
|
Lists.newArrayList(metricReq), user);
|
||||||
|
|
||||||
|
|
||||||
return MetricConverter.convert2MetricResp(metricDO);
|
return MetricConverter.convert2MetricResp(metricDO);
|
||||||
@@ -93,7 +94,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
.map(MetricConverter::convert2MetricDO).collect(Collectors.toList());
|
.map(MetricConverter::convert2MetricDO).collect(Collectors.toList());
|
||||||
metricRepository.createMetricBatch(metricDOS);
|
metricRepository.createMetricBatch(metricDOS);
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs, user);
|
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs,
|
||||||
|
user);
|
||||||
|
|
||||||
sendEventBatch(metricDOS, EventType.ADD);
|
sendEventBatch(metricDOS, EventType.ADD);
|
||||||
}
|
}
|
||||||
@@ -123,7 +125,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
});
|
});
|
||||||
|
|
||||||
// the bizNames from alter dimensions
|
// the bizNames from alter dimensions
|
||||||
List<String> bizNames = metricReqs.stream().map(MetricReq::getBizName).collect(Collectors.toList());
|
List<String> bizNames =
|
||||||
|
metricReqs.stream().map(MetricReq::getBizName).collect(Collectors.toList());
|
||||||
bizNameMap.keySet().forEach(bizNameInDb -> {
|
bizNameMap.keySet().forEach(bizNameInDb -> {
|
||||||
if (!bizNames.contains(bizNameInDb)) {
|
if (!bizNames.contains(bizNameInDb)) {
|
||||||
metricToDelete.add(bizNameMap.get(bizNameInDb).getId());
|
metricToDelete.add(bizNameMap.get(bizNameInDb).getId());
|
||||||
@@ -163,7 +166,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
sendEvent(dataItem, EventType.UPDATE);
|
sendEvent(dataItem, EventType.UPDATE);
|
||||||
}
|
}
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(metricReq.getModelId(), null, Lists.newArrayList(metricReq), user);
|
modelService.updateModelByDimAndMetric(metricReq.getModelId(), null,
|
||||||
|
Lists.newArrayList(metricReq), user);
|
||||||
return MetricConverter.convert2MetricResp(metricDO);
|
return MetricConverter.convert2MetricResp(metricDO);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,10 +175,12 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
public void updateMetricBatch(List<MetricReq> metricReqs, User user) {
|
public void updateMetricBatch(List<MetricReq> metricReqs, User user) {
|
||||||
MetricCheckUtils.checkParam(metricReqs);
|
MetricCheckUtils.checkParam(metricReqs);
|
||||||
checkExist(metricReqs);
|
checkExist(metricReqs);
|
||||||
List<MetricDO> metricDOS = metricReqs.stream().map(MetricConverter::convert2MetricDO).collect(Collectors.toList());
|
List<MetricDO> metricDOS = metricReqs.stream().map(MetricConverter::convert2MetricDO)
|
||||||
|
.collect(Collectors.toList());
|
||||||
metricRepository.batchUpdateMetric(metricDOS);
|
metricRepository.batchUpdateMetric(metricDOS);
|
||||||
// should update modelDetail as well
|
// should update modelDetail as well
|
||||||
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs, user);
|
modelService.updateModelByDimAndMetric(metricReqs.get(0).getModelId(), null, metricReqs,
|
||||||
|
user);
|
||||||
sendEventBatch(metricDOS, EventType.UPDATE);
|
sendEventBatch(metricDOS, EventType.UPDATE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -277,7 +283,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
metricDO.setUpdatedBy(user.getName());
|
metricDO.setUpdatedBy(user.getName());
|
||||||
metricRepository.updateMetric(metricDO);
|
metricRepository.updateMetric(metricDO);
|
||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.deleteModelDetailByDimAndMetric(metricDO.getModelId(), null, Lists.newArrayList(metricDO));
|
modelService.deleteModelDetailByDimAndMetric(metricDO.getModelId(), null,
|
||||||
|
Lists.newArrayList(metricDO));
|
||||||
sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE);
|
sendEventBatch(Lists.newArrayList(metricDO), EventType.DELETE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -287,7 +294,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
metricsFilter.setMetricIds(idList);
|
metricsFilter.setMetricIds(idList);
|
||||||
List<MetricDO> metricDOList = metricRepository.getMetrics(metricsFilter);
|
List<MetricDO> metricDOList = metricRepository.getMetrics(metricsFilter);
|
||||||
if (CollectionUtils.isEmpty(metricDOList)) {
|
if (CollectionUtils.isEmpty(metricDOList)) {
|
||||||
throw new RuntimeException(String.format("the metrics %s not exist", StringUtils.join(",",idList)));
|
throw new RuntimeException(
|
||||||
|
String.format("the metrics %s not exist", StringUtils.join(",", idList)));
|
||||||
}
|
}
|
||||||
metricDOList.forEach(metricDO -> {
|
metricDOList.forEach(metricDO -> {
|
||||||
metricDO.setStatus(StatusEnum.DELETED.getCode());
|
metricDO.setStatus(StatusEnum.DELETED.getCode());
|
||||||
@@ -296,7 +304,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
});
|
});
|
||||||
metricRepository.batchUpdateStatus(metricDOList);
|
metricRepository.batchUpdateStatus(metricDOList);
|
||||||
// should update modelDetail
|
// should update modelDetail
|
||||||
modelService.deleteModelDetailByDimAndMetric(metricDOList.get(0).getModelId(), null, metricDOList);
|
modelService.deleteModelDetailByDimAndMetric(metricDOList.get(0).getModelId(), null,
|
||||||
|
metricDOList);
|
||||||
sendEventBatch(metricDOList, EventType.DELETE);
|
sendEventBatch(metricDOList, EventType.DELETE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -889,7 +898,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
|
|||||||
|
|
||||||
private boolean isChange(MetricReq metricReq, MetricResp metricResp) {
|
private boolean isChange(MetricReq metricReq, MetricResp metricResp) {
|
||||||
boolean isNameChange = !metricReq.getName().equals(metricResp.getName());
|
boolean isNameChange = !metricReq.getName().equals(metricResp.getName());
|
||||||
boolean isBizNameChange = !Objects.equals(metricReq.getMetricDefineByMeasureParams(),metricResp.getMetricDefineByMeasureParams());
|
boolean isBizNameChange = !Objects.equals(metricReq.getMetricDefineByMeasureParams(),
|
||||||
|
metricResp.getMetricDefineByMeasureParams());
|
||||||
return isNameChange || isBizNameChange;
|
return isNameChange || isBizNameChange;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -521,13 +521,14 @@ public class ModelServiceImpl implements ModelService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void updateModelByDimAndMetric(Long modelId, List<DimensionReq> dimensionReqList, List<MetricReq> metricReqList, User user) {
|
public void updateModelByDimAndMetric(Long modelId, List<DimensionReq> dimensionReqList,
|
||||||
|
List<MetricReq> metricReqList, User user) {
|
||||||
ModelDO modelDO = getModelDO(modelId);
|
ModelDO modelDO = getModelDO(modelId);
|
||||||
ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class);
|
ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class);
|
||||||
if (!CollectionUtils.isEmpty(dimensionReqList)) {
|
if (!CollectionUtils.isEmpty(dimensionReqList)) {
|
||||||
dimensionReqList.forEach(dimensionReq -> {
|
dimensionReqList.forEach(dimensionReq -> {
|
||||||
Optional<Dimension> dimOptional = modelDetail.getDimensions().stream()
|
Optional<Dimension> dimOptional = modelDetail.getDimensions().stream().filter(
|
||||||
.filter(dimension -> dimension.getBizName().equals(dimensionReq.getBizName()))
|
dimension -> dimension.getBizName().equals(dimensionReq.getBizName()))
|
||||||
.findFirst();
|
.findFirst();
|
||||||
if (dimOptional.isPresent()) {
|
if (dimOptional.isPresent()) {
|
||||||
Dimension dimension = dimOptional.get();
|
Dimension dimension = dimOptional.get();
|
||||||
@@ -547,10 +548,13 @@ public class ModelServiceImpl implements ModelService {
|
|||||||
|
|
||||||
if (!CollectionUtils.isEmpty(metricReqList)) {
|
if (!CollectionUtils.isEmpty(metricReqList)) {
|
||||||
// 目前modeltail中的measure
|
// 目前modeltail中的measure
|
||||||
Map<String, Measure> mesureMap = modelDetail.getMeasures().stream().collect(Collectors.toMap(Measure::getBizName, a -> a, (k1, k2) -> k1));
|
Map<String, Measure> mesureMap = modelDetail.getMeasures().stream()
|
||||||
|
.collect(Collectors.toMap(Measure::getBizName, a -> a, (k1, k2) -> k1));
|
||||||
metricReqList.forEach(metricReq -> {
|
metricReqList.forEach(metricReq -> {
|
||||||
if (null != metricReq.getMetricDefineByMeasureParams() && !CollectionUtils.isEmpty(metricReq.getMetricDefineByMeasureParams().getMeasures())) {
|
if (null != metricReq.getMetricDefineByMeasureParams() && !CollectionUtils
|
||||||
for(Measure alterMeasure : metricReq.getMetricDefineByMeasureParams().getMeasures()) {
|
.isEmpty(metricReq.getMetricDefineByMeasureParams().getMeasures())) {
|
||||||
|
for (Measure alterMeasure : metricReq.getMetricDefineByMeasureParams()
|
||||||
|
.getMeasures()) {
|
||||||
if (mesureMap.containsKey(alterMeasure.getBizName())) {
|
if (mesureMap.containsKey(alterMeasure.getBizName())) {
|
||||||
Measure measure = mesureMap.get(alterMeasure.getBizName());
|
Measure measure = mesureMap.get(alterMeasure.getBizName());
|
||||||
BeanUtils.copyProperties(alterMeasure, measure);
|
BeanUtils.copyProperties(alterMeasure, measure);
|
||||||
@@ -569,13 +573,14 @@ public class ModelServiceImpl implements ModelService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void deleteModelDetailByDimAndMetric(Long modelId, List<DimensionDO> dimensionList, List<MetricDO> metricReqList) {
|
public void deleteModelDetailByDimAndMetric(Long modelId, List<DimensionDO> dimensionList,
|
||||||
|
List<MetricDO> metricReqList) {
|
||||||
ModelDO modelDO = getModelDO(modelId);
|
ModelDO modelDO = getModelDO(modelId);
|
||||||
ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class);
|
ModelDetail modelDetail = JsonUtil.toObject(modelDO.getModelDetail(), ModelDetail.class);
|
||||||
if (!CollectionUtils.isEmpty(dimensionList)) {
|
if (!CollectionUtils.isEmpty(dimensionList)) {
|
||||||
dimensionList.forEach(dimensionReq -> {
|
dimensionList.forEach(dimensionReq -> {
|
||||||
Optional<Dimension> dimOptional = modelDetail.getDimensions().stream()
|
Optional<Dimension> dimOptional = modelDetail.getDimensions().stream().filter(
|
||||||
.filter(dimension -> dimension.getBizName().equals(dimensionReq.getBizName()))
|
dimension -> dimension.getBizName().equals(dimensionReq.getBizName()))
|
||||||
.findFirst();
|
.findFirst();
|
||||||
if (dimOptional.isPresent()) {
|
if (dimOptional.isPresent()) {
|
||||||
Dimension dimension = dimOptional.get();
|
Dimension dimension = dimOptional.get();
|
||||||
|
|||||||
@@ -198,8 +198,7 @@ public class DataSetSchemaBuilder {
|
|||||||
}).collect(Collectors.toList());
|
}).collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void setDefaultTimeFormat(SchemaElement dimToAdd,
|
private static void setDefaultTimeFormat(SchemaElement dimToAdd, String timeFormat) {
|
||||||
String timeFormat) {
|
|
||||||
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
|
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,7 +56,8 @@ public class MetricCheckUtils {
|
|||||||
if (StringUtils.isBlank(expr)) {
|
if (StringUtils.isBlank(expr)) {
|
||||||
throw new InvalidArgumentException("表达式不可为空");
|
throw new InvalidArgumentException("表达式不可为空");
|
||||||
}
|
}
|
||||||
String forbiddenCharacters = NameCheckUtils.findForbiddenCharacters(metricReq.getName());
|
String forbiddenCharacters =
|
||||||
|
NameCheckUtils.findForbiddenCharacters(metricReq.getName());
|
||||||
if (StringUtils.isNotBlank(forbiddenCharacters)) {
|
if (StringUtils.isNotBlank(forbiddenCharacters)) {
|
||||||
throw new InvalidArgumentException(
|
throw new InvalidArgumentException(
|
||||||
String.format("名称包含特殊字符%s, 请修改", forbiddenCharacters));
|
String.format("名称包含特殊字符%s, 请修改", forbiddenCharacters));
|
||||||
|
|||||||
@@ -294,7 +294,8 @@ public class ModelConverter {
|
|||||||
List<Dimension> dimensions = modelReq.getModelDetail().getDimensions();
|
List<Dimension> dimensions = modelReq.getModelDetail().getDimensions();
|
||||||
List<Identify> identifiers = modelReq.getModelDetail().getIdentifiers();
|
List<Identify> identifiers = modelReq.getModelDetail().getIdentifiers();
|
||||||
List<Field> fields = modelReq.getModelDetail().getFields();
|
List<Field> fields = modelReq.getModelDetail().getFields();
|
||||||
List<String> fieldNames = fields.stream().map(Field::getFieldName).collect(Collectors.toList());
|
List<String> fieldNames =
|
||||||
|
fields.stream().map(Field::getFieldName).collect(Collectors.toList());
|
||||||
|
|
||||||
if (measures != null) {
|
if (measures != null) {
|
||||||
for (Measure measure : measures) {
|
for (Measure measure : measures) {
|
||||||
@@ -302,7 +303,8 @@ public class ModelConverter {
|
|||||||
&& StringUtils.isBlank(measure.getExpr())) {
|
&& StringUtils.isBlank(measure.getExpr())) {
|
||||||
measure.setExpr(measure.getBizName());
|
measure.setExpr(measure.getBizName());
|
||||||
}
|
}
|
||||||
if (StringUtils.isNotBlank(measure.getBizName()) && !fieldNames.contains(measure.getBizName())) {
|
if (StringUtils.isNotBlank(measure.getBizName())
|
||||||
|
&& !fieldNames.contains(measure.getBizName())) {
|
||||||
fields.add(new Field(measure.getBizName(), ""));
|
fields.add(new Field(measure.getBizName(), ""));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -313,18 +315,21 @@ public class ModelConverter {
|
|||||||
&& StringUtils.isBlank(dimension.getExpr())) {
|
&& StringUtils.isBlank(dimension.getExpr())) {
|
||||||
dimension.setExpr(dimension.getBizName());
|
dimension.setExpr(dimension.getBizName());
|
||||||
}
|
}
|
||||||
if (StringUtils.isNotBlank(dimension.getBizName()) && !fieldNames.contains(dimension.getBizName())) {
|
if (StringUtils.isNotBlank(dimension.getBizName())
|
||||||
|
&& !fieldNames.contains(dimension.getBizName())) {
|
||||||
fields.add(new Field(dimension.getBizName(), ""));
|
fields.add(new Field(dimension.getBizName(), ""));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (identifiers != null) {
|
if (identifiers != null) {
|
||||||
for (Identify identify : identifiers) {
|
for (Identify identify : identifiers) {
|
||||||
if (StringUtils.isNotBlank(identify.getBizName()) && StringUtils.isBlank(identify.getName())) {
|
if (StringUtils.isNotBlank(identify.getBizName())
|
||||||
|
&& StringUtils.isBlank(identify.getName())) {
|
||||||
identify.setName(identify.getBizName());
|
identify.setName(identify.getBizName());
|
||||||
}
|
}
|
||||||
identify.setIsCreateDimension(1);
|
identify.setIsCreateDimension(1);
|
||||||
if (StringUtils.isNotBlank(identify.getBizName()) && !fieldNames.contains(identify.getBizName())) {
|
if (StringUtils.isNotBlank(identify.getBizName())
|
||||||
|
&& !fieldNames.contains(identify.getBizName())) {
|
||||||
fields.add(new Field(identify.getBizName(), ""));
|
fields.add(new Field(identify.getBizName(), ""));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user