mirror of
https://github.com/tencentmusic/supersonic.git
synced 2026-01-06 01:08:16 +08:00
(improvement) Move out the datasource and merge the datasource with the model, and adapt the chat module (#423)
Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
jipeli
@@ -20,10 +20,10 @@ import com.tencent.supersonic.semantic.api.materialization.response.Materializat
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
|
||||
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
|
||||
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
|
||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.semantic.materialization.domain.MaterializationConfService;
|
||||
import com.tencent.supersonic.semantic.materialization.domain.pojo.Materialization;
|
||||
@@ -32,8 +32,11 @@ import com.tencent.supersonic.semantic.materialization.domain.repository.Materia
|
||||
import com.tencent.supersonic.semantic.materialization.domain.repository.MaterializationRepository;
|
||||
import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationConverter;
|
||||
import com.tencent.supersonic.semantic.materialization.domain.utils.MaterializationZipperUtils;
|
||||
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
|
||||
import com.tencent.supersonic.semantic.model.domain.ModelService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
@@ -46,11 +49,6 @@ import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -59,18 +57,16 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
|
||||
private final MaterializationRepository materializationRepository;
|
||||
private final MaterializationElementRepository materializationElementRepository;
|
||||
private final ModelService modelService;
|
||||
private final DatasourceService datasourceService;
|
||||
private final MaterializationZipperUtils materializationZipperUtils;
|
||||
private String typeAndIdSplit = "_";
|
||||
|
||||
public MaterializationConfServiceImpl(MaterializationRepository materializationRepository,
|
||||
MaterializationElementRepository materializationElementRepository,
|
||||
ModelService modelService, DatasourceService datasourceService,
|
||||
ModelService modelService,
|
||||
MaterializationZipperUtils materializationZipperUtils) {
|
||||
this.materializationRepository = materializationRepository;
|
||||
this.materializationElementRepository = materializationElementRepository;
|
||||
this.modelService = modelService;
|
||||
this.datasourceService = datasourceService;
|
||||
this.materializationZipperUtils = materializationZipperUtils;
|
||||
}
|
||||
|
||||
@@ -309,7 +305,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
|
||||
ModelSchemaFilterReq modelFilter = new ModelSchemaFilterReq();
|
||||
modelFilter.setModelIds(Arrays.asList(materializationResp.getModelId()));
|
||||
List<ModelSchemaResp> modelSchemaRespList = modelService.fetchModelSchema(modelFilter);
|
||||
List<MeasureResp> measureRespList = datasourceService.getMeasureListOfModel(
|
||||
List<MeasureResp> measureRespList = modelService.getMeasureListOfModel(
|
||||
Lists.newArrayList(materializationResp.getModelId()));
|
||||
Map<String, DimSchemaResp> dimSchemaRespMap = new HashMap<>();
|
||||
Map<String, MetricSchemaResp> metricSchemaRespHashMap = new HashMap<>();
|
||||
@@ -363,13 +359,11 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
|
||||
@Override
|
||||
public List<MaterializationSourceResp> getMaterializationSourceResp(
|
||||
Long materializationId) {
|
||||
|
||||
MaterializationResp materializationResp = materializationRepository.getMaterialization(
|
||||
materializationId);
|
||||
Long modelId = materializationResp.getModelId();
|
||||
List<DatasourceResp> modelDataSources = datasourceService.getDatasourceList(modelId);
|
||||
|
||||
Set<Long> dataSourceIds = new HashSet<>();
|
||||
Set<Long> modelIds = new HashSet<>();
|
||||
Map<Long, Map<Long, String>> dataSourceDimensions = new HashMap<>();
|
||||
Map<Long, Map<Long, String>> dataSourceMetrics = new HashMap<>();
|
||||
MaterializationConfFilter materializationConfFilter = new MaterializationConfFilter();
|
||||
@@ -380,7 +374,7 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
|
||||
ModelSchemaFilterReq modelSchemaFilterReq = new ModelSchemaFilterReq();
|
||||
modelSchemaFilterReq.setModelIds(Arrays.asList(modelId));
|
||||
List<ModelSchemaResp> modelSchemaRespList = modelService.fetchModelSchema(modelSchemaFilterReq);
|
||||
List<MeasureResp> measureRespList = datasourceService.getMeasureListOfModel(Lists.newArrayList(modelId));
|
||||
List<MeasureResp> measureRespList = modelService.getMeasureListOfModel(Lists.newArrayList(modelId));
|
||||
Set<Long> dimensionIds = new HashSet<>();
|
||||
Set<Long> metricIds = new HashSet<>();
|
||||
materializationElementRespList.stream().forEach(e -> {
|
||||
@@ -393,11 +387,11 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
|
||||
});
|
||||
modelSchemaRespList.stream().forEach(m -> {
|
||||
m.getDimensions().stream().filter(mm -> dimensionIds.contains(mm.getId())).forEach(mm -> {
|
||||
if (!dataSourceDimensions.containsKey(mm.getDatasourceId())) {
|
||||
dataSourceDimensions.put(mm.getDatasourceId(), new HashMap<>());
|
||||
if (!dataSourceDimensions.containsKey(mm.getModelId())) {
|
||||
dataSourceDimensions.put(mm.getModelId(), new HashMap<>());
|
||||
}
|
||||
dataSourceDimensions.get(mm.getDatasourceId()).put(mm.getId(), mm.getBizName());
|
||||
dataSourceIds.add(mm.getDatasourceId());
|
||||
dataSourceDimensions.get(mm.getModelId()).put(mm.getId(), mm.getBizName());
|
||||
modelIds.add(mm.getModelId());
|
||||
});
|
||||
m.getMetrics().stream().filter(mm -> metricIds.contains(mm.getId())).forEach(mm -> {
|
||||
Long sourceId = 0L;
|
||||
@@ -412,27 +406,29 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
|
||||
dataSourceMetrics.put(sourceId, new HashMap<>());
|
||||
}
|
||||
dataSourceMetrics.get(sourceId).put(mm.getId(), mm.getBizName());
|
||||
dataSourceIds.add(sourceId);
|
||||
modelIds.add(sourceId);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
List<MaterializationSourceResp> materializationSourceResps = new ArrayList<>();
|
||||
for (Long sourceId : dataSourceIds) {
|
||||
Optional<DatasourceResp> datasourceResp = modelDataSources.stream().filter(d -> d.getId().equals(sourceId))
|
||||
.findFirst();
|
||||
if (datasourceResp.isPresent()) {
|
||||
for (Long sourceId : modelIds) {
|
||||
//todo
|
||||
// Optional<ModelResp> modelResp = modelSchemaRespList.stream().filter(d -> d.getId().equals(sourceId))
|
||||
// .findFirst();
|
||||
Optional<ModelResp> modelResp = Optional.empty();
|
||||
if (modelResp.isPresent()) {
|
||||
MaterializationSourceResp materializationSourceResp = MaterializationSourceResp.builder()
|
||||
.dataSourceId(sourceId)
|
||||
.materializationId(materializationId)
|
||||
.modelId(modelId)
|
||||
.depends(datasourceResp.get().getDepends())
|
||||
.depends(modelResp.get().getDepends())
|
||||
.materializedType(materializationResp.getMaterializedType())
|
||||
.entities(materializationResp.getEntities())
|
||||
.dateInfo(materializationResp.getDateInfo())
|
||||
.updateCycle(materializationResp.getUpdateCycle())
|
||||
.build();
|
||||
setDataSourceDb(datasourceResp.get(), materializationSourceResp);
|
||||
setDataSourceDb(modelResp.get(), materializationSourceResp);
|
||||
materializationSourceResp.setMetrics(
|
||||
dataSourceMetrics.containsKey(sourceId) ? dataSourceMetrics.get(sourceId)
|
||||
: new HashMap<>());
|
||||
@@ -456,16 +452,16 @@ public class MaterializationConfServiceImpl implements MaterializationConfServic
|
||||
return 0L;
|
||||
}
|
||||
|
||||
private void setDataSourceDb(DatasourceResp datasourceResp, MaterializationSourceResp materializationSourceResp) {
|
||||
if (Objects.nonNull(datasourceResp.getDatasourceDetail())) {
|
||||
private void setDataSourceDb(ModelResp datasourceResp, MaterializationSourceResp materializationSourceResp) {
|
||||
if (Objects.nonNull(datasourceResp.getModelDetail())) {
|
||||
String dbTable = "";
|
||||
if (Objects.nonNull(datasourceResp.getDatasourceDetail().getTableQuery())
|
||||
&& !datasourceResp.getDatasourceDetail().getTableQuery().isEmpty()) {
|
||||
dbTable = datasourceResp.getDatasourceDetail().getTableQuery();
|
||||
if (Objects.nonNull(datasourceResp.getModelDetail().getTableQuery())
|
||||
&& !datasourceResp.getModelDetail().getTableQuery().isEmpty()) {
|
||||
dbTable = datasourceResp.getModelDetail().getTableQuery();
|
||||
}
|
||||
if (Objects.nonNull(datasourceResp.getDatasourceDetail().getSqlQuery())
|
||||
&& !datasourceResp.getDatasourceDetail().getSqlQuery().isEmpty()) {
|
||||
dbTable = SqlParserSelectHelper.getDbTableName(datasourceResp.getDatasourceDetail().getSqlQuery());
|
||||
if (Objects.nonNull(datasourceResp.getModelDetail().getSqlQuery())
|
||||
&& !datasourceResp.getModelDetail().getSqlQuery().isEmpty()) {
|
||||
dbTable = SqlParserSelectHelper.getDbTableName(datasourceResp.getModelDetail().getSqlQuery());
|
||||
}
|
||||
if (!dbTable.isEmpty()) {
|
||||
String[] db = dbTable.split("\\.");
|
||||
|
||||
Reference in New Issue
Block a user