topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvment)(chat) optimize parse performance (#197)

Browse Source 
* (improvment)(chat) optimize parse performance
---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-10-12 11:51:57 +08:00
committed by GitHub
parent b753eda9b9
commit e7b8c68dba
26 changed files with 214 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/CatalogImpl.java
View File

@@ -103,15 +103,13 @@ public class CatalogImpl implements Catalog {
}
@Override
public String getAgg(Long modelId, String metricBizName) {
public String getAgg(List<MetricResp> metricResps, List<MeasureResp> measureRespList, String metricBizName) {
try {
List<MetricResp> metricResps = getMetrics(modelId);
if (!CollectionUtils.isEmpty(metricResps)) {
Optional<MetricResp> metric = metricResps.stream()
.filter(m -> m.getBizName().equalsIgnoreCase(metricBizName)).findFirst();
if (metric.isPresent() && Objects.nonNull(metric.get().getTypeParams()) && !CollectionUtils.isEmpty(
metric.get().getTypeParams().getMeasures())) {
List<MeasureResp> measureRespList = datasourceService.getMeasureListOfModel(modelId);
if (!CollectionUtils.isEmpty(measureRespList)) {
String measureName = metric.get().getTypeParams().getMeasures().get(0).getBizName();
Optional<MeasureResp> measure = measureRespList.stream()

29
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DatasourceServiceImpl.java
View File

@@ -146,6 +146,28 @@ public class DatasourceServiceImpl implements DatasourceService {
return measureResps;
}
@Override
public List<MeasureResp> getMeasureListOfModel(List<Long> modelIds) {
if (CollectionUtils.isEmpty(modelIds)) {
return Lists.newArrayList();
}
List<DatasourceResp> datasourceResps = getDatasourceList().stream().filter(datasourceResp ->
modelIds.contains(datasourceResp.getModelId())).collect(Collectors.toList());
List<MeasureResp> measureResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(datasourceResps)) {
for (DatasourceResp datasourceDesc : datasourceResps) {
DatasourceDetail datasourceDetail = datasourceDesc.getDatasourceDetail();
List<Measure> measures = datasourceDetail.getMeasures();
if (!CollectionUtils.isEmpty(measures)) {
measureResps.addAll(
measures.stream().map(measure -> DatasourceConverter.convert(measure, datasourceDesc))
.collect(Collectors.toList()));
}
}
}
return measureResps;
}
private void batchCreateDimension(Datasource datasource, User user) throws Exception {
List<DimensionReq> dimensionReqs = DatasourceConverter.convertDimensionList(datasource);
@@ -239,13 +261,6 @@ public class DatasourceServiceImpl implements DatasourceService {
return DatasourceConverter.convertList(datasourceRepository.getDatasourceList());
}
@Override
public List<DatasourceResp> getDatasourceListByDatabaseId(Long databaseId) {
return getDatasourceList().stream()
.filter(datasourceResp -> datasourceResp.getDatabaseId().equals(databaseId))
.collect(Collectors.toList());
}
@Override
public List<DatasourceResp> getDatasourceListNoMeasurePrefix(Long modelId) {
List<DatasourceResp> datasourceResps = getDatasourceList(modelId);

25
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java
View File

@@ -120,19 +120,16 @@ public class DimensionServiceImpl implements DimensionService {
Dimension dimension = DimensionConverter.convert(dimensionReq);
dimension.updatedBy(user.getName());
log.info("[update dimension] object:{}", JSONObject.toJSONString(dimension));
List<DimensionResp> dimensionRespList = getDimensions(dimensionReq.getModelId()).stream().filter(
o -> o.getId().equals(dimensionReq.getId())).collect(Collectors.toList());
updateDimension(dimension);
DimensionResp dimensionResp = getDimension(dimensionReq.getId());
//动态更新字典
String type = DictWordType.DIMENSION.getType();
if (!CollectionUtils.isEmpty(dimensionRespList)) {
log.info("dimensionRespList size:{}", dimensionRespList.size());
log.info("name:{}", dimensionRespList.get(0).getName());
if (dimensionResp != null) {
applicationEventPublisher.publishEvent(
new DataUpdateEvent(this, dimensionRespList.get(0).getName(),
new DataUpdateEvent(this, dimensionResp.getName(),
dimensionReq.getName(),
dimension.getModelId(),
dimensionRespList.get(0).getId(), type));
dimensionResp.getId(), type));
}
}
@@ -156,6 +153,12 @@ public class DimensionServiceImpl implements DimensionService {
return null;
}
@Override
public DimensionResp getDimension(Long id) {
DimensionDO dimensionDO = dimensionRepository.getDimensionById(id);
return DimensionConverter.convert2DimensionResp(dimensionDO, new HashMap<>(), new HashMap<>());
}
@Override
public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq) {
DimensionFilter dimensionFilter = new DimensionFilter();
@@ -199,6 +202,12 @@ public class DimensionServiceImpl implements DimensionService {
return convertList(getDimensionDOS(), datasourceService.getDatasourceMap());
}
@Override
public List<DimensionResp> getDimensionsByModelIds(List<Long> modelIds) {
List<DimensionDO> dimensionDOS = dimensionRepository.getDimensionListOfmodelIds(modelIds);
return convertList(dimensionDOS, datasourceService.getDatasourceMap());
}
@Override
public List<DimensionResp> getDimensionsByDatasource(Long datasourceId) {
List<DimensionResp> dimensionResps = Lists.newArrayList();
@@ -239,7 +248,7 @@ public class DimensionServiceImpl implements DimensionService {
protected List<DimensionDO> getDimensionDOS(Long modelId) {
return dimensionRepository.getDimensionListOfDomain(modelId);
return dimensionRepository.getDimensionListOfmodel(modelId);
}
protected List<DimensionDO> getDimensionDOS() {

9
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java
View File

@@ -121,7 +121,14 @@ public class MetricServiceImpl implements MetricService {
}
public List<MetricResp> getMetrics(List<Long> ids) {
return convertList(metricRepository.getMetricListByIds(ids));
List<MetricDO> metricDOS = metricRepository.getMetricListByIds(ids);
return convertList(metricDOS);
}
@Override
public List<MetricResp> getMetricsByModelIds(List<Long> modelIds) {
List<MetricDO> metricDOS = metricRepository.getMetricList(modelIds);
return convertList(metricDOS);
}
@Override

71
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/ModelServiceImpl.java
View File

@@ -14,6 +14,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
@@ -226,7 +227,7 @@ public class ModelServiceImpl implements ModelService {
@Override
public Map<Long, String> getModelFullPathMap() {
return getModelList().stream().filter(m -> m != null).collect(Collectors.toMap(ModelResp::getId,
return getModelList().stream().filter(Objects::nonNull).collect(Collectors.toMap(ModelResp::getId,
ModelResp::getFullPath, (k1, k2) -> k1));
}
@@ -277,12 +278,29 @@ public class ModelServiceImpl implements ModelService {
if (CollectionUtils.isEmpty(modelIds)) {
modelIds = generateModelIdsReq(modelSchemaFilterReq);
}
modelIds.stream().forEach(modelId -> {
ModelSchemaResp modelSchemaResp = fetchSingleModelSchema(modelId);
if (Objects.nonNull(modelSchemaResp)) {
modelSchemaRespList.add(modelSchemaResp);
Map<Long, List<MetricResp>> metricRespMap = metricService.getMetricsByModelIds(modelIds)
.stream().collect(Collectors.groupingBy(MetricResp::getModelId));
Map<Long, List<DimensionResp>> dimensionRespsMap = dimensionService.getDimensionsByModelIds(modelIds)
.stream().collect(Collectors.groupingBy(DimensionResp::getModelId));
Map<Long, List<MeasureResp>> measureRespsMap = datasourceService.getMeasureListOfModel(modelIds)
.stream().collect(Collectors.groupingBy(MeasureResp::getModelId));
for (Long modelId : modelIds) {
ModelResp modelResp = getModelMap().get(modelId);
if (modelResp == null) {
continue;
}
});
List<MeasureResp> measureResps = measureRespsMap.getOrDefault(modelId, Lists.newArrayList());
List<MetricResp> metricResps = metricRespMap.getOrDefault(modelId, Lists.newArrayList());
List<MetricSchemaResp> metricSchemaResps = metricResps.stream().map(metricResp ->
convert(metricResp, metricResps, measureResps)).collect(Collectors.toList());
List<DimSchemaResp> dimensionResps = dimensionRespsMap.getOrDefault(modelId, Lists.newArrayList())
.stream().map(this::convert).collect(Collectors.toList());
ModelSchemaResp modelSchemaResp = new ModelSchemaResp();
BeanUtils.copyProperties(modelResp, modelSchemaResp);
modelSchemaResp.setDimensions(dimensionResps);
modelSchemaResp.setMetrics(metricSchemaResps);
modelSchemaRespList.add(modelSchemaResp);
}
return modelSchemaRespList;
}
@@ -298,30 +316,33 @@ public class ModelServiceImpl implements ModelService {
private List<MetricSchemaResp> generateMetricSchema(Long modelId) {
List<MetricSchemaResp> metricSchemaDescList = new ArrayList<>();
List<MetricResp> metricDescList = metricService.getMetrics(modelId);
metricDescList.stream().forEach(metricDesc -> {
MetricSchemaResp metricSchemaDesc = new MetricSchemaResp();
BeanUtils.copyProperties(metricDesc, metricSchemaDesc);
metricSchemaDesc.setUseCnt(0L);
String agg = catalog.getAgg(modelId, metricSchemaDesc.getBizName());
metricSchemaDesc.setDefaultAgg(agg);
metricSchemaDescList.add(metricSchemaDesc);
}
);
List<MetricResp> metricResps = metricService.getMetrics(modelId);
List<MeasureResp> measureResps = datasourceService.getMeasureListOfModel(modelId);
metricResps.stream().forEach(metricResp ->
metricSchemaDescList.add(convert(metricResp, metricResps, measureResps)));
return metricSchemaDescList;
}
private List<DimSchemaResp> generateDimSchema(Long modelId) {
List<DimSchemaResp> dimSchemaDescList = new ArrayList<>();
List<DimensionResp> dimDescList = dimensionService.getDimensions(modelId);
dimDescList.stream().forEach(dimDesc -> {
DimSchemaResp dimSchemaDesc = new DimSchemaResp();
BeanUtils.copyProperties(dimDesc, dimSchemaDesc);
dimSchemaDesc.setUseCnt(0L);
dimSchemaDescList.add(dimSchemaDesc);
}
);
return dimSchemaDescList;
return dimDescList.stream().map(this::convert).collect(Collectors.toList());
}
private DimSchemaResp convert(DimensionResp dimensionResp) {
DimSchemaResp dimSchemaResp = new DimSchemaResp();
BeanUtils.copyProperties(dimensionResp, dimSchemaResp);
dimSchemaResp.setUseCnt(0L);
return dimSchemaResp;
}
private MetricSchemaResp convert(MetricResp metricResp, List<MetricResp> metricResps,
List<MeasureResp> measureResps) {
MetricSchemaResp metricSchemaResp = new MetricSchemaResp();
BeanUtils.copyProperties(metricResp, metricSchemaResp);
metricSchemaResp.setUseCnt(0L);
String agg = catalog.getAgg(metricResps, measureResps, metricSchemaResp.getBizName());
metricSchemaResp.setDefaultAgg(agg);
return metricSchemaResp;
}
private List<Long> generateModelIdsReq(ModelSchemaFilterReq filter) {

3
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/Catalog.java
View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MeasureResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.model.yaml.DimensionYamlTpl;
@@ -37,6 +38,6 @@ public interface Catalog {
ItemDateResp getItemDate(ItemDateFilter dimension, ItemDateFilter metric);
String getAgg(Long modelId, String metricBizName);
String getAgg(List<MetricResp> metricResps, List<MeasureResp> measureRespList, String metricBizName);
}

4
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DatasourceService.java
View File

@@ -24,8 +24,6 @@ public interface DatasourceService {
List<DatasourceResp> getDatasourceListNoMeasurePrefix(Long modelId);
List<DatasourceResp> getDatasourceListByDatabaseId(Long databaseId);
List<DatasourceResp> getDatasourceList();
List<DatasourceResp> getDatasourceList(Long modelId);
@@ -44,6 +42,8 @@ public interface DatasourceService {
List<MeasureResp> getMeasureListOfModel(Long modelId);
List<MeasureResp> getMeasureListOfModel(List<Long> modelIds);
void getModelYamlTplByModelIds(Set<Long> modelIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DatasourceYamlTpl> datasourceYamlTplList, List<MetricYamlTpl> metricYamlTplList);

6
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DimensionService.java
View File

@@ -16,7 +16,11 @@ public interface DimensionService {
List<DimensionResp> getDimensions();
DimensionResp getDimension(String bizName, Long domainId);
DimensionResp getDimension(Long id);
DimensionResp getDimension(String bizName, Long modelId);
List<DimensionResp> getDimensionsByModelIds(List<Long> modelIds);
void createDimension(DimensionReq dimensionReq, User user) throws Exception;

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/MetricService.java
View File

@@ -23,6 +23,8 @@ public interface MetricService {
void createMetricBatch(List<MetricReq> metricReqs, User user) throws Exception;
List<MetricResp> getMetricsByModelIds(List<Long> modelIds);
PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user);
MetricResp getMetric(Long modelId, String bizName);

4
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/DimensionRepository.java
View File

@@ -15,7 +15,9 @@ public interface DimensionRepository {
List<DimensionDO> getDimensionListOfDatasource(Long datasourceId);
List<DimensionDO> getDimensionListOfDomain(Long domainId);
List<DimensionDO> getDimensionListOfmodel(Long domainId);
List<DimensionDO> getDimensionListOfmodelIds(List<Long> modelIds);
List<DimensionDO> getDimensionList();

2
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/repository/MetricRepository.java
View File

@@ -17,6 +17,8 @@ public interface MetricRepository {
List<MetricDO> getMetricList(Long domainId);
List<MetricDO> getMetricList(List<Long> modelIds);
List<MetricDO> getMetricList();
List<MetricDO> getMetricListByIds(List<Long> ids);

15
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatasourceConverter.java
View File

@@ -88,13 +88,14 @@ public class DatasourceConverter {
return datasourceDesc;
}
public static MeasureResp convert(Measure measure, DatasourceResp datasourceDesc) {
MeasureResp measureDesc = new MeasureResp();
BeanUtils.copyProperties(measure, measureDesc);
measureDesc.setDatasourceId(datasourceDesc.getId());
measureDesc.setDatasourceName(datasourceDesc.getName());
measureDesc.setDatasourceBizName(datasourceDesc.getBizName());
return measureDesc;
public static MeasureResp convert(Measure measure, DatasourceResp datasourceResp) {
MeasureResp measureResp = new MeasureResp();
BeanUtils.copyProperties(measure, measureResp);
measureResp.setDatasourceId(datasourceResp.getId());
measureResp.setDatasourceName(datasourceResp.getName());
measureResp.setDatasourceBizName(datasourceResp.getBizName());
measureResp.setModelId(datasourceResp.getModelId());
return measureResp;
}
public static DimensionReq convert(Dim dim, Datasource datasource) {

3
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/NameCheckUtils.java
View File

@@ -5,6 +5,9 @@ import java.util.regex.Pattern;
public class NameCheckUtils {
public static boolean containsSpecialCharacters(String str) {
if (str == null) {
return false;
}
String regex = "^[^a-zA-Z\\u4E00-\\u9FA5_\\d].*|^\\d.*";
return Pattern.compile(regex).matcher(str).find();
}

11
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java
View File

@@ -48,9 +48,16 @@ public class DimensionRepositoryImpl implements DimensionRepository {
}
@Override
public List<DimensionDO> getDimensionListOfDomain(Long domainId) {
public List<DimensionDO> getDimensionListOfmodel(Long modelId) {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
dimensionDOExample.createCriteria().andModelIdEqualTo(domainId);
dimensionDOExample.createCriteria().andModelIdEqualTo(modelId);
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}
@Override
public List<DimensionDO> getDimensionListOfmodelIds(List<Long> modelIds) {
DimensionDOExample dimensionDOExample = new DimensionDOExample();
dimensionDOExample.createCriteria().andModelIdIn(modelIds);
return dimensionDOMapper.selectByExampleWithBLOBs(dimensionDOExample);
}

11
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java
View File

@@ -42,9 +42,16 @@ public class MetricRepositoryImpl implements MetricRepository {
}
@Override
public List<MetricDO> getMetricList(Long domainId) {
public List<MetricDO> getMetricList(Long modelId) {
MetricDOExample metricDOExample = new MetricDOExample();
metricDOExample.createCriteria().andModelIdEqualTo(domainId);
metricDOExample.createCriteria().andModelIdEqualTo(modelId);
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
}
@Override
public List<MetricDO> getMetricList(List<Long> modelIds) {
MetricDOExample metricDOExample = new MetricDOExample();
metricDOExample.createCriteria().andModelIdIn(modelIds);
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
}