topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-18 16:27:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project] supersonic 0.6.0 version update (#16)

Browse Source 
Co-authored-by: lexluo <lexluo@tencent.com>
This commit is contained in:
lexluo09
2023-07-16 21:32:33 +08:00
committed by GitHub
parent a0869dc7bd
commit 041daad1e4
261 changed files with 12031 additions and 3266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/application/CatalogImpl.java Normal file
View File

@@ -0,0 +1,93 @@
package com.tencent.supersonic.semantic.core.application;
import com.tencent.supersonic.semantic.api.core.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.api.core.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.core.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.semantic.core.domain.Catalog;
import com.tencent.supersonic.semantic.core.domain.DatasourceService;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.semantic.core.domain.dataobject.DatabaseDO;
import com.tencent.supersonic.semantic.core.domain.repository.DatabaseRepository;
import com.tencent.supersonic.semantic.core.domain.utils.DatabaseConverter;
import java.util.List;
import java.util.Map;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class CatalogImpl implements Catalog {
private final DatabaseRepository databaseRepository;
private final DomainService domainService;
private final DimensionService dimensionService;
private final DatasourceService datasourceService;
private final MetricService metricService;
public CatalogImpl(DatabaseRepository databaseRepository,
DomainService domainService, DimensionService dimensionService,
DatasourceService datasourceService,
MetricService metricService) {
this.databaseRepository = databaseRepository;
this.domainService = domainService;
this.dimensionService = dimensionService;
this.datasourceService = datasourceService;
this.metricService = metricService;
}
public DatabaseResp getDatabase(Long id) {
DatabaseDO databaseDO = databaseRepository.getDatabase(id);
return DatabaseConverter.convert(databaseDO);
}
@Override
public String getDomainFullPath(Long domainId) {
return domainService.getDomainFullPath(domainId);
}
@Override
public Map<Long, String> getDomainFullPath() {
return domainService.getDomainFullPath();
}
@Override
public DimensionResp getDimension(String bizName, Long domainId) {
return dimensionService.getDimension(bizName, domainId);
}
@Override
public void getModelYamlTplByDomainIds(Set<Long> domainIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DatasourceYamlTpl> datasourceYamlTplList, List<MetricYamlTpl> metricYamlTplList) {
datasourceService.getModelYamlTplByDomainIds(domainIds, dimensionYamlMap, datasourceYamlTplList,
metricYamlTplList);
}
@Override
public List<DimensionResp> getDimensions(Long domainId) {
return dimensionService.getDimensions(domainId);
}
@Override
public List<DatasourceResp> getDatasourceList(Long domainId) {
return datasourceService.getDatasourceList(domainId);
}
@Override
public List<MetricResp> getMetrics(Long domainId) {
return metricService.getMetrics(domainId);
}
@Override
public ItemDateResp getDateDate(ItemDateFilter dimension, ItemDateFilter metric) {
return datasourceService.getDateDate(dimension, metric);
}
}

17
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/application/DatasourceServiceImpl.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.core.application;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.util.json.JsonUtil;
import com.tencent.supersonic.semantic.api.core.pojo.DatasourceDetail;
import com.tencent.supersonic.semantic.api.core.pojo.Dim;
import com.tencent.supersonic.semantic.api.core.pojo.ItemDateFilter;
@@ -22,25 +23,22 @@ import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.core.response.MeasureResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import com.tencent.supersonic.common.util.json.JsonUtil;
import com.tencent.supersonic.semantic.core.domain.DatabaseService;
import com.tencent.supersonic.semantic.core.domain.DatasourceService;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.semantic.core.domain.dataobject.DatasourceDO;
import com.tencent.supersonic.semantic.core.domain.dataobject.DatasourceRelaDO;
import com.tencent.supersonic.semantic.core.domain.dataobject.DateInfoDO;
import com.tencent.supersonic.semantic.core.domain.manager.DatasourceYamlManager;
import com.tencent.supersonic.semantic.core.domain.manager.DimensionYamlManager;
import com.tencent.supersonic.semantic.core.domain.manager.MetricYamlManager;
import com.tencent.supersonic.semantic.core.domain.pojo.Datasource;
import com.tencent.supersonic.semantic.core.domain.repository.DatasourceRepository;
import com.tencent.supersonic.semantic.core.domain.repository.DateInfoRepository;
import com.tencent.supersonic.semantic.core.domain.utils.DatasourceConverter;
import com.tencent.supersonic.semantic.core.domain.utils.DimensionConverter;
import com.tencent.supersonic.semantic.core.domain.utils.MetricConverter;
import com.tencent.supersonic.semantic.core.domain.DatabaseService;
import com.tencent.supersonic.semantic.core.domain.DatasourceService;
import com.tencent.supersonic.semantic.core.domain.DimensionService;
import com.tencent.supersonic.semantic.core.domain.DomainService;
import com.tencent.supersonic.semantic.core.domain.MetricService;
import com.tencent.supersonic.semantic.core.domain.pojo.Datasource;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -338,6 +336,7 @@ public class DatasourceServiceImpl implements DatasourceService {
String dateFormat = itemDates.get(0).getDateFormat();
String startDate = itemDates.get(0).getStartDate();
String endDate = itemDates.get(0).getEndDate();
String datePeriod = itemDates.get(0).getDatePeriod();
List<String> unavailableDateList = itemDates.get(0).getUnavailableDateList();
for (DateInfoReq item : itemDates) {
String startDate1 = item.getStartDate();
@@ -354,7 +353,7 @@ public class DatasourceServiceImpl implements DatasourceService {
}
}
return new ItemDateResp(dateFormat, startDate, endDate, unavailableDateList);
return new ItemDateResp(dateFormat, startDate, endDate, datePeriod, unavailableDateList);
}

38
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/Catalog.java Normal file
View File

@@ -0,0 +1,38 @@
package com.tencent.supersonic.semantic.core.domain;
import com.tencent.supersonic.semantic.api.core.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DatasourceYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.semantic.api.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.semantic.api.core.response.DatabaseResp;
import com.tencent.supersonic.semantic.api.core.response.DatasourceResp;
import com.tencent.supersonic.semantic.api.core.response.DimensionResp;
import com.tencent.supersonic.semantic.api.core.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.core.response.MetricResp;
import java.util.List;
import java.util.Map;
import java.util.Set;
public interface Catalog {
DatabaseResp getDatabase(Long id);
List<DatasourceResp> getDatasourceList(Long domainId);
String getDomainFullPath(Long domainId);
Map<Long, String> getDomainFullPath();
DimensionResp getDimension(String bizName, Long domainId);
List<DimensionResp> getDimensions(Long domainId);
List<MetricResp> getMetrics(Long domainId);
void getModelYamlTplByDomainIds(Set<Long> domainIds, Map<String, List<DimensionYamlTpl>> dimensionYamlMap,
List<DatasourceYamlTpl> datasourceYamlTplList, List<MetricYamlTpl> metricYamlTplList);
ItemDateResp getDateDate(ItemDateFilter dimension, ItemDateFilter metric);
}

1
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/dataobject/DateInfoDO.java
View File

@@ -14,6 +14,7 @@ public class DateInfoDO {
private String unavailableDateList;
private String createdBy;
private String updatedBy;
private String datePeriod;
}

11
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/manager/DatasourceYamlManager.java
View File

@@ -15,11 +15,11 @@ import com.tencent.supersonic.semantic.core.domain.adaptor.engineadapter.EngineA
import com.tencent.supersonic.semantic.core.domain.adaptor.engineadapter.EngineAdaptorFactory;
import com.tencent.supersonic.semantic.core.domain.pojo.Datasource;
import com.tencent.supersonic.semantic.core.domain.pojo.DatasourceQueryEnum;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Service
@@ -30,7 +30,7 @@ public class DatasourceYamlManager {
DatasourceDetail datasourceDetail = datasource.getDatasourceDetail();
EngineAdaptor engineAdaptor = EngineAdaptorFactory.getEngineAdaptor(databaseResp.getType());
SysTimeDimensionBuilder.addSysTimeDimension(datasourceDetail.getDimensions(), engineAdaptor);
addInterCntMetric(datasource.getBizName(), datasourceDetail.getMeasures());
addInterCntMetric(datasource.getBizName(), datasourceDetail);
DatasourceYamlTpl datasourceYamlTpl = new DatasourceYamlTpl();
BeanUtils.copyProperties(datasourceDetail, datasourceYamlTpl);
datasourceYamlTpl.setIdentifiers(datasourceDetail.getIdentifiers().stream().map(DatasourceYamlManager::convert)
@@ -77,13 +77,16 @@ public class DatasourceYamlManager {
}
private static void addInterCntMetric(String datasourceEnName, List<Measure> measures) {
private static void addInterCntMetric(String datasourceEnName, DatasourceDetail datasourceDetail) {
Measure measure = new Measure();
measure.setExpr("1");
if (!CollectionUtils.isEmpty(datasourceDetail.getIdentifiers())) {
measure.setExpr(datasourceDetail.getIdentifiers().get(0).getBizName());
}
measure.setAgg("count");
measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt"));
measure.setCreateMetric("true");
measures.add(measure);
datasourceDetail.getMeasures().add(measure);
}

3
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/domain/pojo/MetricFilter.java
View File

@@ -1,7 +1,10 @@
package com.tencent.supersonic.semantic.core.domain.pojo;
import lombok.Data;
@Data
public class MetricFilter extends MetaFilter {
private String type;
}

4
semantic/core/src/main/java/com/tencent/supersonic/semantic/core/infrastructure/repository/MetricRepositoryImpl.java
View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.semantic.core.domain.repository.MetricRepository;
import com.tencent.supersonic.semantic.core.infrastructure.mapper.MetricDOCustomMapper;
import com.tencent.supersonic.semantic.core.infrastructure.mapper.MetricDOMapper;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
@@ -96,6 +97,9 @@ public class MetricRepositoryImpl implements MetricRepository {
if (metricFilter.getStatus() != null) {
metricDOExample.getOredCriteria().get(0).andStatusEqualTo(metricFilter.getStatus());
}
if (StringUtils.isNotBlank(metricFilter.getType())) {
metricDOExample.getOredCriteria().get(0).andTypeEqualTo(metricFilter.getType());
}
return metricDOMapper.selectByExampleWithBLOBs(metricDOExample);
}

8
semantic/core/src/main/resources/mapper/custom/DateInfoMapper.xml
View File

@@ -14,21 +14,23 @@
<result column="unavailable_date" jdbcType="INTEGER" property="unavailableDateList"/>
<result column="created_by" jdbcType="TIMESTAMP" property="createdBy"/>
<result column="updated_by" jdbcType="VARCHAR" property="updatedBy"/>
<result column="date_period" jdbcType="VARCHAR" property="datePeriod"/>
</resultMap>
<insert id="upsertDateInfo">
insert into s2_available_date_info
(`type`, item_id, date_format, start_date, end_date, unavailable_date, created_by,
updated_by)
updated_by,date_period)
values (#{type}, #{itemId}, #{dateFormat}, #{startDate}, #{endDate}, #{unavailableDateList},
#{createdBy}, #{updatedBy}) ON DUPLICATE KEY
#{createdBy}, #{updatedBy}, #{datePeriod}) ON DUPLICATE KEY
UPDATE
date_format = #{dateFormat},
start_date = #{startDate},
end_date = #{endDate},
unavailable_date = #{unavailableDateList},
created_by = #{createdBy},
updated_by = #{updatedBy}
updated_by = #{updatedBy},
date_period = #{datePeriod}
</insert>
<select id="getDateInfos" resultMap="BaseResultMap">