topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Use QueryWrapper to replace hard-code SQL in mapper xml.

Browse Source
This commit is contained in:
jerryjzhang
2024-12-02 00:00:35 +08:00
parent 4592131b55
commit 350b6089dc
10 changed files with 99 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java
View File

@@ -38,8 +38,8 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
new Parameter("s2.embedding.store.timeout", "60", "超时时间(秒)", "", "number", MODULE_NAME);
public static final Parameter EMBEDDING_STORE_DIMENSION =
new Parameter("s2.embedding.store.dimension", "", "向量维度", "", "number", MODULE_NAME, null,
getDimensionDependency());
new Parameter("s2.embedding.store.dimension", "", "向量维度", "", "number", MODULE_NAME,
null, getDimensionDependency());
public static final Parameter EMBEDDING_STORE_DATABASE_NAME =
new Parameter("s2.embedding.store.databaseName", "", "DatabaseName", "", "string",
MODULE_NAME, null, getDatabaseNameDependency());
@@ -144,6 +144,6 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
EmbeddingStoreType.PGVECTOR.name()),
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus",
EmbeddingStoreType.PGVECTOR.name(), "postgres"));
EmbeddingStoreType.PGVECTOR.name(), "postgres"));
}
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import org.apache.ibatis.annotations.Mapper;
@@ -12,11 +11,7 @@ public interface DimensionDOCustomMapper {
void batchInsert(List<DimensionDO> dimensionDOS);
void batchUpdate(List<DimensionDO> dimensionDOS);
void batchUpdateStatus(List<DimensionDO> dimensionDOS);
List<DimensionDO> query(DimensionFilter dimensionFilter);
List<DimensionDO> queryDimensions(DimensionsFilter dimensionsFilter);
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import org.apache.ibatis.annotations.Mapper;
@@ -20,7 +19,5 @@ public interface MetricDOCustomMapper {
void updateClassificationsBatch(List<MetricDO> metricDOS);
List<MetricDO> query(MetricFilter metricFilter);
List<MetricDO> queryMetrics(MetricsFilter metricsFilter);
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java
View File

@@ -1,15 +1,9 @@
package com.tencent.supersonic.headless.server.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.server.persistence.dataobject.QueryStatDO;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@Mapper
public interface StatMapper extends BaseMapper<QueryStatDO> {
List<QueryStat> getStatInfo(ItemUseReq itemUseCommend);
}

38
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java
View File

@@ -1,14 +1,17 @@
package com.tencent.supersonic.headless.server.persistence.repository.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOCustomMapper;
import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper;
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Objects;
@Service
public class DimensionRepositoryImpl implements DimensionRepository {
@@ -50,7 +53,40 @@ public class DimensionRepositoryImpl implements DimensionRepository {
@Override
public List<DimensionDO> getDimension(DimensionFilter dimensionFilter) {
return dimensionDOCustomMapper.query(dimensionFilter);
QueryWrapper<DimensionDO> queryWrapper = new QueryWrapper<>();
queryWrapper.ne("status", 3);
if (Objects.nonNull(dimensionFilter.getIds()) && !dimensionFilter.getIds().isEmpty()) {
queryWrapper.in("id", dimensionFilter.getIds());
}
if (StringUtils.isNotBlank(dimensionFilter.getId())) {
queryWrapper.eq("id", dimensionFilter.getId());
}
if (Objects.nonNull(dimensionFilter.getModelIds())
&& !dimensionFilter.getModelIds().isEmpty()) {
queryWrapper.in("model_id", dimensionFilter.getModelIds());
}
if (StringUtils.isNotBlank(dimensionFilter.getName())) {
queryWrapper.like("name", dimensionFilter.getName());
}
if (StringUtils.isNotBlank(dimensionFilter.getId())) {
queryWrapper.like("biz_name", dimensionFilter.getBizName());
}
if (Objects.nonNull(dimensionFilter.getStatus())) {
queryWrapper.eq("status", dimensionFilter.getStatus());
}
if (Objects.nonNull(dimensionFilter.getSensitiveLevel())) {
queryWrapper.eq("sensitive_level", dimensionFilter.getSensitiveLevel());
}
if (StringUtils.isNotBlank(dimensionFilter.getCreatedBy())) {
queryWrapper.eq("created_by", dimensionFilter.getCreatedBy());
}
if (StringUtils.isNotBlank(dimensionFilter.getKey())) {
String key = dimensionFilter.getKey();
queryWrapper.like("name", key).or().like("biz_name", key).or().like("description", key)
.or().like("alias", key).or().like("created_by", key);
}
return dimensionDOMapper.selectList(queryWrapper);
}
@Override

43
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java
View File

@@ -9,9 +9,11 @@ import com.tencent.supersonic.headless.server.persistence.mapper.MetricQueryDefa
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Objects;
@Component
public class MetricRepositoryImpl implements MetricRepository {
@@ -73,7 +75,46 @@ public class MetricRepositoryImpl implements MetricRepository {
@Override
public List<MetricDO> getMetric(MetricFilter metricFilter) {
return metricDOCustomMapper.query(metricFilter);
QueryWrapper<MetricDO> queryWrapper = new QueryWrapper<>();
queryWrapper.ne("status", 3);
if (Objects.nonNull(metricFilter.getIds()) && !metricFilter.getIds().isEmpty()) {
queryWrapper.in("id", metricFilter.getIds());
}
if (StringUtils.isNotBlank(metricFilter.getId())) {
queryWrapper.eq("id", metricFilter.getId());
}
if (Objects.nonNull(metricFilter.getModelIds()) && !metricFilter.getModelIds().isEmpty()) {
queryWrapper.in("model_id", metricFilter.getModelIds());
}
if (StringUtils.isNotBlank(metricFilter.getType())) {
queryWrapper.eq("type", metricFilter.getType());
}
if (StringUtils.isNotBlank(metricFilter.getName())) {
queryWrapper.like("name", metricFilter.getName());
}
if (StringUtils.isNotBlank(metricFilter.getId())) {
queryWrapper.like("biz_name", metricFilter.getBizName());
}
if (Objects.nonNull(metricFilter.getStatus())) {
queryWrapper.eq("status", metricFilter.getStatus());
}
if (Objects.nonNull(metricFilter.getSensitiveLevel())) {
queryWrapper.eq("sensitive_level", metricFilter.getSensitiveLevel());
}
if (StringUtils.isNotBlank(metricFilter.getCreatedBy())) {
queryWrapper.eq("created_by", metricFilter.getCreatedBy());
}
if (Objects.nonNull(metricFilter.getIsPublish()) && metricFilter.getIsPublish() == 1) {
queryWrapper.eq("is_publish", metricFilter.getIsPublish());
}
if (StringUtils.isNotBlank(metricFilter.getKey())) {
String key = metricFilter.getKey();
queryWrapper.like("name", key).or().like("biz_name", key).or().like("description", key)
.or().like("alias", key).or().like("classifications", key).or()
.like("created_by", key);
}
return metricDOMapper.selectList(queryWrapper);
}
@Override

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.server.persistence.repository.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
@@ -47,7 +48,7 @@ public class StatRepositoryImpl implements StatRepository {
@SneakyThrows
public List<ItemUseResp> getStatInfo(ItemUseReq itemUseReq) {
List<ItemUseResp> result = new ArrayList<>();
List<QueryStat> statInfos = statMapper.getStatInfo(itemUseReq);
List<QueryStatDO> statInfos = getQueryStats(itemUseReq);
Map<String, Long> map = new ConcurrentHashMap<>();
statInfos.stream().forEach(stat -> {
String dimensions = stat.getDimensions();
@@ -70,6 +71,21 @@ public class StatRepositoryImpl implements StatRepository {
.collect(Collectors.toList());
}
private List<QueryStatDO> getQueryStats(ItemUseReq itemUseReq) {
QueryWrapper<QueryStatDO> queryWrapper = new QueryWrapper<>();
if (Objects.nonNull(itemUseReq.getModelId())) {
queryWrapper.eq("model_id", itemUseReq.getModelId());
}
if (Objects.nonNull(itemUseReq.getModelIds()) && !itemUseReq.getModelIds().isEmpty()) {
queryWrapper.in("model_id", itemUseReq.getModelIds());
}
if (Objects.nonNull(itemUseReq.getMetric())) {
queryWrapper.like("metrics", itemUseReq.getMetric());
}
return statMapper.selectList(queryWrapper);
}
private void updateStatMapInfo(Map<String, Long> map, String dimensions, String type,
Long dataSetId) {
if (StringUtils.isNotEmpty(dimensions)) {

24
headless/server/src/main/resources/mapper/StatMapper.xml
View File

@@ -40,28 +40,4 @@
<result column="query_opt_mode" property="queryOptMode"/>
</resultMap>
<select id="getStatInfo"
resultType="com.tencent.supersonic.headless.api.pojo.QueryStat">
select *
from s2_query_stat_info
<where>
<if test="startTime != null">
and start_time >= #{startTime}String.valueOf(queryFilter.getValue())
</if>
<if test="modelId != null">
and model_id = #{modelId}
</if>
<if test="modelIds != null and modelIds.size() > 0">
and model_id in
<foreach item="id" collection="modelIds" open="(" separator="," close=")">
#{id}
</foreach>
</if>
<if test="metric != null">
and metrics like concat('%',#{metric},'%')
</if>
</where>
</select>
</mapper>

57
headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml
View File

@@ -116,63 +116,6 @@
</foreach>
</update>
<select id="query" resultMap="ResultMapWithBLOBs">
select t.*, (case when t1.id is not null then 1 else 0 end) as isTag
from s2_dimension t
left join (
select *
from s2_tag
where type = 'DIMENSION'
) t1 on t.id = t1.item_id
where status != 3
<if test="key != null and key != ''">
and ( t.id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.alias like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.description like CONCAT('%',#{key , jdbcType=VARCHAR},'%') )
</if>
<if test="id != null">
and t.id like CONCAT('%',#{id , jdbcType=VARCHAR},'%')
</if>
<if test="name != null and name != '' ">
and t.name like CONCAT('%',#{name , jdbcType=VARCHAR},'%')
</if>
<if test="bizName != null and bizName != ''">
and t.biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%')
</if>
<if test="sensitiveLevel != null">
and t.sensitive_level = #{sensitiveLevel}
</if>
<if test="status != null">
and t.status = #{status}
</if>
<if test="modelIds != null and modelIds.size >0">
and t.model_id in
<foreach collection="modelIds" index="index" item="model" open="(" close=")"
separator=",">
#{model}
</foreach>
</if>
<if test="ids != null and ids.size >0">
and t.id in
<foreach collection="ids" index="index" item="id" open="(" close=")"
separator=",">
#{id}
</foreach>
</if>
<if test="createdBy != null">
and t.created_by = #{createdBy}
</if>
<if test="isTag != null and isTag == 1">
and t1.id is not null
</if>
<if test="isTag != null and isTag == 0">
and t1.id is null
</if>
</select>
<select id="queryDimensions" resultMap="ResultMapWithBLOBs">
select *
from s2_dimension

64
headless/server/src/main/resources/mapper/custom/MetricDOCustomMapper.xml
View File

@@ -127,70 +127,6 @@
</foreach>
</update>
<select id="query" resultMap="ResultMapWithBLOBs">
select t.*, (case when t1.id is not null then 1 else 0 end) as isTag
from s2_metric t
left join (
select *
from s2_tag
where type = 'METRIC'
) t1 on t.id = t1.item_id
where t.status != 3
<if test="type != null and type != ''">
and t.type = #{type}
</if>
<if test="key != null and key != ''">
and ( t.id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.description like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.alias like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.classifications like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
t.created_by like CONCAT('%',#{key , jdbcType=VARCHAR},'%') )
</if>
<if test="id != null">
and t.id like CONCAT('%',#{id , jdbcType=VARCHAR},'%')
</if>
<if test="name != null and name != '' ">
and t.name like CONCAT('%',#{name , jdbcType=VARCHAR},'%')
</if>
<if test="bizName != null and bizName != ''">
and t.biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%')
</if>
<if test="sensitiveLevel != null">
and t.sensitive_level = #{sensitiveLevel}
</if>
<if test="status != null">
and t.status = #{status}
</if>
<if test="modelIds != null and modelIds.size >0">
and t.model_id in
<foreach collection="modelIds" index="index" item="model" open="(" close=")"
separator=",">
#{model}
</foreach>
</if>
<if test="ids != null and ids.size >0">
and t.id in
<foreach collection="ids" index="index" item="id" open="(" close=")"
separator=",">
#{id}
</foreach>
</if>
<if test="createdBy != null">
and t.created_by = #{createdBy}
</if>
<if test="isTag != null and isTag == 1">
and t1.id is not null
</if>
<if test="isTag != null and isTag == 0">
and t1.id is null
</if>
<if test="isPublish != null and isPublish == 1">
and (t.created_by = #{userName} or t.is_publish = 1)
</if>
</select>
<select id="queryMetrics" resultMap="ResultMapWithBLOBs">
select *
from s2_metric