From 350b6089dcbc3f5aac312aaa2164c73566189783 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Mon, 2 Dec 2024 00:00:35 +0800 Subject: [PATCH] [improvement][headless]Use QueryWrapper to replace hard-code SQL in mapper xml. --- .../config/EmbeddingStoreParameterConfig.java | 6 +- .../mapper/DimensionDOCustomMapper.java | 5 -- .../mapper/MetricDOCustomMapper.java | 3 - .../server/persistence/mapper/StatMapper.java | 6 -- .../impl/DimensionRepositoryImpl.java | 38 ++++++++++- .../repository/impl/MetricRepositoryImpl.java | 43 ++++++++++++- .../repository/impl/StatRepositoryImpl.java | 18 +++++- .../src/main/resources/mapper/StatMapper.xml | 24 ------- .../mapper/custom/DimensionDOCustomMapper.xml | 57 ----------------- .../mapper/custom/MetricDOCustomMapper.xml | 64 ------------------- 10 files changed, 99 insertions(+), 165 deletions(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java index ef7ae7fb1..126d8d7d6 100644 --- a/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/config/EmbeddingStoreParameterConfig.java @@ -38,8 +38,8 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { new Parameter("s2.embedding.store.timeout", "60", "超时时间(秒)", "", "number", MODULE_NAME); public static final Parameter EMBEDDING_STORE_DIMENSION = - new Parameter("s2.embedding.store.dimension", "", "向量维度", "", "number", MODULE_NAME, null, - getDimensionDependency()); + new Parameter("s2.embedding.store.dimension", "", "向量维度", "", "number", MODULE_NAME, + null, getDimensionDependency()); public static final Parameter EMBEDDING_STORE_DATABASE_NAME = new Parameter("s2.embedding.store.databaseName", "", "DatabaseName", "", "string", MODULE_NAME, null, getDatabaseNameDependency()); @@ -144,6 +144,6 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig { Lists.newArrayList(EmbeddingStoreType.MILVUS.name(), EmbeddingStoreType.PGVECTOR.name()), ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus", - EmbeddingStoreType.PGVECTOR.name(), "postgres")); + EmbeddingStoreType.PGVECTOR.name(), "postgres")); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java index 746afe6a5..741220eec 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/DimensionDOCustomMapper.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; -import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; import org.apache.ibatis.annotations.Mapper; @@ -12,11 +11,7 @@ public interface DimensionDOCustomMapper { void batchInsert(List dimensionDOS); - void batchUpdate(List dimensionDOS); - void batchUpdateStatus(List dimensionDOS); - List query(DimensionFilter dimensionFilter); - List queryDimensions(DimensionsFilter dimensionsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java index 651e927d3..f53795790 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/MetricDOCustomMapper.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; -import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; import org.apache.ibatis.annotations.Mapper; @@ -20,7 +19,5 @@ public interface MetricDOCustomMapper { void updateClassificationsBatch(List metricDOS); - List query(MetricFilter metricFilter); - List queryMetrics(MetricsFilter metricsFilter); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java index 32095d40f..60e0b8743 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/StatMapper.java @@ -1,15 +1,9 @@ package com.tencent.supersonic.headless.server.persistence.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.tencent.supersonic.headless.api.pojo.QueryStat; -import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.server.persistence.dataobject.QueryStatDO; import org.apache.ibatis.annotations.Mapper; -import java.util.List; - @Mapper public interface StatMapper extends BaseMapper { - - List getStatInfo(ItemUseReq itemUseCommend); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java index 4fe25fcda..61f3d157f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java @@ -1,14 +1,17 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOCustomMapper; import com.tencent.supersonic.headless.server.persistence.mapper.DimensionDOMapper; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.pojo.DimensionsFilter; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import java.util.List; +import java.util.Objects; @Service public class DimensionRepositoryImpl implements DimensionRepository { @@ -50,7 +53,40 @@ public class DimensionRepositoryImpl implements DimensionRepository { @Override public List getDimension(DimensionFilter dimensionFilter) { - return dimensionDOCustomMapper.query(dimensionFilter); + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.ne("status", 3); + if (Objects.nonNull(dimensionFilter.getIds()) && !dimensionFilter.getIds().isEmpty()) { + queryWrapper.in("id", dimensionFilter.getIds()); + } + if (StringUtils.isNotBlank(dimensionFilter.getId())) { + queryWrapper.eq("id", dimensionFilter.getId()); + } + if (Objects.nonNull(dimensionFilter.getModelIds()) + && !dimensionFilter.getModelIds().isEmpty()) { + queryWrapper.in("model_id", dimensionFilter.getModelIds()); + } + if (StringUtils.isNotBlank(dimensionFilter.getName())) { + queryWrapper.like("name", dimensionFilter.getName()); + } + if (StringUtils.isNotBlank(dimensionFilter.getId())) { + queryWrapper.like("biz_name", dimensionFilter.getBizName()); + } + if (Objects.nonNull(dimensionFilter.getStatus())) { + queryWrapper.eq("status", dimensionFilter.getStatus()); + } + if (Objects.nonNull(dimensionFilter.getSensitiveLevel())) { + queryWrapper.eq("sensitive_level", dimensionFilter.getSensitiveLevel()); + } + if (StringUtils.isNotBlank(dimensionFilter.getCreatedBy())) { + queryWrapper.eq("created_by", dimensionFilter.getCreatedBy()); + } + if (StringUtils.isNotBlank(dimensionFilter.getKey())) { + String key = dimensionFilter.getKey(); + queryWrapper.like("name", key).or().like("biz_name", key).or().like("description", key) + .or().like("alias", key).or().like("created_by", key); + } + + return dimensionDOMapper.selectList(queryWrapper); } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java index f79d8eabd..4b39af5be 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/MetricRepositoryImpl.java @@ -9,9 +9,11 @@ import com.tencent.supersonic.headless.server.persistence.mapper.MetricQueryDefa import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; import com.tencent.supersonic.headless.server.pojo.MetricFilter; import com.tencent.supersonic.headless.server.pojo.MetricsFilter; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import java.util.List; +import java.util.Objects; @Component public class MetricRepositoryImpl implements MetricRepository { @@ -73,7 +75,46 @@ public class MetricRepositoryImpl implements MetricRepository { @Override public List getMetric(MetricFilter metricFilter) { - return metricDOCustomMapper.query(metricFilter); + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.ne("status", 3); + if (Objects.nonNull(metricFilter.getIds()) && !metricFilter.getIds().isEmpty()) { + queryWrapper.in("id", metricFilter.getIds()); + } + if (StringUtils.isNotBlank(metricFilter.getId())) { + queryWrapper.eq("id", metricFilter.getId()); + } + if (Objects.nonNull(metricFilter.getModelIds()) && !metricFilter.getModelIds().isEmpty()) { + queryWrapper.in("model_id", metricFilter.getModelIds()); + } + if (StringUtils.isNotBlank(metricFilter.getType())) { + queryWrapper.eq("type", metricFilter.getType()); + } + if (StringUtils.isNotBlank(metricFilter.getName())) { + queryWrapper.like("name", metricFilter.getName()); + } + if (StringUtils.isNotBlank(metricFilter.getId())) { + queryWrapper.like("biz_name", metricFilter.getBizName()); + } + if (Objects.nonNull(metricFilter.getStatus())) { + queryWrapper.eq("status", metricFilter.getStatus()); + } + if (Objects.nonNull(metricFilter.getSensitiveLevel())) { + queryWrapper.eq("sensitive_level", metricFilter.getSensitiveLevel()); + } + if (StringUtils.isNotBlank(metricFilter.getCreatedBy())) { + queryWrapper.eq("created_by", metricFilter.getCreatedBy()); + } + if (Objects.nonNull(metricFilter.getIsPublish()) && metricFilter.getIsPublish() == 1) { + queryWrapper.eq("is_publish", metricFilter.getIsPublish()); + } + if (StringUtils.isNotBlank(metricFilter.getKey())) { + String key = metricFilter.getKey(); + queryWrapper.like("name", key).or().like("biz_name", key).or().like("description", key) + .or().like("alias", key).or().like("classifications", key).or() + .like("created_by", key); + } + + return metricDOMapper.selectList(queryWrapper); } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java index f670242c0..9383bc607 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/StatRepositoryImpl.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.tencent.supersonic.common.pojo.enums.TypeEnums; @@ -47,7 +48,7 @@ public class StatRepositoryImpl implements StatRepository { @SneakyThrows public List getStatInfo(ItemUseReq itemUseReq) { List result = new ArrayList<>(); - List statInfos = statMapper.getStatInfo(itemUseReq); + List statInfos = getQueryStats(itemUseReq); Map map = new ConcurrentHashMap<>(); statInfos.stream().forEach(stat -> { String dimensions = stat.getDimensions(); @@ -70,6 +71,21 @@ public class StatRepositoryImpl implements StatRepository { .collect(Collectors.toList()); } + private List getQueryStats(ItemUseReq itemUseReq) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + if (Objects.nonNull(itemUseReq.getModelId())) { + queryWrapper.eq("model_id", itemUseReq.getModelId()); + } + if (Objects.nonNull(itemUseReq.getModelIds()) && !itemUseReq.getModelIds().isEmpty()) { + queryWrapper.in("model_id", itemUseReq.getModelIds()); + } + if (Objects.nonNull(itemUseReq.getMetric())) { + queryWrapper.like("metrics", itemUseReq.getMetric()); + } + + return statMapper.selectList(queryWrapper); + } + private void updateStatMapInfo(Map map, String dimensions, String type, Long dataSetId) { if (StringUtils.isNotEmpty(dimensions)) { diff --git a/headless/server/src/main/resources/mapper/StatMapper.xml b/headless/server/src/main/resources/mapper/StatMapper.xml index 9ed8b8cdf..845625898 100644 --- a/headless/server/src/main/resources/mapper/StatMapper.xml +++ b/headless/server/src/main/resources/mapper/StatMapper.xml @@ -40,28 +40,4 @@ - - \ No newline at end of file diff --git a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml index daae81cee..6f903887e 100644 --- a/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml +++ b/headless/server/src/main/resources/mapper/custom/DimensionDOCustomMapper.xml @@ -116,63 +116,6 @@ - - - - select t.*, (case when t1.id is not null then 1 else 0 end) as isTag - from s2_metric t - left join ( - select * - from s2_tag - where type = 'METRIC' - ) t1 on t.id = t1.item_id - where t.status != 3 - - and t.type = #{type} - - - and ( t.id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.biz_name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.description like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.alias like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.classifications like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or - t.created_by like CONCAT('%',#{key , jdbcType=VARCHAR},'%') ) - - - and t.id like CONCAT('%',#{id , jdbcType=VARCHAR},'%') - - - and t.name like CONCAT('%',#{name , jdbcType=VARCHAR},'%') - - - and t.biz_name like CONCAT('%',#{bizName , jdbcType=VARCHAR},'%') - - - and t.sensitive_level = #{sensitiveLevel} - - - and t.status = #{status} - - - and t.model_id in - - #{model} - - - - and t.id in - - #{id} - - - - and t.created_by = #{createdBy} - - - and t1.id is not null - - - and t1.id is null - - - and (t.created_by = #{userName} or t.is_publish = 1) - - -