From 36c8938ff7029494d6a576a635c87fb6837ddb25 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Tue, 14 Nov 2023 21:41:59 +0800 Subject: [PATCH] (improvement) (common) add sys parameter setting (#384) Co-authored-by: jolunoluo --- .../api/authorization/pojo/AuthGroup.java | 2 +- .../auth/api/authorization/pojo/AuthRes.java | 4 +- .../request/QueryAuthResReq.java | 2 +- .../application/AuthServiceImpl.java | 22 ++++----- .../chat/config/OptimizationConfig.java | 6 --- .../supersonic/common/pojo/Parameter.java | 1 + .../supersonic/common/pojo/SysParameter.java | 47 +++++++++++++------ .../service/impl/SysParameterServiceImpl.java | 17 ++++--- .../supersonic/common/util/DateUtils.java | 6 ++- .../tencent/supersonic/LoadModelDataDemo.java | 4 +- .../src/main/resources/db/sql-update.sql | 5 +- .../query/service/AuthCommonService.java | 15 +++--- .../query/service/DownloadServiceImpl.java | 4 +- .../query/utils/DataPermissionAOP.java | 8 ++-- .../query/utils/DataTransformUtils.java | 17 ++++++- .../query/utils/DataTransformUtilsTest.java | 4 +- 16 files changed, 98 insertions(+), 66 deletions(-) diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java index 5e4ffba0d..683729a3c 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java @@ -6,7 +6,7 @@ import lombok.Data; @Data public class AuthGroup { - private String modelId; + private Long modelId; private String name; private Integer groupId; private List authRules; diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java index f15f02131..fca4b9ade 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java @@ -7,13 +7,13 @@ import lombok.ToString; @ToString public class AuthRes { - private String modelId; + private Long modelId; private String name; public AuthRes() { } - public AuthRes(String modelId, String name) { + public AuthRes(Long modelId, String name) { this.modelId = modelId; this.name = name; } diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java index 0c723c325..c1dc18207 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java @@ -15,5 +15,5 @@ public class QueryAuthResReq { private List resources; - private String modelId; + private Long modelId; } diff --git a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java index 25f7ba3b5..b3ba28775 100644 --- a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java +++ b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java @@ -13,7 +13,6 @@ import com.tencent.supersonic.auth.api.authorization.service.AuthService; import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -80,17 +79,14 @@ public class AuthServiceImpl implements AuthService { @Override public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, User user) { Set userOrgIds = userService.getUserAllOrgId(user.getName()); - if (!CollectionUtils.isEmpty(userOrgIds)) { - req.setDepartmentIds(new ArrayList<>(userOrgIds)); - } - List groups = getAuthGroups(req, user.getName()); + List groups = getAuthGroups(req.getModelId(), user.getName(), new ArrayList<>(userOrgIds)); AuthorizedResourceResp resource = new AuthorizedResourceResp(); - Map> authGroupsByModelId = groups.stream() + Map> authGroupsByModelId = groups.stream() .collect(Collectors.groupingBy(AuthGroup::getModelId)); - Map> reqAuthRes = req.getResources().stream() + Map> reqAuthRes = req.getResources().stream() .collect(Collectors.groupingBy(AuthRes::getModelId)); - for (String modelId : reqAuthRes.keySet()) { + for (Long modelId : reqAuthRes.keySet()) { List reqResourcesList = reqAuthRes.get(modelId); AuthResGrp rg = new AuthResGrp(); if (authGroupsByModelId.containsKey(modelId)) { @@ -113,7 +109,7 @@ public class AuthServiceImpl implements AuthService { } } - if (StringUtils.isNotEmpty(req.getModelId())) { + if (req.getModelId() != null) { List authGroups = authGroupsByModelId.get(req.getModelId()); if (!CollectionUtils.isEmpty(authGroups)) { for (AuthGroup group : authGroups) { @@ -130,17 +126,17 @@ public class AuthServiceImpl implements AuthService { return resource; } - private List getAuthGroups(QueryAuthResReq req, String userName) { + private List getAuthGroups(Long modelId, String userName, List departmentIds) { List groups = load().stream() .filter(group -> { - if (!Objects.equals(group.getModelId(), req.getModelId())) { + if (modelId != null && Objects.equals(group.getModelId(), modelId)) { return false; } if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers() .contains(userName)) { return true; } - for (String departmentId : req.getDepartmentIds()) { + for (String departmentId : departmentIds) { if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds()) && group.getAuthorizedDepartmentIds().contains(departmentId)) { return true; @@ -148,7 +144,7 @@ public class AuthServiceImpl implements AuthService { } return false; }).collect(Collectors.toList()); - log.info("user:{} department:{} authGroups:{}", userName, req.getDepartmentIds(), groups); + log.info("user:{} department:{} authGroups:{}", userName, departmentIds, groups); return groups; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/OptimizationConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/config/OptimizationConfig.java index 186e01215..fcdd2af65 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/OptimizationConfig.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/config/OptimizationConfig.java @@ -24,9 +24,6 @@ public class OptimizationConfig { @Value("${dimension.value.threshold}") private Double dimensionValueThresholdConfig; - @Value("${function.bonus.threshold}") - private Double functionBonusThreshold; - @Value("${long.text.threshold}") private Double longTextThreshold; @@ -36,9 +33,6 @@ public class OptimizationConfig { @Value("${query.text.length.threshold}") private Integer queryTextLengthThreshold; - @Value("${candidate.threshold}") - private Double candidateThreshold; - @Value("${use.s2SQL.switch:false}") private boolean useS2SqlSwitch; diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java index 2a62c6f89..846740392 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java @@ -13,4 +13,5 @@ public class Parameter { private String value; private String comment; private String dataType; + private String module; } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java b/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java index 49c554314..b6b62aca8 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java @@ -23,7 +23,7 @@ public class SysParameter { return StringUtils.join(admins, ","); } - public void setAdmin(String admin) { + public void setAdminList(String admin) { if (StringUtils.isNotBlank(admin)) { admins = Arrays.asList(admin.split(",")); } else { @@ -34,19 +34,38 @@ public class SysParameter { public void init() { parameters = Lists.newArrayList(); admins = Lists.newArrayList("admin"); - parameters.add(new Parameter("llm.model.name", "gpt4", "模型名称", "string")); - parameters.add(new Parameter("llm.api.key", "sk-afdasdasd", "模型密钥", "string")); - parameters.add(new Parameter("one.detection.size", "8", "一次探测个数", "number")); - parameters.add(new Parameter("one.detection.max.size", "20", "阈值", "number")); - parameters.add(new Parameter("metric.dimension.min.threshold", "0.3", "指标名、维度名最小文本相似度", "number")); - parameters.add(new Parameter("metric.dimension.threshold", "0.3", "指标名、维度名文本相似度", "number")); - parameters.add(new Parameter("dimension.value.threshold", "0.5", "维度值最小文本相似度", "number")); - parameters.add(new Parameter("embedding.mapper.word.min", "0.3", "用于向量召回最小的文本长度", "number")); - parameters.add(new Parameter("embedding.mapper.word.max", "0.3", "用于向量召回最大的文本长度", "number")); - parameters.add(new Parameter("embedding.mapper.batch", "0.3", "批量向量召回文本请求个数", "number")); - parameters.add(new Parameter("embedding.mapper.number", "0.3", "批量向量召回文本返回结果个数", "number")); - parameters.add(new Parameter("embedding.mapper.distance.threshold", "0.3", "Mapper阶段向量召回相似度阈值", "number")); - parameters.add(new Parameter("use.s2SQL.switch", "true", "是否打开S2SQL开关", "bool")); + parameters.add(new Parameter("llm.model.name", "gpt4", + "模型名称(大语言模型相关配置)", "string", "大语言模型相关配置")); + parameters.add(new Parameter("llm.api.key", "sk-afdasdasd", + "模型密钥(大语言模型相关配置)", "string", "大语言模型相关配置")); + parameters.add(new Parameter("one.detection.size", "8", + "一次探测个数(hanlp相关配置)", "number", "hanlp相关配置")); + parameters.add(new Parameter("one.detection.max.size", "20", + "一次探测最大个数(hanlp相关配置)", "number", "hanlp相关配置")); + parameters.add(new Parameter("metric.dimension.min.threshold", "0.3", + "指标名、维度名最小文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置")); + parameters.add(new Parameter("metric.dimension.threshold", "0.3", + "指标名、维度名文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置")); + parameters.add(new Parameter("dimension.value.threshold", "0.5", + "维度值最小文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置")); + parameters.add(new Parameter("query.text.length.threshold", "0.5", + "文本长短阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置")); + parameters.add(new Parameter("short.text.threshold", "0.5", + "短文本匹配阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置")); + parameters.add(new Parameter("long.text.threshold", "0.5", + "长文本匹配阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置")); + parameters.add(new Parameter("embedding.mapper.word.min", + "0.3", "用于向量召回最小的文本长度(向量召回mapper相关配置)", "number", "向量召回mapper相关配置")); + parameters.add(new Parameter("embedding.mapper.word.max", "0.3", + "用于向量召回最大的文本长度(向量召回mapper相关配置)", "number", "向量召回mapper相关配置")); + parameters.add(new Parameter("embedding.mapper.batch", "0.3", + "批量向量召回文本请求个数(向量召回mapper相关配置)", "number", "向量召回mapper相关配置")); + parameters.add(new Parameter("embedding.mapper.number", "0.3", + "批量向量召回文本返回结果个数(向量召回mapper相关配置)", "number", "向量召回mapper相关配置")); + parameters.add(new Parameter("embedding.mapper.distance.threshold", + "0.3", "Mapper阶段向量召回相似度阈值(向量召回mapper相关配置)", "number", "向量召回mapper相关配置")); + parameters.add(new Parameter("use.s2SQL.switch", "true", + "是否打开S2SQL转换开关(S2SQL相关配置)", "bool", "S2SQL相关配置")); } } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/impl/SysParameterServiceImpl.java b/common/src/main/java/com/tencent/supersonic/common/service/impl/SysParameterServiceImpl.java index 97c00e198..7fefcd428 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/impl/SysParameterServiceImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/SysParameterServiceImpl.java @@ -6,7 +6,6 @@ import com.tencent.supersonic.common.pojo.SysParameter; import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO; import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper; import com.tencent.supersonic.common.service.SysParameterService; -import com.tencent.supersonic.common.util.BeanMapper; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import java.util.List; @@ -26,21 +25,21 @@ public class SysParameterServiceImpl @Override public void save(SysParameter sysParameter) { - SysParameterDO chatParameterDO = convert(sysParameter); - saveOrUpdate(chatParameterDO); + SysParameterDO sysParameterDO = convert(sysParameter); + saveOrUpdate(sysParameterDO); } private SysParameter convert(SysParameterDO sysParameterDO) { - SysParameter chatParameter = new SysParameter(); - BeanMapper.mapper(sysParameterDO, chatParameter); - chatParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class)); - chatParameter.setAdmin(sysParameterDO.getAdmin()); - return chatParameter; + SysParameter sysParameter = new SysParameter(); + sysParameter.setId(sysParameterDO.getId()); + sysParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class)); + sysParameter.setAdminList(sysParameterDO.getAdmin()); + return sysParameter; } private SysParameterDO convert(SysParameter sysParameter) { SysParameterDO sysParameterDO = new SysParameterDO(); - BeanMapper.mapper(sysParameter, sysParameterDO); + sysParameterDO.setId(sysParameter.getId()); sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters())); sysParameterDO.setAdmin(sysParameter.getAdmin()); return sysParameterDO; diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java index 1bbebcd8b..985b45c17 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java @@ -135,14 +135,16 @@ public class DateUtils { List datesInRange = new ArrayList<>(); LocalDate currentDate = startDate; - + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM"); while (!currentDate.isAfter(endDate)) { - datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE)); if (Constants.MONTH.equals(period)) { + datesInRange.add(currentDate.format(formatter)); currentDate = currentDate.plusMonths(1); } else if (Constants.WEEK.equals(period)) { + datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE)); currentDate = currentDate.plusWeeks(1); } else { + datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE)); currentDate = currentDate.plusDays(1); } } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java index 08a1ec2df..0882b1f45 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java @@ -300,7 +300,7 @@ public class LoadModelDataDemo implements CommandLineRunner { public void addAuthGroup_1() { AuthGroup authGroupReq = new AuthGroup(); - authGroupReq.setModelId("1"); + authGroupReq.setModelId(1L); authGroupReq.setName("admin-permission"); List authRules = new ArrayList<>(); @@ -317,7 +317,7 @@ public class LoadModelDataDemo implements CommandLineRunner { public void addAuthGroup_2() { AuthGroup authGroupReq = new AuthGroup(); - authGroupReq.setModelId("1"); + authGroupReq.setModelId(1L); authGroupReq.setName("tom_sales_permission"); List authRules = new ArrayList<>(); diff --git a/launchers/standalone/src/main/resources/db/sql-update.sql b/launchers/standalone/src/main/resources/db/sql-update.sql index ec53ed8e1..207a57ba2 100644 --- a/launchers/standalone/src/main/resources/db/sql-update.sql +++ b/launchers/standalone/src/main/resources/db/sql-update.sql @@ -87,4 +87,7 @@ CREATE TABLE s2_sys_parameter id int primary key AUTO_INCREMENT COMMENT '主键id', admin varchar(500) COMMENT '系统管理员', parameters text null COMMENT '配置项' -); \ No newline at end of file +); + +--20231114 +alter table s2_chat_config add column `llm_examples` text COMMENT 'llm examples'; \ No newline at end of file diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java index 961857557..86eb07d82 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java @@ -106,29 +106,31 @@ public class AuthCommonService { return highSensitiveCols; } - public AuthorizedResourceResp getAuthorizedResource(User user, Long domainId, + public AuthorizedResourceResp getAuthorizedResource(User user, Long modelId, Set sensitiveResReq) { List resourceReqList = new ArrayList<>(); - sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res))); + sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelId, res))); QueryAuthResReq queryAuthResReq = new QueryAuthResReq(); queryAuthResReq.setResources(resourceReqList); - queryAuthResReq.setModelId(domainId + ""); + queryAuthResReq.setModelId(modelId); AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user); - log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId, + log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelId, authorizedResource); return authorizedResource; } + private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) { log.info("queryAuthResReq:{}", queryAuthResReq); return authService.queryAuthorizedResources(queryAuthResReq, user); } - public Set getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long domainId) { + + public Set getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) { Set resAuthName = new HashSet<>(); List authResGrpList = authorizedResource.getResources(); authResGrpList.stream().forEach(authResGrp -> { List cols = authResGrp.getGroup(); if (!CollectionUtils.isEmpty(cols)) { - cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getModelId()))) + cols.stream().filter(col -> modelId.equals(col.getModelId())) .forEach(col -> resAuthName.add(col.getName())); } @@ -136,6 +138,7 @@ public class AuthCommonService { log.info("resAuthName:{}", resAuthName); return resAuthName; } + public boolean allSensitiveResReqIsOk(Set sensitiveResReq, Set resAuthSet) { if (resAuthSet.containsAll(sensitiveResReq)) { return true; diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java index c3d9eb508..b2db92e67 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java @@ -149,7 +149,7 @@ public class DownloadServiceImpl implements DownloadService { List groups = dimensions.stream().map(DimensionResp::getBizName).collect(Collectors.toList()); List dateList = getDateList(dateConf); List> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), dateList, - metricSchemaResp.getBizName(), groups); + metricSchemaResp.getBizName(), groups, dateConf); List> headers = buildHeader(dimensions, dateList); List> data = buildData(headers, getDimensionNameMap(dimensions), dataTransformed, metricSchemaResp); @@ -202,7 +202,7 @@ public class DownloadServiceImpl implements DownloadService { queryStructReq.setAggregators(Lists.newArrayList(aggregator)); queryStructReq.setDateInfo(dateConf); queryStructReq.setModelId(metricResp.getModelId()); - return queryService.queryByStruct(queryStructReq, user); + return queryService.queryByStructWithAuth(queryStructReq, user); } private String getTimeDimension(DateConf dateConf) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java index 26f698d49..85ca5607e 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java @@ -244,13 +244,13 @@ public class DataPermissionAOP { return false; } - private Set getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long domainId) { + private Set getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) { Set resAuthName = new HashSet<>(); List authResGrpList = authorizedResource.getResources(); authResGrpList.stream().forEach(authResGrp -> { List cols = authResGrp.getGroup(); if (!CollectionUtils.isEmpty(cols)) { - cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getModelId()))) + cols.stream().filter(col -> modelId.equals(col.getModelId())) .forEach(col -> resAuthName.add(col.getName())); } @@ -262,10 +262,10 @@ public class DataPermissionAOP { private AuthorizedResourceResp getAuthorizedResource(User user, Long domainId, Set sensitiveResReq) { List resourceReqList = new ArrayList<>(); - sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res))); + sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId, res))); QueryAuthResReq queryAuthResReq = new QueryAuthResReq(); queryAuthResReq.setResources(resourceReqList); - queryAuthResReq.setModelId(domainId + ""); + queryAuthResReq.setModelId(domainId); AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user); log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId, authorizedResource); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java index 451748a84..e4ba8b3bc 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java @@ -1,6 +1,8 @@ package com.tencent.supersonic.semantic.query.utils; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; @@ -12,7 +14,8 @@ import java.util.stream.Collectors; public class DataTransformUtils { public static List> transform(List> originalData, - List dateList, String metric, List groups) { + List dateList, String metric, List groups, + DateConf dateConf) { List> transposedData = new ArrayList<>(); for (Map originalRow : originalData) { Map transposedRow = new HashMap<>(); @@ -21,7 +24,7 @@ public class DataTransformUtils { transposedRow.put(key, originalRow.get(key)); } } - transposedRow.put(String.valueOf(originalRow.get(TimeDimensionEnum.DAY.getName())), + transposedRow.put(String.valueOf(originalRow.get(getTimeDimension(dateConf))), originalRow.get(metric)); transposedData.add(transposedRow); } @@ -55,4 +58,14 @@ public class DataTransformUtils { return StringUtils.join(values, "_"); } + private static String getTimeDimension(DateConf dateConf) { + if (Constants.MONTH.equals(dateConf.getPeriod())) { + return TimeDimensionEnum.MONTH.getName(); + } else if (Constants.WEEK.equals(dateConf.getPeriod())) { + return TimeDimensionEnum.WEEK.getName(); + } else { + return TimeDimensionEnum.DAY.getName(); + } + } + } diff --git a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java index 6dfa85b5c..187d9632c 100644 --- a/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java +++ b/semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.semantic.query.utils; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.DateConf; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -23,7 +24,8 @@ class DataTransformUtilsTest { List dateList = Lists.newArrayList("2023/10/11", "2023/10/12", "2023/10/13", "2023/10/14", "2023/10/15"); String metric = "m1"; - List> resultData = DataTransformUtils.transform(inputData, dateList, metric, groups); + List> resultData = DataTransformUtils.transform(inputData, dateList, + metric, groups, new DateConf()); Assertions.assertEquals(3, resultData.size()); }