topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 12:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement) (common) add sys parameter setting (#384)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-14 21:41:59 +08:00
committed by GitHub
parent 3271db4ca6
commit 36c8938ff7
16 changed files with 98 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java
View File

@@ -6,7 +6,7 @@ import lombok.Data;
@Data @Data
public class AuthGroup { public class AuthGroup {
private String modelId; private Long modelId;
private String name; private String name;
private Integer groupId; private Integer groupId;
private List<AuthRule> authRules; private List<AuthRule> authRules;

4
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java
View File

@@ -7,13 +7,13 @@ import lombok.ToString;
@ToString @ToString
public class AuthRes { public class AuthRes {
private String modelId; private Long modelId;
private String name; private String name;
public AuthRes() { public AuthRes() {
} }
public AuthRes(String modelId, String name) { public AuthRes(Long modelId, String name) {
this.modelId = modelId; this.modelId = modelId;
this.name = name; this.name = name;
} }

2
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java
View File

@@ -15,5 +15,5 @@ public class QueryAuthResReq {
private List<AuthRes> resources; private List<AuthRes> resources;
private String modelId; private Long modelId;
} }

22
auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java
View File

@@ -13,7 +13,6 @@ import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup; import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
@@ -80,17 +79,14 @@ public class AuthServiceImpl implements AuthService {
@Override @Override
public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, User user) { public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, User user) {
Set<String> userOrgIds = userService.getUserAllOrgId(user.getName()); Set<String> userOrgIds = userService.getUserAllOrgId(user.getName());
if (!CollectionUtils.isEmpty(userOrgIds)) { List<AuthGroup> groups = getAuthGroups(req.getModelId(), user.getName(), new ArrayList<>(userOrgIds));
req.setDepartmentIds(new ArrayList<>(userOrgIds));
}
List<AuthGroup> groups = getAuthGroups(req, user.getName());
AuthorizedResourceResp resource = new AuthorizedResourceResp(); AuthorizedResourceResp resource = new AuthorizedResourceResp();
Map<String, List<AuthGroup>> authGroupsByModelId = groups.stream() Map<Long, List<AuthGroup>> authGroupsByModelId = groups.stream()
.collect(Collectors.groupingBy(AuthGroup::getModelId)); .collect(Collectors.groupingBy(AuthGroup::getModelId));
Map<String, List<AuthRes>> reqAuthRes = req.getResources().stream() Map<Long, List<AuthRes>> reqAuthRes = req.getResources().stream()
.collect(Collectors.groupingBy(AuthRes::getModelId)); .collect(Collectors.groupingBy(AuthRes::getModelId));
for (String modelId : reqAuthRes.keySet()) { for (Long modelId : reqAuthRes.keySet()) {
List<AuthRes> reqResourcesList = reqAuthRes.get(modelId); List<AuthRes> reqResourcesList = reqAuthRes.get(modelId);
AuthResGrp rg = new AuthResGrp(); AuthResGrp rg = new AuthResGrp();
if (authGroupsByModelId.containsKey(modelId)) { if (authGroupsByModelId.containsKey(modelId)) {
@@ -113,7 +109,7 @@ public class AuthServiceImpl implements AuthService {
} }
} }
if (StringUtils.isNotEmpty(req.getModelId())) { if (req.getModelId() != null) {
List<AuthGroup> authGroups = authGroupsByModelId.get(req.getModelId()); List<AuthGroup> authGroups = authGroupsByModelId.get(req.getModelId());
if (!CollectionUtils.isEmpty(authGroups)) { if (!CollectionUtils.isEmpty(authGroups)) {
for (AuthGroup group : authGroups) { for (AuthGroup group : authGroups) {
@@ -130,17 +126,17 @@ public class AuthServiceImpl implements AuthService {
return resource; return resource;
} }
private List<AuthGroup> getAuthGroups(QueryAuthResReq req, String userName) { private List<AuthGroup> getAuthGroups(Long modelId, String userName, List<String> departmentIds) {
List<AuthGroup> groups = load().stream() List<AuthGroup> groups = load().stream()
.filter(group -> { .filter(group -> {
if (!Objects.equals(group.getModelId(), req.getModelId())) { if (modelId != null && Objects.equals(group.getModelId(), modelId)) {
return false; return false;
} }
if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers() if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers()
.contains(userName)) { .contains(userName)) {
return true; return true;
} }
for (String departmentId : req.getDepartmentIds()) { for (String departmentId : departmentIds) {
if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds()) if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds())
&& group.getAuthorizedDepartmentIds().contains(departmentId)) { && group.getAuthorizedDepartmentIds().contains(departmentId)) {
return true; return true;
@@ -148,7 +144,7 @@ public class AuthServiceImpl implements AuthService {
} }
return false; return false;
}).collect(Collectors.toList()); }).collect(Collectors.toList());
log.info("user:{} department:{} authGroups:{}", userName, req.getDepartmentIds(), groups); log.info("user:{} department:{} authGroups:{}", userName, departmentIds, groups);
return groups; return groups;
} }

6
chat/core/src/main/java/com/tencent/supersonic/chat/config/OptimizationConfig.java
View File

@@ -24,9 +24,6 @@ public class OptimizationConfig {
@Value("${dimension.value.threshold}") @Value("${dimension.value.threshold}")
private Double dimensionValueThresholdConfig; private Double dimensionValueThresholdConfig;
@Value("${function.bonus.threshold}")
private Double functionBonusThreshold;
@Value("${long.text.threshold}") @Value("${long.text.threshold}")
private Double longTextThreshold; private Double longTextThreshold;
@@ -36,9 +33,6 @@ public class OptimizationConfig {
@Value("${query.text.length.threshold}") @Value("${query.text.length.threshold}")
private Integer queryTextLengthThreshold; private Integer queryTextLengthThreshold;
@Value("${candidate.threshold}")
private Double candidateThreshold;
@Value("${use.s2SQL.switch:false}") @Value("${use.s2SQL.switch:false}")
private boolean useS2SqlSwitch; private boolean useS2SqlSwitch;

1
common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java
View File

@@ -13,4 +13,5 @@ public class Parameter {
private String value; private String value;
private String comment; private String comment;
private String dataType; private String dataType;
private String module;
} }

47
common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java
View File

@@ -23,7 +23,7 @@ public class SysParameter {
return StringUtils.join(admins, ","); return StringUtils.join(admins, ",");
} }
public void setAdmin(String admin) { public void setAdminList(String admin) {
if (StringUtils.isNotBlank(admin)) { if (StringUtils.isNotBlank(admin)) {
admins = Arrays.asList(admin.split(",")); admins = Arrays.asList(admin.split(","));
} else { } else {
@@ -34,19 +34,38 @@ public class SysParameter {
public void init() { public void init() {
parameters = Lists.newArrayList(); parameters = Lists.newArrayList();
admins = Lists.newArrayList("admin"); admins = Lists.newArrayList("admin");
parameters.add(new Parameter("llm.model.name", "gpt4", "模型名称", "string")); parameters.add(new Parameter("llm.model.name", "gpt4",
parameters.add(new Parameter("llm.api.key", "sk-afdasdasd", "模型密钥", "string")); "模型名称(大语言模型相关配置)", "string", "大语言模型相关配置"));
parameters.add(new Parameter("one.detection.size", "8", "一次探测个数", "number")); parameters.add(new Parameter("llm.api.key", "sk-afdasdasd",
parameters.add(new Parameter("one.detection.max.size", "20", "阈值", "number")); "模型密钥(大语言模型相关配置)", "string", "大语言模型相关配置"));
parameters.add(new Parameter("metric.dimension.min.threshold", "0.3", "指标名、维度名最小文本相似度", "number")); parameters.add(new Parameter("one.detection.size", "8",
parameters.add(new Parameter("metric.dimension.threshold", "0.3", "指标名、维度名文本相似度", "number")); "一次探测个数(hanlp相关配置)", "number", "hanlp相关配置"));
parameters.add(new Parameter("dimension.value.threshold", "0.5", "维度值最小文本相似度", "number")); parameters.add(new Parameter("one.detection.max.size", "20",
parameters.add(new Parameter("embedding.mapper.word.min", "0.3", "用于向量召回最小的文本长度", "number")); "一次探测最大个数(hanlp相关配置)", "number", "hanlp相关配置"));
parameters.add(new Parameter("embedding.mapper.word.max", "0.3", "用于向量召回最大的文本长度", "number")); parameters.add(new Parameter("metric.dimension.min.threshold", "0.3",
parameters.add(new Parameter("embedding.mapper.batch", "0.3", "批量向量召回文本请求个数", "number")); "指标名、维度名最小文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
parameters.add(new Parameter("embedding.mapper.number", "0.3", "批量向量召回文本返回结果个数", "number")); parameters.add(new Parameter("metric.dimension.threshold", "0.3",
parameters.add(new Parameter("embedding.mapper.distance.threshold", "0.3", "Mapper阶段向量召回相似度阈值", "number")); "指标名、维度名文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
parameters.add(new Parameter("use.s2SQL.switch", "true", "是否打开S2SQL开关", "bool")); parameters.add(new Parameter("dimension.value.threshold", "0.5",
"维度值最小文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
parameters.add(new Parameter("query.text.length.threshold", "0.5",
"文本长短阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
parameters.add(new Parameter("short.text.threshold", "0.5",
"短文本匹配阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
parameters.add(new Parameter("long.text.threshold", "0.5",
"长文本匹配阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
parameters.add(new Parameter("embedding.mapper.word.min",
"0.3", "用于向量召回最小的文本长度(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.word.max", "0.3",
"用于向量召回最大的文本长度(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.batch", "0.3",
"批量向量召回文本请求个数(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.number", "0.3",
"批量向量召回文本返回结果个数(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.distance.threshold",
"0.3", "Mapper阶段向量召回相似度阈值(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("use.s2SQL.switch", "true",
"是否打开S2SQL转换开关(S2SQL相关配置)", "bool", "S2SQL相关配置"));
} }
} }

17
common/src/main/java/com/tencent/supersonic/common/service/impl/SysParameterServiceImpl.java
View File

@@ -6,7 +6,6 @@ import com.tencent.supersonic.common.pojo.SysParameter;
import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO; import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper; import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper;
import com.tencent.supersonic.common.service.SysParameterService; import com.tencent.supersonic.common.service.SysParameterService;
import com.tencent.supersonic.common.util.BeanMapper;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.List; import java.util.List;
@@ -26,21 +25,21 @@ public class SysParameterServiceImpl
@Override @Override
public void save(SysParameter sysParameter) { public void save(SysParameter sysParameter) {
SysParameterDO chatParameterDO = convert(sysParameter); SysParameterDO sysParameterDO = convert(sysParameter);
saveOrUpdate(chatParameterDO); saveOrUpdate(sysParameterDO);
} }
private SysParameter convert(SysParameterDO sysParameterDO) { private SysParameter convert(SysParameterDO sysParameterDO) {
SysParameter chatParameter = new SysParameter(); SysParameter sysParameter = new SysParameter();
BeanMapper.mapper(sysParameterDO, chatParameter); sysParameter.setId(sysParameterDO.getId());
chatParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class)); sysParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class));
chatParameter.setAdmin(sysParameterDO.getAdmin()); sysParameter.setAdminList(sysParameterDO.getAdmin());
return chatParameter; return sysParameter;
} }
private SysParameterDO convert(SysParameter sysParameter) { private SysParameterDO convert(SysParameter sysParameter) {
SysParameterDO sysParameterDO = new SysParameterDO(); SysParameterDO sysParameterDO = new SysParameterDO();
BeanMapper.mapper(sysParameter, sysParameterDO); sysParameterDO.setId(sysParameter.getId());
sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters())); sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters()));
sysParameterDO.setAdmin(sysParameter.getAdmin()); sysParameterDO.setAdmin(sysParameter.getAdmin());
return sysParameterDO; return sysParameterDO;

6
common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
View File

@@ -135,14 +135,16 @@ public class DateUtils {
List<String> datesInRange = new ArrayList<>(); List<String> datesInRange = new ArrayList<>();
LocalDate currentDate = startDate; LocalDate currentDate = startDate;
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM");
while (!currentDate.isAfter(endDate)) { while (!currentDate.isAfter(endDate)) {
datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE));
if (Constants.MONTH.equals(period)) { if (Constants.MONTH.equals(period)) {
datesInRange.add(currentDate.format(formatter));
currentDate = currentDate.plusMonths(1); currentDate = currentDate.plusMonths(1);
} else if (Constants.WEEK.equals(period)) { } else if (Constants.WEEK.equals(period)) {
datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE));
currentDate = currentDate.plusWeeks(1); currentDate = currentDate.plusWeeks(1);
} else { } else {
datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE));
currentDate = currentDate.plusDays(1); currentDate = currentDate.plusDays(1);
} }
} }

4
launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java
View File

@@ -300,7 +300,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
public void addAuthGroup_1() { public void addAuthGroup_1() {
AuthGroup authGroupReq = new AuthGroup(); AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId("1"); authGroupReq.setModelId(1L);
authGroupReq.setName("admin-permission"); authGroupReq.setName("admin-permission");
List<AuthRule> authRules = new ArrayList<>(); List<AuthRule> authRules = new ArrayList<>();
@@ -317,7 +317,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
public void addAuthGroup_2() { public void addAuthGroup_2() {
AuthGroup authGroupReq = new AuthGroup(); AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId("1"); authGroupReq.setModelId(1L);
authGroupReq.setName("tom_sales_permission"); authGroupReq.setName("tom_sales_permission");
List<AuthRule> authRules = new ArrayList<>(); List<AuthRule> authRules = new ArrayList<>();

3
launchers/standalone/src/main/resources/db/sql-update.sql
View File

@@ -88,3 +88,6 @@ CREATE TABLE s2_sys_parameter
admin varchar(500) COMMENT '系统管理员', admin varchar(500) COMMENT '系统管理员',
parameters text null COMMENT '配置项' parameters text null COMMENT '配置项'
); );
--20231114
alter table s2_chat_config add column `llm_examples` text COMMENT 'llm examples';

15
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java
View File

@@ -106,29 +106,31 @@ public class AuthCommonService {
return highSensitiveCols; return highSensitiveCols;
} }
public AuthorizedResourceResp getAuthorizedResource(User user, Long domainId, public AuthorizedResourceResp getAuthorizedResource(User user, Long modelId,
Set<String> sensitiveResReq) { Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>(); List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res))); sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelId, res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq(); QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList); queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelId(domainId + ""); queryAuthResReq.setModelId(modelId);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user); AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId, log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelId,
authorizedResource); authorizedResource);
return authorizedResource; return authorizedResource;
} }
private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) { private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) {
log.info("queryAuthResReq:{}", queryAuthResReq); log.info("queryAuthResReq:{}", queryAuthResReq);
return authService.queryAuthorizedResources(queryAuthResReq, user); return authService.queryAuthorizedResources(queryAuthResReq, user);
} }
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long domainId) {
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) {
Set<String> resAuthName = new HashSet<>(); Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources(); List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> { authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup(); List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) { if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getModelId()))) cols.stream().filter(col -> modelId.equals(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName())); .forEach(col -> resAuthName.add(col.getName()));
} }
@@ -136,6 +138,7 @@ public class AuthCommonService {
log.info("resAuthName:{}", resAuthName); log.info("resAuthName:{}", resAuthName);
return resAuthName; return resAuthName;
} }
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) { public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
if (resAuthSet.containsAll(sensitiveResReq)) { if (resAuthSet.containsAll(sensitiveResReq)) {
return true; return true;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java
View File

@@ -149,7 +149,7 @@ public class DownloadServiceImpl implements DownloadService {
List<String> groups = dimensions.stream().map(DimensionResp::getBizName).collect(Collectors.toList()); List<String> groups = dimensions.stream().map(DimensionResp::getBizName).collect(Collectors.toList());
List<String> dateList = getDateList(dateConf); List<String> dateList = getDateList(dateConf);
List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), dateList, List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), dateList,
metricSchemaResp.getBizName(), groups); metricSchemaResp.getBizName(), groups, dateConf);
List<List<String>> headers = buildHeader(dimensions, dateList); List<List<String>> headers = buildHeader(dimensions, dateList);
List<List<String>> data = buildData(headers, getDimensionNameMap(dimensions), List<List<String>> data = buildData(headers, getDimensionNameMap(dimensions),
dataTransformed, metricSchemaResp); dataTransformed, metricSchemaResp);
@@ -202,7 +202,7 @@ public class DownloadServiceImpl implements DownloadService {
queryStructReq.setAggregators(Lists.newArrayList(aggregator)); queryStructReq.setAggregators(Lists.newArrayList(aggregator));
queryStructReq.setDateInfo(dateConf); queryStructReq.setDateInfo(dateConf);
queryStructReq.setModelId(metricResp.getModelId()); queryStructReq.setModelId(metricResp.getModelId());
return queryService.queryByStruct(queryStructReq, user); return queryService.queryByStructWithAuth(queryStructReq, user);
} }
private String getTimeDimension(DateConf dateConf) { private String getTimeDimension(DateConf dateConf) {

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -244,13 +244,13 @@ public class DataPermissionAOP {
return false; return false;
} }
private Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long domainId) { private Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) {
Set<String> resAuthName = new HashSet<>(); Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources(); List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> { authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup(); List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) { if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getModelId()))) cols.stream().filter(col -> modelId.equals(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName())); .forEach(col -> resAuthName.add(col.getName()));
} }
@@ -262,10 +262,10 @@ public class DataPermissionAOP {
private AuthorizedResourceResp getAuthorizedResource(User user, Long domainId, private AuthorizedResourceResp getAuthorizedResource(User user, Long domainId,
Set<String> sensitiveResReq) { Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>(); List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res))); sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId, res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq(); QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList); queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelId(domainId + ""); queryAuthResReq.setModelId(domainId);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user); AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId, log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId,
authorizedResource); authorizedResource);

17
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java
View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.semantic.query.utils; package com.tencent.supersonic.semantic.query.utils;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
@@ -12,7 +14,8 @@ import java.util.stream.Collectors;
public class DataTransformUtils { public class DataTransformUtils {
public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData, public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData,
List<String> dateList, String metric, List<String> groups) { List<String> dateList, String metric, List<String> groups,
DateConf dateConf) {
List<Map<String, Object>> transposedData = new ArrayList<>(); List<Map<String, Object>> transposedData = new ArrayList<>();
for (Map<String, Object> originalRow : originalData) { for (Map<String, Object> originalRow : originalData) {
Map<String, Object> transposedRow = new HashMap<>(); Map<String, Object> transposedRow = new HashMap<>();
@@ -21,7 +24,7 @@ public class DataTransformUtils {
transposedRow.put(key, originalRow.get(key)); transposedRow.put(key, originalRow.get(key));
} }
} }
transposedRow.put(String.valueOf(originalRow.get(TimeDimensionEnum.DAY.getName())), transposedRow.put(String.valueOf(originalRow.get(getTimeDimension(dateConf))),
originalRow.get(metric)); originalRow.get(metric));
transposedData.add(transposedRow); transposedData.add(transposedRow);
} }
@@ -55,4 +58,14 @@ public class DataTransformUtils {
return StringUtils.join(values, "_"); return StringUtils.join(values, "_");
} }
private static String getTimeDimension(DateConf dateConf) {
if (Constants.MONTH.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.MONTH.getName();
} else if (Constants.WEEK.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.WEEK.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
}
} }

4
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.semantic.query.utils; package com.tencent.supersonic.semantic.query.utils;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DateConf;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.util.ArrayList; import java.util.ArrayList;
@@ -23,7 +24,8 @@ class DataTransformUtilsTest {
List<String> dateList = Lists.newArrayList("2023/10/11", "2023/10/12", List<String> dateList = Lists.newArrayList("2023/10/11", "2023/10/12",
"2023/10/13", "2023/10/14", "2023/10/15"); "2023/10/13", "2023/10/14", "2023/10/15");
String metric = "m1"; String metric = "m1";
List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData, dateList, metric, groups); List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData, dateList,
metric, groups, new DateConf());
Assertions.assertEquals(3, resultData.size()); Assertions.assertEquals(3, resultData.size());
} }