topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement) (common) add sys parameter setting (#384)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-14 21:41:59 +08:00
committed by GitHub
parent 3271db4ca6
commit 36c8938ff7
16 changed files with 98 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthGroup.java
View File

@@ -6,7 +6,7 @@ import lombok.Data;
@Data
public class AuthGroup {
private String modelId;
private Long modelId;
private String name;
private Integer groupId;
private List<AuthRule> authRules;

4
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthRes.java
View File

@@ -7,13 +7,13 @@ import lombok.ToString;
@ToString
public class AuthRes {
private String modelId;
private Long modelId;
private String name;
public AuthRes() {
}
public AuthRes(String modelId, String name) {
public AuthRes(Long modelId, String name) {
this.modelId = modelId;
this.name = name;
}

2
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java
View File

@@ -15,5 +15,5 @@ public class QueryAuthResReq {
private List<AuthRes> resources;
private String modelId;
private Long modelId;
}

22
auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java
View File

@@ -13,7 +13,6 @@ import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -80,17 +79,14 @@ public class AuthServiceImpl implements AuthService {
@Override
public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, User user) {
Set<String> userOrgIds = userService.getUserAllOrgId(user.getName());
if (!CollectionUtils.isEmpty(userOrgIds)) {
req.setDepartmentIds(new ArrayList<>(userOrgIds));
}
List<AuthGroup> groups = getAuthGroups(req, user.getName());
List<AuthGroup> groups = getAuthGroups(req.getModelId(), user.getName(), new ArrayList<>(userOrgIds));
AuthorizedResourceResp resource = new AuthorizedResourceResp();
Map<String, List<AuthGroup>> authGroupsByModelId = groups.stream()
Map<Long, List<AuthGroup>> authGroupsByModelId = groups.stream()
.collect(Collectors.groupingBy(AuthGroup::getModelId));
Map<String, List<AuthRes>> reqAuthRes = req.getResources().stream()
Map<Long, List<AuthRes>> reqAuthRes = req.getResources().stream()
.collect(Collectors.groupingBy(AuthRes::getModelId));
for (String modelId : reqAuthRes.keySet()) {
for (Long modelId : reqAuthRes.keySet()) {
List<AuthRes> reqResourcesList = reqAuthRes.get(modelId);
AuthResGrp rg = new AuthResGrp();
if (authGroupsByModelId.containsKey(modelId)) {
@@ -113,7 +109,7 @@ public class AuthServiceImpl implements AuthService {
}
}
if (StringUtils.isNotEmpty(req.getModelId())) {
if (req.getModelId() != null) {
List<AuthGroup> authGroups = authGroupsByModelId.get(req.getModelId());
if (!CollectionUtils.isEmpty(authGroups)) {
for (AuthGroup group : authGroups) {
@@ -130,17 +126,17 @@ public class AuthServiceImpl implements AuthService {
return resource;
}
private List<AuthGroup> getAuthGroups(QueryAuthResReq req, String userName) {
private List<AuthGroup> getAuthGroups(Long modelId, String userName, List<String> departmentIds) {
List<AuthGroup> groups = load().stream()
.filter(group -> {
if (!Objects.equals(group.getModelId(), req.getModelId())) {
if (modelId != null && Objects.equals(group.getModelId(), modelId)) {
return false;
}
if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers()
.contains(userName)) {
return true;
}
for (String departmentId : req.getDepartmentIds()) {
for (String departmentId : departmentIds) {
if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds())
&& group.getAuthorizedDepartmentIds().contains(departmentId)) {
return true;
@@ -148,7 +144,7 @@ public class AuthServiceImpl implements AuthService {
}
return false;
}).collect(Collectors.toList());
log.info("user:{} department:{} authGroups:{}", userName, req.getDepartmentIds(), groups);
log.info("user:{} department:{} authGroups:{}", userName, departmentIds, groups);
return groups;
}

6
chat/core/src/main/java/com/tencent/supersonic/chat/config/OptimizationConfig.java
View File

@@ -24,9 +24,6 @@ public class OptimizationConfig {
@Value("${dimension.value.threshold}")
private Double dimensionValueThresholdConfig;
@Value("${function.bonus.threshold}")
private Double functionBonusThreshold;
@Value("${long.text.threshold}")
private Double longTextThreshold;
@@ -36,9 +33,6 @@ public class OptimizationConfig {
@Value("${query.text.length.threshold}")
private Integer queryTextLengthThreshold;
@Value("${candidate.threshold}")
private Double candidateThreshold;
@Value("${use.s2SQL.switch:false}")
private boolean useS2SqlSwitch;

1
common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java
View File

@@ -13,4 +13,5 @@ public class Parameter {
private String value;
private String comment;
private String dataType;
private String module;
}

47
common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java
View File

@@ -23,7 +23,7 @@ public class SysParameter {
return StringUtils.join(admins, ",");
}
public void setAdmin(String admin) {
public void setAdminList(String admin) {
if (StringUtils.isNotBlank(admin)) {
admins = Arrays.asList(admin.split(","));
} else {
@@ -34,19 +34,38 @@ public class SysParameter {
public void init() {
parameters = Lists.newArrayList();
admins = Lists.newArrayList("admin");
parameters.add(new Parameter("llm.model.name", "gpt4", "模型名称", "string"));
parameters.add(new Parameter("llm.api.key", "sk-afdasdasd", "模型密钥", "string"));
parameters.add(new Parameter("one.detection.size", "8", "一次探测个数", "number"));
parameters.add(new Parameter("one.detection.max.size", "20", "阈值", "number"));
parameters.add(new Parameter("metric.dimension.min.threshold", "0.3", "指标名、维度名最小文本相似度", "number"));
parameters.add(new Parameter("metric.dimension.threshold", "0.3", "指标名、维度名文本相似度", "number"));
parameters.add(new Parameter("dimension.value.threshold", "0.5", "维度值最小文本相似度", "number"));
parameters.add(new Parameter("embedding.mapper.word.min", "0.3", "用于向量召回最小的文本长度", "number"));
parameters.add(new Parameter("embedding.mapper.word.max", "0.3", "用于向量召回最大的文本长度", "number"));
parameters.add(new Parameter("embedding.mapper.batch", "0.3", "批量向量召回文本请求个数", "number"));
parameters.add(new Parameter("embedding.mapper.number", "0.3", "批量向量召回文本返回结果个数", "number"));
parameters.add(new Parameter("embedding.mapper.distance.threshold", "0.3", "Mapper阶段向量召回相似度阈值", "number"));
parameters.add(new Parameter("use.s2SQL.switch", "true", "是否打开S2SQL开关", "bool"));
parameters.add(new Parameter("llm.model.name", "gpt4",
"模型名称(大语言模型相关配置)", "string", "大语言模型相关配置"));
parameters.add(new Parameter("llm.api.key", "sk-afdasdasd",
"模型密钥(大语言模型相关配置)", "string", "大语言模型相关配置"));
parameters.add(new Parameter("one.detection.size", "8",
"一次探测个数(hanlp相关配置)", "number", "hanlp相关配置"));
parameters.add(new Parameter("one.detection.max.size", "20",
"一次探测最大个数(hanlp相关配置)", "number", "hanlp相关配置"));
parameters.add(new Parameter("metric.dimension.min.threshold", "0.3",
"指标名、维度名最小文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
parameters.add(new Parameter("metric.dimension.threshold", "0.3",
"指标名、维度名文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
parameters.add(new Parameter("dimension.value.threshold", "0.5",
"维度值最小文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
parameters.add(new Parameter("query.text.length.threshold", "0.5",
"文本长短阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
parameters.add(new Parameter("short.text.threshold", "0.5",
"短文本匹配阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
parameters.add(new Parameter("long.text.threshold", "0.5",
"长文本匹配阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
parameters.add(new Parameter("embedding.mapper.word.min",
"0.3", "用于向量召回最小的文本长度(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.word.max", "0.3",
"用于向量召回最大的文本长度(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.batch", "0.3",
"批量向量召回文本请求个数(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.number", "0.3",
"批量向量召回文本返回结果个数(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.distance.threshold",
"0.3", "Mapper阶段向量召回相似度阈值(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("use.s2SQL.switch", "true",
"是否打开S2SQL转换开关(S2SQL相关配置)", "bool", "S2SQL相关配置"));
}
}

17
common/src/main/java/com/tencent/supersonic/common/service/impl/SysParameterServiceImpl.java
View File

@@ -6,7 +6,6 @@ import com.tencent.supersonic.common.pojo.SysParameter;
import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper;
import com.tencent.supersonic.common.service.SysParameterService;
import com.tencent.supersonic.common.util.BeanMapper;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
@@ -26,21 +25,21 @@ public class SysParameterServiceImpl
@Override
public void save(SysParameter sysParameter) {
SysParameterDO chatParameterDO = convert(sysParameter);
saveOrUpdate(chatParameterDO);
SysParameterDO sysParameterDO = convert(sysParameter);
saveOrUpdate(sysParameterDO);
}
private SysParameter convert(SysParameterDO sysParameterDO) {
SysParameter chatParameter = new SysParameter();
BeanMapper.mapper(sysParameterDO, chatParameter);
chatParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class));
chatParameter.setAdmin(sysParameterDO.getAdmin());
return chatParameter;
SysParameter sysParameter = new SysParameter();
sysParameter.setId(sysParameterDO.getId());
sysParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class));
sysParameter.setAdminList(sysParameterDO.getAdmin());
return sysParameter;
}
private SysParameterDO convert(SysParameter sysParameter) {
SysParameterDO sysParameterDO = new SysParameterDO();
BeanMapper.mapper(sysParameter, sysParameterDO);
sysParameterDO.setId(sysParameter.getId());
sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters()));
sysParameterDO.setAdmin(sysParameter.getAdmin());
return sysParameterDO;

6
common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
View File

@@ -135,14 +135,16 @@ public class DateUtils {
List<String> datesInRange = new ArrayList<>();
LocalDate currentDate = startDate;
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM");
while (!currentDate.isAfter(endDate)) {
datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE));
if (Constants.MONTH.equals(period)) {
datesInRange.add(currentDate.format(formatter));
currentDate = currentDate.plusMonths(1);
} else if (Constants.WEEK.equals(period)) {
datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE));
currentDate = currentDate.plusWeeks(1);
} else {
datesInRange.add(currentDate.format(DateTimeFormatter.ISO_DATE));
currentDate = currentDate.plusDays(1);
}
}

4
launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java
View File

@@ -300,7 +300,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
public void addAuthGroup_1() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId("1");
authGroupReq.setModelId(1L);
authGroupReq.setName("admin-permission");
List<AuthRule> authRules = new ArrayList<>();
@@ -317,7 +317,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
public void addAuthGroup_2() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId("1");
authGroupReq.setModelId(1L);
authGroupReq.setName("tom_sales_permission");
List<AuthRule> authRules = new ArrayList<>();

5
launchers/standalone/src/main/resources/db/sql-update.sql
View File

@@ -87,4 +87,7 @@ CREATE TABLE s2_sys_parameter
id int primary key AUTO_INCREMENT COMMENT '主键id',
admin varchar(500) COMMENT '系统管理员',
parameters text null COMMENT '配置项'
);
);
--20231114
alter table s2_chat_config add column `llm_examples` text COMMENT 'llm examples';

15
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java
View File

@@ -106,29 +106,31 @@ public class AuthCommonService {
return highSensitiveCols;
}
public AuthorizedResourceResp getAuthorizedResource(User user, Long domainId,
public AuthorizedResourceResp getAuthorizedResource(User user, Long modelId,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res)));
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelId, res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelId(domainId + "");
queryAuthResReq.setModelId(modelId);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId,
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelId,
authorizedResource);
return authorizedResource;
}
private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) {
log.info("queryAuthResReq:{}", queryAuthResReq);
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long domainId) {
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) {
Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getModelId())))
cols.stream().filter(col -> modelId.equals(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName()));
}
@@ -136,6 +138,7 @@ public class AuthCommonService {
log.info("resAuthName:{}", resAuthName);
return resAuthName;
}
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
if (resAuthSet.containsAll(sensitiveResReq)) {
return true;

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/DownloadServiceImpl.java
View File

@@ -149,7 +149,7 @@ public class DownloadServiceImpl implements DownloadService {
List<String> groups = dimensions.stream().map(DimensionResp::getBizName).collect(Collectors.toList());
List<String> dateList = getDateList(dateConf);
List<Map<String, Object>> dataTransformed = DataTransformUtils.transform(queryResult.getResultList(), dateList,
metricSchemaResp.getBizName(), groups);
metricSchemaResp.getBizName(), groups, dateConf);
List<List<String>> headers = buildHeader(dimensions, dateList);
List<List<String>> data = buildData(headers, getDimensionNameMap(dimensions),
dataTransformed, metricSchemaResp);
@@ -202,7 +202,7 @@ public class DownloadServiceImpl implements DownloadService {
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
queryStructReq.setDateInfo(dateConf);
queryStructReq.setModelId(metricResp.getModelId());
return queryService.queryByStruct(queryStructReq, user);
return queryService.queryByStructWithAuth(queryStructReq, user);
}
private String getTimeDimension(DateConf dateConf) {

8
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -244,13 +244,13 @@ public class DataPermissionAOP {
return false;
}
private Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long domainId) {
private Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, Long modelId) {
Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> domainId.equals(Long.parseLong(col.getModelId())))
cols.stream().filter(col -> modelId.equals(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName()));
}
@@ -262,10 +262,10 @@ public class DataPermissionAOP {
private AuthorizedResourceResp getAuthorizedResource(User user, Long domainId,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res)));
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId, res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelId(domainId + "");
queryAuthResReq.setModelId(domainId);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId,
authorizedResource);

17
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtils.java
View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.semantic.query.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
@@ -12,7 +14,8 @@ import java.util.stream.Collectors;
public class DataTransformUtils {
public static List<Map<String, Object>> transform(List<Map<String, Object>> originalData,
List<String> dateList, String metric, List<String> groups) {
List<String> dateList, String metric, List<String> groups,
DateConf dateConf) {
List<Map<String, Object>> transposedData = new ArrayList<>();
for (Map<String, Object> originalRow : originalData) {
Map<String, Object> transposedRow = new HashMap<>();
@@ -21,7 +24,7 @@ public class DataTransformUtils {
transposedRow.put(key, originalRow.get(key));
}
}
transposedRow.put(String.valueOf(originalRow.get(TimeDimensionEnum.DAY.getName())),
transposedRow.put(String.valueOf(originalRow.get(getTimeDimension(dateConf))),
originalRow.get(metric));
transposedData.add(transposedRow);
}
@@ -55,4 +58,14 @@ public class DataTransformUtils {
return StringUtils.join(values, "_");
}
private static String getTimeDimension(DateConf dateConf) {
if (Constants.MONTH.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.MONTH.getName();
} else if (Constants.WEEK.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.WEEK.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
}
}

4
semantic/query/src/test/java/com/tencent/supersonic/semantic/query/utils/DataTransformUtilsTest.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.semantic.query.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DateConf;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
@@ -23,7 +24,8 @@ class DataTransformUtilsTest {
List<String> dateList = Lists.newArrayList("2023/10/11", "2023/10/12",
"2023/10/13", "2023/10/14", "2023/10/15");
String metric = "m1";
List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData, dateList, metric, groups);
List<Map<String, Object>> resultData = DataTransformUtils.transform(inputData, dateList,
metric, groups, new DateConf());
Assertions.assertEquals(3, resultData.size());
}