topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](project) Parameters are uniformly obtained from system settings, removing optimization.properties, and modifying SysParameter parameters (#399)

Browse Source
This commit is contained in:
lexluo09
2023-11-17 18:11:07 +08:00
committed by GitHub
parent 8f19584ad7
commit d6a386ad03
5 changed files with 165 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java
View File

@@ -1,11 +1,14 @@
package com.tencent.supersonic.common.pojo;
import com.google.common.collect.Lists;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.List;
import retrofit2.http.HEAD;
@Data
public class SysParameter {
@@ -23,6 +26,15 @@ public class SysParameter {
return StringUtils.join(admins, ",");
}
public String getParameterByName(String name) {
if (StringUtils.isBlank(name)) {
return "";
}
Map<String, String> nameToValue = parameters.stream()
.collect(Collectors.toMap(a -> a.getName(), a -> a.getValue(), (k1, k2) -> k1));
return nameToValue.get(name);
}
public void setAdminList(String admin) {
if (StringUtils.isNotBlank(admin)) {
admins = Arrays.asList(admin.split(","));
@@ -34,40 +46,53 @@ public class SysParameter {
public void init() {
parameters = Lists.newArrayList();
admins = Lists.newArrayList("admin");
Parameter parameter = new Parameter("llm.model.name", "gpt4",
"模型名称", "list", "大语言模型相关配置");
parameter.setCandidateValues(Lists.newArrayList("gpt3.5", "gpt3.5-16k"));
parameters.add(parameter);
parameters.add(new Parameter("llm.api.key", "sk-secret",
"模型密钥", "string", "大语言模型相关配置"));
//llm config
parameters.add(new Parameter("llm.model.name", "gpt4",
"模型名称(大语言模型相关配置)", "string", "大语言模型相关配置"));
parameters.add(new Parameter("llm.api.key", "sk-afdasdasd",
"模型密钥(大语言模型相关配置)", "string", "大语言模型相关配置"));
parameters.add(new Parameter("llm.temperature", "0.0",
"温度值", "number", "大语言模型相关配置"));
//detect config
parameters.add(new Parameter("one.detection.size", "8",
"一次探测个数", "number", "[mapper]hanlp相关配置"));
"一次探测个数(hanlp相关配置)", "number", "hanlp相关配置"));
parameters.add(new Parameter("one.detection.max.size", "20",
"一次探测最大个数", "number", "[mapper]hanlp相关配置"));
"一次探测最大个数(hanlp相关配置)", "number", "hanlp相关配置"));
//mapper config
parameters.add(new Parameter("metric.dimension.min.threshold", "0.3",
"指标名、维度名最小文本相似度", "number", "[mapper]模糊匹配相关配置"));
"指标名、维度名最小文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
parameters.add(new Parameter("metric.dimension.threshold", "0.3",
"指标名、维度名文本相似度", "number", "[mapper]模糊匹配相关配置"));
"指标名、维度名文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
parameters.add(new Parameter("dimension.value.threshold", "0.5",
"维度值最小文本相似度", "number", "[mapper]模糊匹配相关配置"));
"维度值最小文本相似度(mapper模糊匹配相关配置)", "number", "mapper模糊匹配相关配置"));
//skip config
parameters.add(new Parameter("query.text.length.threshold", "10",
"文本长短阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
parameters.add(new Parameter("short.text.threshold", "5",
"短文本匹配阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
parameters.add(new Parameter("long.text.threshold", "0.8",
"长文本匹配阈值(是否跳过当前parser相关配置)", "number", "是否跳过当前parser相关配置"));
//embedding mapper config
parameters.add(new Parameter("embedding.mapper.word.min",
"0.3", "用于向量召回最小的文本长度", "number", "[mapper]向量召回相关配置"));
parameters.add(new Parameter("embedding.mapper.word.max", "0.3",
"用于向量召回最大的文本长度", "number", "[mapper]向量召回相关配置"));
parameters.add(new Parameter("embedding.mapper.batch", "0.3",
"批量向量召回文本请求个数", "number", "[mapper]向量召回相关配置"));
parameters.add(new Parameter("embedding.mapper.number", "0.3",
"批量向量召回文本返回结果个数", "number", "[mapper]向量召回相关配置"));
"4", "用于向量召回最小的文本长度(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.word.max", "5",
"用于向量召回最大的文本长度(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.batch", "50",
"批量向量召回文本请求个数(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.number", "5",
"批量向量召回文本返回结果个数(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
parameters.add(new Parameter("embedding.mapper.distance.threshold",
"0.3", "Mapper阶段向量召回相似度阈值", "number", "[mapper]向量召回相关配置"));
parameters.add(new Parameter("query.text.length.threshold", "0.5",
"文本长短阈值", "number", "[parser]是否跳过当前parser相关配置"));
parameters.add(new Parameter("short.text.threshold", "0.5",
"短文本匹配阈值", "number", "[parser]是否跳过当前parser相关配置"));
parameters.add(new Parameter("long.text.threshold", "0.5",
"长文本匹配阈值", "number", "[parser]是否跳过当前parser相关配置"));
parameters.add(new Parameter("use.s2SQL.switch", "true",
"是否打开S2SQL转换开关", "bool", "S2SQL相关配置"));
"0.58", "Mapper阶段向量召回相似度阈值(向量召回mapper相关配置)", "number", "向量召回mapper相关配置"));
//s2SQL config
parameters.add(new Parameter("s2SQL.generation", "2-steps",
"S2SQL生成方式", "string", "S2SQL相关配置"));
parameters.add(new Parameter("s2SQL.linking.value.switch", "true",
"是否将linkingValues提供给大模型", "bool", "S2SQL相关配置"));
}
}

10
common/src/main/java/com/tencent/supersonic/common/service/impl/SysParameterServiceImpl.java
View File

@@ -2,13 +2,16 @@ package com.tencent.supersonic.common.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.tencent.supersonic.common.pojo.SysParameter;
import com.fasterxml.jackson.core.type.TypeReference;
import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper;
import com.tencent.supersonic.common.pojo.Parameter;
import com.tencent.supersonic.common.pojo.SysParameter;
import com.tencent.supersonic.common.service.SysParameterService;
import com.tencent.supersonic.common.util.JsonUtil;
import java.util.List;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
@Service
public class SysParameterServiceImpl
@@ -36,7 +39,8 @@ public class SysParameterServiceImpl
private SysParameter convert(SysParameterDO sysParameterDO) {
SysParameter sysParameter = new SysParameter();
sysParameter.setId(sysParameterDO.getId());
sysParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class));
List<Parameter> parameters = JsonUtil.toObject(sysParameterDO.getParameters(), new TypeReference<List<Parameter>>() {});
sysParameter.setParameters(parameters);
sysParameter.setAdminList(sysParameterDO.getAdmin());
return sysParameter;
}