(improvement)(headless&chat)Refactor system parameter impl

2025-12-11 03:58:14 +00:00 · 2024-06-01 01:42:00 +08:00
parent 28960668ce
commit 0f0847824f
32 changed files with 494 additions and 432 deletions
--- a/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java
+++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java
@@ -10,15 +10,17 @@ import java.util.List;
@NoArgsConstructor
 public class Parameter {
    private String name;
-    private String value;
+    private String defaultValue;
    private String comment;
    private String description;
    private String dataType;
    private String module;
    private List<Object> candidateValues;
-    public Parameter(String name, String value, String comment, String description, String dataType, String module) {
+
+    public Parameter(String name, String defaultValue, String comment,
+                     String description, String dataType, String module) {
        this.name = name;
-        this.value = value;
+        this.defaultValue = defaultValue;
        this.comment = comment;
        this.description = description;
        this.dataType = dataType;
--- a/common/src/main/java/com/tencent/supersonic/common/pojo/ParameterConfig.java
+++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ParameterConfig.java
@@ -0,0 +1,49 @@
+package com.tencent.supersonic.common.pojo;
+
+import com.tencent.supersonic.common.service.SystemConfigService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+@Slf4j
+public abstract class ParameterConfig {
+
+    @Autowired
+    private SystemConfigService sysConfigService;
+
+    @Autowired
+    private Environment environment;
+
+    protected abstract List<Parameter> getSysParameters();
+
+    /**
+     * Parameter value will be derived by following orders:
+     * 1. `system config` set with user interface
+     * 2. `system property` set with application.yaml
+     * 3. `default value` set with parameter declaration
+     * @param parameter
+     * @return parameter value
+     */
+    public String getParameterValue(Parameter parameter) {
+        String paramName = parameter.getName();
+        String value = sysConfigService.getSysParameter().getParameterByName(paramName);
+        try {
+            if (StringUtils.isBlank(value)) {
+                if (environment.containsProperty(paramName)) {
+                    value = environment.getProperty(paramName);
+                } else {
+                    value = parameter.getDefaultValue();
+                }
+            }
+        } catch (Exception e) {
+            log.error("Failed to get parameter value for {} with exception: {}", paramName, e);
+        }
+
+        return value;
+    }
+}
--- a/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java
+++ b/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java
@@ -1,111 +0,0 @@
-package com.tencent.supersonic.common.pojo;
-
-import com.google.common.collect.Lists;
-import lombok.Data;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.util.CollectionUtils;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-@Data
-public class SysParameter {
-
-    private Integer id;
-
-    private List<String> admins;
-
-    private List<Parameter> parameters;
-
-    public String getAdmin() {
-        if (CollectionUtils.isEmpty(admins)) {
-            return "";
-        }
-        return StringUtils.join(admins, ",");
-    }
-
-    public String getParameterByName(String name) {
-        if (StringUtils.isBlank(name)) {
-            return "";
-        }
-        Map<String, String> nameToValue = parameters.stream()
-                .collect(Collectors.toMap(Parameter::getName, Parameter::getValue, (k1, k2) -> k1));
-        return nameToValue.get(name);
-    }
-
-    public void setAdminList(String admin) {
-        if (StringUtils.isNotBlank(admin)) {
-            admins = Arrays.asList(admin.split(","));
-        } else {
-            admins = Lists.newArrayList();
-        }
-    }
-
-    public void init() {
-        parameters = Lists.newArrayList();
-        admins = Lists.newArrayList("admin");
-
-        //detect config
-        parameters.add(new Parameter("s2.one.detection.size", "8",
-                "一次探测返回结果个数", "在每次探测后, 将前后缀匹配的结果合并, 并根据相似度阈值过滤后的结果个数",
-                "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.one.detection.max.size", "20",
-                "一次探测前后缀匹配结果返回个数", "单次前后缀匹配返回的结果个数", "number", "Mapper相关配置"));
-
-        //mapper config
-        parameters.add(new Parameter("s2.metric.dimension.threshold", "0.3",
-                "指标名、维度名文本相似度阈值", "文本片段和匹配到的指标、维度名计算出来的编辑距离阈值, 若超出该阈值, 则舍弃",
-                "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.metric.dimension.min.threshold", "0.25",
-                "指标名、维度名最小文本相似度阈值", "指标名、维度名相似度阈值在动态调整中的最低值",
-                "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.dimension.value.threshold", "0.5",
-                "维度值文本相似度阈值", "文本片段和匹配到的维度值计算出来的编辑距离阈值, 若超出该阈值, 则舍弃",
-                "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.dimension.value.min.threshold", "0.3",
-                "维度值最小文本相似度阈值", "维度值相似度阈值在动态调整中的最低值",
-                "number", "Mapper相关配置"));
-
-        //embedding mapper config
-        parameters.add(new Parameter("s2.embedding.mapper.word.min",
-                "4", "用于向量召回最小的文本长度", "为提高向量召回效率, 小于该长度的文本不进行向量语义召回", "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.embedding.mapper.word.max", "5",
-                "用于向量召回最大的文本长度", "为提高向量召回效率, 大于该长度的文本不进行向量语义召回", "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.embedding.mapper.batch", "50",
-                "批量向量召回文本请求个数", "每次进行向量语义召回的原始文本片段个数", "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.embedding.mapper.number", "5",
-                "批量向量召回文本返回结果个数", "每个文本进行向量语义召回的文本结果个数", "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.embedding.mapper.threshold",
-                "0.99", "向量召回相似度阈值", "相似度小于该阈值的则舍弃", "number", "Mapper相关配置"));
-        parameters.add(new Parameter("s2.embedding.mapper.min.threshold",
-                "0.9", "向量召回最小相似度阈值", "向量召回相似度阈值在动态调整中的最低值", "number", "Mapper相关配置"));
-
-        //parser config
-        Parameter s2SQLParameter = new Parameter("s2.parser.strategy",
-                "TWO_PASS_AUTO_COT_SELF_CONSISTENCY",
-                "LLM解析生成S2SQL策略",
-                "ONE_PASS_AUTO_COT_SELF_CONSISTENCY: 通过思维链且投票方式一步生成sql"
-                + "\nTWO_PASS_AUTO_COT_SELF_CONSISTENCY: 通过思维链且投票方式两步生成sql", "list", "Parser相关配置");
-
-        s2SQLParameter.setCandidateValues(Lists.newArrayList(
-                "ONE_PASS_AUTO_COT_SELF_CONSISTENCY", "TWO_PASS_AUTO_COT_SELF_CONSISTENCY"));
-        parameters.add(s2SQLParameter);
-        parameters.add(new Parameter("s2.s2SQL.linking.value.switch", "true",
-                "是否将Mapper探测识别到的维度值提供给大模型", "为了数据安全考虑, 这里可进行开关选择",
-                "bool", "Parser相关配置"));
-        parameters.add(new Parameter("s2.query.text.length.threshold", "10",
-                "用户输入文本长短阈值", "文本超过该阈值为长文本", "number", "Parser相关配置"));
-        parameters.add(new Parameter("s2.short.text.threshold", "0.5",
-                "短文本匹配阈值", "由于请求大模型耗时较长, 因此如果有规则类型的Query得分达到阈值,则跳过大模型的调用,\n如果是短文本, 若query得分/文本长度>该阈值, 则跳过当前parser",
-                "number", "Parser相关配置"));
-        parameters.add(new Parameter("s2.long.text.threshold", "0.8",
-                "长文本匹配阈值", "如果是长文本, 若query得分/文本长度>该阈值, 则跳过当前parser",
-                "number", "Parser相关配置"));
-        parameters.add(new Parameter("s2.parse.show-count", "3",
-                "解析结果个数", "前端展示的解析个数",
-                "number", "Parser相关配置"));
-    }
-
-}
--- a/common/src/main/java/com/tencent/supersonic/common/pojo/SystemConfig.java
+++ b/common/src/main/java/com/tencent/supersonic/common/pojo/SystemConfig.java
@@ -0,0 +1,59 @@
+package com.tencent.supersonic.common.pojo;
+
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.common.util.ContextUtils;
+import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.util.CollectionUtils;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+@Data
+public class SystemConfig {
+
+    private Integer id;
+
+    private List<String> admins;
+
+    private List<Parameter> parameters;
+
+    public String getAdmin() {
+        if (CollectionUtils.isEmpty(admins)) {
+            return "";
+        }
+        return StringUtils.join(admins, ",");
+    }
+
+    public String getParameterByName(String name) {
+        if (StringUtils.isBlank(name)) {
+            return "";
+        }
+        Map<String, String> nameToValue = parameters.stream()
+                .collect(Collectors.toMap(Parameter::getName, Parameter::getDefaultValue, (k1, k2) -> k1));
+        return nameToValue.get(name);
+    }
+
+    public void setAdminList(String admin) {
+        if (StringUtils.isNotBlank(admin)) {
+            admins = Arrays.asList(admin.split(","));
+        } else {
+            admins = Lists.newArrayList();
+        }
+    }
+
+    public void init() {
+        parameters = Lists.newArrayList();
+        admins = Lists.newArrayList("admin");
+
+        Collection<ParameterConfig> configurableParameters =
+                ContextUtils.getBeansOfType(ParameterConfig.class).values();
+        for (ParameterConfig configParameters : configurableParameters) {
+            parameters.addAll(configParameters.getSysParameters());
+        }
+    }
+
+}
--- a/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java
+++ b/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java
@@ -1,7 +1,7 @@
 package com.tencent.supersonic.common.rest;

-import com.tencent.supersonic.common.pojo.SysParameter;
-import com.tencent.supersonic.common.service.SysParameterService;
+import com.tencent.supersonic.common.pojo.SystemConfig;
+import com.tencent.supersonic.common.service.SystemConfigService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -11,20 +11,20 @@ import org.springframework.web.bind.annotation.RestController;

@RestController
@RequestMapping({"/api/semantic/parameter"})
-public class SysParameterController {
+public class SystemConfigController {

    @Autowired
-    private SysParameterService sysParameterService;
+    private SystemConfigService sysConfigService;

    @PostMapping
-    public Boolean save(@RequestBody SysParameter sysParameter) {
-        sysParameterService.save(sysParameter);
+    public Boolean save(@RequestBody SystemConfig sysParameter) {
+        sysConfigService.save(sysParameter);
        return true;
    }

    @GetMapping
-    public SysParameter get() {
-        return sysParameterService.getSysParameter();
+    public SystemConfig get() {
+        return sysConfigService.getSysParameter();
    }

 }
--- a/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java
+++ b/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java
@@ -2,12 +2,12 @@ package com.tencent.supersonic.common.service;

 import com.baomidou.mybatisplus.extension.service.IService;
 import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
-import com.tencent.supersonic.common.pojo.SysParameter;
+import com.tencent.supersonic.common.pojo.SystemConfig;

-public interface SysParameterService extends IService<SysParameterDO> {
+public interface SystemConfigService extends IService<SysParameterDO> {

-    SysParameter getSysParameter();
+    SystemConfig getSysParameter();

-    void save(SysParameter sysParameter);
+    void save(SystemConfig sysConfig);

 }
--- a/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java
+++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java
@@ -6,22 +6,22 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
 import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper;
 import com.tencent.supersonic.common.pojo.Parameter;
-import com.tencent.supersonic.common.pojo.SysParameter;
-import com.tencent.supersonic.common.service.SysParameterService;
+import com.tencent.supersonic.common.pojo.SystemConfig;
+import com.tencent.supersonic.common.service.SystemConfigService;
 import com.tencent.supersonic.common.util.JsonUtil;
 import java.util.List;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;

@Service
-public class SysParameterServiceImpl
-        extends ServiceImpl<SysParameterMapper, SysParameterDO> implements SysParameterService {
+public class SystemConfigServiceImpl
+        extends ServiceImpl<SysParameterMapper, SysParameterDO> implements SystemConfigService {

    @Override
-    public SysParameter getSysParameter() {
+    public SystemConfig getSysParameter() {
        List<SysParameterDO> list = list();
        if (CollectionUtils.isEmpty(list)) {
-            SysParameter sysParameter = new SysParameter();
+            SystemConfig sysParameter = new SystemConfig();
            sysParameter.setId(1);
            sysParameter.init();
            save(sysParameter);
@@ -31,13 +31,13 @@ public class SysParameterServiceImpl
    }

    @Override
-    public void save(SysParameter sysParameter) {
-        SysParameterDO sysParameterDO = convert(sysParameter);
+    public void save(SystemConfig sysConfig) {
+        SysParameterDO sysParameterDO = convert(sysConfig);
        saveOrUpdate(sysParameterDO);
    }

-    private SysParameter convert(SysParameterDO sysParameterDO) {
-        SysParameter sysParameter = new SysParameter();
+    private SystemConfig convert(SysParameterDO sysParameterDO) {
+        SystemConfig sysParameter = new SystemConfig();
        sysParameter.setId(sysParameterDO.getId());
        List<Parameter> parameters = JsonUtil.toObject(sysParameterDO.getParameters(),
                new TypeReference<List<Parameter>>() {
@@ -47,7 +47,7 @@ public class SysParameterServiceImpl
        return sysParameter;
    }

-    private SysParameterDO convert(SysParameter sysParameter) {
+    private SysParameterDO convert(SystemConfig sysParameter) {
        SysParameterDO sysParameterDO = new SysParameterDO();
        sysParameterDO.setId(sysParameter.getId());
        sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters()));