From f07e80587de157406bc4ca08d141209414ca18a1 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Sat, 1 Jun 2024 12:09:09 +0800 Subject: [PATCH] (improvement)(headless&chat) System parameter compatible with historical data and front-end (#1070) This reverts commit 59f6169569f56fd2f0b271508d6efc4872e10b5e. Co-authored-by: jolunoluo --- .../api/authentication/utils/UserHolder.java | 6 +-- .../service/UserServiceImpl.java | 6 +-- ...ysParameterDO.java => SystemConfigDO.java} | 4 +- ...terMapper.java => SystemConfigMapper.java} | 4 +- .../supersonic/common/pojo/Parameter.java | 20 ++++++++++ .../common/pojo/ParameterConfig.java | 2 +- .../supersonic/common/pojo/SystemConfig.java | 26 +++++++++++-- .../common/rest/SystemConfigController.java | 6 +-- .../common/service/SystemConfigService.java | 8 ++-- .../service/impl/SystemConfigServiceImpl.java | 38 +++++++++---------- .../headless/core/config/MapperConfig.java | 4 +- .../headless/core/config/ParserConfig.java | 4 +- .../resources/config.update/sql-update.sql | 5 ++- .../src/main/resources/db/schema-h2.sql | 2 +- .../src/main/resources/db/schema-mysql.sql | 2 +- .../src/test/resources/db/schema-h2.sql | 2 +- .../supersonic-fe/src/pages/System/index.tsx | 7 ++-- .../supersonic-fe/src/pages/System/types.ts | 1 - 18 files changed, 93 insertions(+), 54 deletions(-) rename common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/{SysParameterDO.java => SystemConfigDO.java} (85%) rename common/src/main/java/com/tencent/supersonic/common/persistence/mapper/{SysParameterMapper.java => SystemConfigMapper.java} (53%) diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java index 0397e667e..1425b1c37 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/utils/UserHolder.java @@ -21,9 +21,9 @@ public final class UserHolder { public static User findUser(HttpServletRequest request, HttpServletResponse response) { User user = REPO.findUser(request, response); SystemConfigService sysParameterService = ContextUtils.getBean(SystemConfigService.class); - SystemConfig sysParameter = sysParameterService.getSysParameter(); - if (!CollectionUtils.isEmpty(sysParameter.getAdmins()) - && sysParameter.getAdmins().contains(user.getName())) { + SystemConfig systemConfig = sysParameterService.getSystemConfig(); + if (!CollectionUtils.isEmpty(systemConfig.getAdmins()) + && systemConfig.getAdmins().contains(user.getName())) { user.setIsAdmin(1); } return user; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java index 4b3bf7eb4..ee41ff8d8 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java @@ -28,9 +28,9 @@ public class UserServiceImpl implements UserService { public User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) { User user = UserHolder.findUser(httpServletRequest, httpServletResponse); if (user != null) { - SystemConfig sysParameter = sysParameterService.getSysParameter(); - if (!CollectionUtils.isEmpty(sysParameter.getAdmins()) - && sysParameter.getAdmins().contains(user.getName())) { + SystemConfig systemConfig = sysParameterService.getSystemConfig(); + if (!CollectionUtils.isEmpty(systemConfig.getAdmins()) + && systemConfig.getAdmins().contains(user.getName())) { user.setIsAdmin(1); } } diff --git a/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java b/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SystemConfigDO.java similarity index 85% rename from common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java rename to common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SystemConfigDO.java index 7f7ed272c..77a11d18d 100644 --- a/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java +++ b/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SystemConfigDO.java @@ -6,8 +6,8 @@ import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; @Data -@TableName("s2_sys_parameter") -public class SysParameterDO { +@TableName("s2_system_config") +public class SystemConfigDO { @TableId(type = IdType.AUTO) private Integer id; diff --git a/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java b/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SystemConfigMapper.java similarity index 53% rename from common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java rename to common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SystemConfigMapper.java index bb4be6c58..982bc5a8e 100644 --- a/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java +++ b/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SystemConfigMapper.java @@ -1,11 +1,11 @@ package com.tencent.supersonic.common.persistence.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO; +import com.tencent.supersonic.common.persistence.dataobject.SystemConfigDO; import org.apache.ibatis.annotations.Mapper; @Mapper -public interface SysParameterMapper extends BaseMapper { +public interface SystemConfigMapper extends BaseMapper { } \ No newline at end of file diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java index efb29bab9..60c352705 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.common.pojo; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; import java.util.List; @Data @@ -15,6 +16,7 @@ public class Parameter { private String description; private String dataType; private String module; + private String value; private List candidateValues; public Parameter(String name, String defaultValue, String comment, @@ -27,4 +29,22 @@ public class Parameter { this.module = module; } + public Parameter(String name, String defaultValue, String comment, String description, + String dataType, String module, List candidateValues) { + this.name = name; + this.defaultValue = defaultValue; + this.comment = comment; + this.description = description; + this.dataType = dataType; + this.module = module; + this.candidateValues = candidateValues; + } + + public String getValue() { + if (StringUtils.isBlank(value)) { + return defaultValue; + } + return value; + } + } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ParameterConfig.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ParameterConfig.java index b6c7310ed..57321cf74 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ParameterConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ParameterConfig.java @@ -31,7 +31,7 @@ public abstract class ParameterConfig { */ public String getParameterValue(Parameter parameter) { String paramName = parameter.getName(); - String value = sysConfigService.getSysParameter().getParameterByName(paramName); + String value = sysConfigService.getSystemConfig().getParameterByName(paramName); try { if (StringUtils.isBlank(value)) { if (environment.containsProperty(paramName)) { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/SystemConfig.java b/common/src/main/java/com/tencent/supersonic/common/pojo/SystemConfig.java index 4d5e62fc6..a90aaf776 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/SystemConfig.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/SystemConfig.java @@ -32,8 +32,8 @@ public class SystemConfig { if (StringUtils.isBlank(name)) { return ""; } - Map nameToValue = parameters.stream() - .collect(Collectors.toMap(Parameter::getName, Parameter::getDefaultValue, (k1, k2) -> k1)); + Map nameToValue = getParameters().stream() + .collect(Collectors.toMap(Parameter::getName, Parameter::getValue, (k1, k2) -> k1)); return nameToValue.get(name); } @@ -46,14 +46,32 @@ public class SystemConfig { } public void init() { - parameters = Lists.newArrayList(); + parameters = buildDefaultParameters(); admins = Lists.newArrayList("admin"); + } + private List buildDefaultParameters() { + List defaultParameters = Lists.newArrayList(); Collection configurableParameters = ContextUtils.getBeansOfType(ParameterConfig.class).values(); for (ParameterConfig configParameters : configurableParameters) { - parameters.addAll(configParameters.getSysParameters()); + defaultParameters.addAll(configParameters.getSysParameters()); } + return defaultParameters; + } + + public List getParameters() { + List defaultParameters = buildDefaultParameters(); + if (CollectionUtils.isEmpty(parameters)) { + return defaultParameters; + } + Map parameterNameValueMap = parameters.stream() + .collect(Collectors.toMap(Parameter::getName, Parameter::getValue, (v1, v2) -> v2)); + for (Parameter parameter : defaultParameters) { + parameter.setValue(parameterNameValueMap.getOrDefault(parameter.getName(), + parameter.getDefaultValue())); + } + return defaultParameters; } } diff --git a/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java b/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java index 00308d186..2a6fbf6bf 100644 --- a/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java +++ b/common/src/main/java/com/tencent/supersonic/common/rest/SystemConfigController.java @@ -17,14 +17,14 @@ public class SystemConfigController { private SystemConfigService sysConfigService; @PostMapping - public Boolean save(@RequestBody SystemConfig sysParameter) { - sysConfigService.save(sysParameter); + public Boolean save(@RequestBody SystemConfig systemConfig) { + sysConfigService.save(systemConfig); return true; } @GetMapping public SystemConfig get() { - return sysConfigService.getSysParameter(); + return sysConfigService.getSystemConfig(); } } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java b/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java index ff830737f..e510893b3 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/SystemConfigService.java @@ -1,13 +1,13 @@ package com.tencent.supersonic.common.service; import com.baomidou.mybatisplus.extension.service.IService; -import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO; +import com.tencent.supersonic.common.persistence.dataobject.SystemConfigDO; import com.tencent.supersonic.common.pojo.SystemConfig; -public interface SystemConfigService extends IService { +public interface SystemConfigService extends IService { - SystemConfig getSysParameter(); + SystemConfig getSystemConfig(); - void save(SystemConfig sysConfig); + void save(SystemConfig systemConfig); } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java b/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java index e85593214..1ffb9dae3 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/SystemConfigServiceImpl.java @@ -3,52 +3,52 @@ package com.tencent.supersonic.common.service.impl; import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO; -import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper; +import com.tencent.supersonic.common.persistence.dataobject.SystemConfigDO; +import com.tencent.supersonic.common.persistence.mapper.SystemConfigMapper; import com.tencent.supersonic.common.pojo.Parameter; import com.tencent.supersonic.common.pojo.SystemConfig; import com.tencent.supersonic.common.service.SystemConfigService; import com.tencent.supersonic.common.util.JsonUtil; -import java.util.List; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.List; @Service public class SystemConfigServiceImpl - extends ServiceImpl implements SystemConfigService { + extends ServiceImpl implements SystemConfigService { @Override - public SystemConfig getSysParameter() { - List list = list(); + public SystemConfig getSystemConfig() { + List list = list(); if (CollectionUtils.isEmpty(list)) { - SystemConfig sysParameter = new SystemConfig(); - sysParameter.setId(1); - sysParameter.init(); - save(sysParameter); - return sysParameter; + SystemConfig systemConfig = new SystemConfig(); + systemConfig.setId(1); + systemConfig.init(); + save(systemConfig); + return systemConfig; } return convert(list.iterator().next()); } @Override public void save(SystemConfig sysConfig) { - SysParameterDO sysParameterDO = convert(sysConfig); - saveOrUpdate(sysParameterDO); + SystemConfigDO systemConfigDO = convert(sysConfig); + saveOrUpdate(systemConfigDO); } - private SystemConfig convert(SysParameterDO sysParameterDO) { + private SystemConfig convert(SystemConfigDO systemConfigDO) { SystemConfig sysParameter = new SystemConfig(); - sysParameter.setId(sysParameterDO.getId()); - List parameters = JsonUtil.toObject(sysParameterDO.getParameters(), + sysParameter.setId(systemConfigDO.getId()); + List parameters = JsonUtil.toObject(systemConfigDO.getParameters(), new TypeReference>() { }); sysParameter.setParameters(parameters); - sysParameter.setAdminList(sysParameterDO.getAdmin()); + sysParameter.setAdminList(systemConfigDO.getAdmin()); return sysParameter; } - private SysParameterDO convert(SystemConfig sysParameter) { - SysParameterDO sysParameterDO = new SysParameterDO(); + private SystemConfigDO convert(SystemConfig sysParameter) { + SystemConfigDO sysParameterDO = new SystemConfigDO(); sysParameterDO.setId(sysParameter.getId()); sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters())); sysParameterDO.setAdmin(sysParameter.getAdmin()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/MapperConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/MapperConfig.java index d57ab0aa9..79ea76660 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/MapperConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/MapperConfig.java @@ -36,8 +36,8 @@ public class MapperConfig extends ParameterConfig { public static final Parameter MAPPER_DIMENSION_VALUE_SIZE = new Parameter("s2.mapper.value.size", "1", - "指标名、维度名最小文本相似度阈值", - "指标名、维度名相似度阈值在动态调整中的最低值", + "一次探测返回维度值结果个数", + "在每次探测后, 将前后缀匹配的结果合并, 并根据相似度阈值过滤后的维度值结果个数", "number", "Mapper相关配置"); public static final Parameter MAPPER_VALUE_THRESHOLD = diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ParserConfig.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ParserConfig.java index 5d1d9609d..20ea2b39a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ParserConfig.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ParserConfig.java @@ -31,14 +31,14 @@ public class ParserConfig extends ParameterConfig { "number", "Parser相关配置"); public static final Parameter PARSER_TEXT_LENGTH_THRESHOLD_SHORT = - new Parameter("s2.parser.text.threshold", "0.5", + new Parameter("s2.parser.text.threshold.short", "0.5", "短文本匹配阈值", "由于请求大模型耗时较长, 因此如果有规则类型的Query得分达到阈值,则跳过大模型的调用," + "\n如果是短文本, 若query得分/文本长度>该阈值, 则跳过当前parser", "number", "Parser相关配置"); public static final Parameter PARSER_TEXT_LENGTH_THRESHOLD_LONG = - new Parameter("s2.parser.text.threshold", "0.8", + new Parameter("s2.parser.text.threshold.long", "0.8", "长文本匹配阈值", "如果是长文本, 若query得分/文本长度>该阈值, 则跳过当前parser", "number", "Parser相关配置"); diff --git a/launchers/standalone/src/main/resources/config.update/sql-update.sql b/launchers/standalone/src/main/resources/config.update/sql-update.sql index 05b7c3c66..380509785 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql @@ -313,4 +313,7 @@ CREATE TABLE IF NOT EXISTS `s2_term` ( alter table s2_agent add column `llm_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL; alter table s2_agent add column `multi_turn_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL; -alter table s2_model add column `ext` varchar(1000) DEFAULT NULL; \ No newline at end of file +alter table s2_model add column `ext` varchar(1000) DEFAULT NULL; + +--20240601 +alter table s2_sys_parameter rename to s2_system_config; \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 8981ebcfb..fc4e06a17 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -517,7 +517,7 @@ CREATE TABLE IF NOT EXISTS `company_brand_revenue` ( -CREATE TABLE IF NOT EXISTS s2_sys_parameter +CREATE TABLE IF NOT EXISTS s2_system_config ( id INT PRIMARY KEY AUTO_INCREMENT, admin varchar(500), diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index e2eb69d46..91c918d52 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -431,7 +431,7 @@ create table s2_user PRIMARY KEY (`id`) ); -CREATE TABLE s2_sys_parameter +CREATE TABLE s2_system_config ( id int primary key AUTO_INCREMENT COMMENT '主键id', admin varchar(500) COMMENT '系统管理员', diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index ac7a9549b..550000109 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -517,7 +517,7 @@ CREATE TABLE IF NOT EXISTS `company_brand_revenue` ( -CREATE TABLE IF NOT EXISTS s2_sys_parameter +CREATE TABLE IF NOT EXISTS s2_system_config ( id INT PRIMARY KEY AUTO_INCREMENT, admin varchar(500), diff --git a/webapp/packages/supersonic-fe/src/pages/System/index.tsx b/webapp/packages/supersonic-fe/src/pages/System/index.tsx index e09c18bf3..2597916fd 100644 --- a/webapp/packages/supersonic-fe/src/pages/System/index.tsx +++ b/webapp/packages/supersonic-fe/src/pages/System/index.tsx @@ -15,7 +15,7 @@ import { } from 'antd'; import React, { useState, useEffect } from 'react'; import { getSystemConfig, saveSystemConfig } from '@/services/user'; -import { ProCard } from '@ant-design/pro-components'; +import { ProCard } from '@ant-design/pro-components'; import SelectTMEPerson from '@/components/SelectTMEPerson'; import { ConfigParametersItem, SystemConfig } from './types'; import FormItemTitle from '@/components/FormHelper/FormItemTitle'; @@ -59,10 +59,10 @@ const System: React.FC = () => { const setInitData = (admins: string[], systemConfigParameters: ConfigParametersItem[]) => { const fieldsValue = systemConfigParameters.reduce( (fields, item) => { - const { name, value, defaultValue } = item; + const { name, value } = item; return { ...fields, - [name]: value || defaultValue, + [name]: value, }; }, { admins }, @@ -81,7 +81,6 @@ const System: React.FC = () => { return { ...item, value: submitData[name], - defaultValue: submitData[name], }; } return item; diff --git a/webapp/packages/supersonic-fe/src/pages/System/types.ts b/webapp/packages/supersonic-fe/src/pages/System/types.ts index 9cd8f7010..ed3f94e73 100644 --- a/webapp/packages/supersonic-fe/src/pages/System/types.ts +++ b/webapp/packages/supersonic-fe/src/pages/System/types.ts @@ -3,7 +3,6 @@ export type ConfigParametersItem = { name: string; comment: string; value: string; - defaultValue?: string; candidateValues: string[]; description: string; require?: boolean;