mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 02:46:56 +00:00
Compare commits
162 Commits
v0.9.10
...
6c5f8fce40
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c5f8fce40 | ||
|
|
04b1edb2e2 | ||
|
|
9857256488 | ||
|
|
d695bed75d | ||
|
|
7490dabdc3 | ||
|
|
fad28ee5ac | ||
|
|
353c8d8b16 | ||
|
|
3dd53bad89 | ||
|
|
2d39ebf38b | ||
|
|
6c472e1c76 | ||
|
|
431aa60e4d | ||
|
|
25df22758a | ||
|
|
9af6499491 | ||
|
|
c992e57b13 | ||
|
|
80aaabe58b | ||
|
|
5a4fd2b888 | ||
|
|
5df0b87da9 | ||
|
|
ab24b1777a | ||
|
|
ff76f8edbd | ||
|
|
76745f38a4 | ||
|
|
ce4cdb62ab | ||
|
|
c2ce3a75b7 | ||
|
|
1f6d217b26 | ||
|
|
af28bc7c2a | ||
|
|
42bf355839 | ||
|
|
91e4b51ef8 | ||
|
|
bf3213e8fb | ||
|
|
c75233e37f | ||
|
|
785bda6cd9 | ||
|
|
6bd8970849 | ||
|
|
c33a85b583 | ||
|
|
62b9db6791 | ||
|
|
6d907b6adf | ||
|
|
da172a030e | ||
|
|
47c2595fb8 | ||
|
|
9bddd4457e | ||
|
|
55ac3d1aa5 | ||
|
|
0427917624 | ||
|
|
d8fe2ed2b3 | ||
|
|
11d1264d38 | ||
|
|
32675387d7 | ||
|
|
e408204690 | ||
|
|
269f146c11 | ||
|
|
6f497b142e | ||
|
|
79a44b27ee | ||
|
|
76cc5ee111 | ||
|
|
320fcf04bd | ||
|
|
75fc83010c | ||
|
|
37673c82da | ||
|
|
3ae0d645a7 | ||
|
|
256a6bcb3f | ||
|
|
1faf84e372 | ||
|
|
7e6639df83 | ||
|
|
075ae4c0af | ||
|
|
08133ccbfb | ||
|
|
164d2a9e23 | ||
|
|
f899d23b63 | ||
|
|
944beddafc | ||
|
|
019d737f07 | ||
|
|
0721df2e66 | ||
|
|
303392f492 | ||
|
|
e5a41765b4 | ||
|
|
87355533b4 | ||
|
|
06fb6ba744 | ||
|
|
9ffdba956e | ||
|
|
df70a3cf15 | ||
|
|
2552e2ae4b | ||
|
|
4bfa10ba7c | ||
|
|
958aca945d | ||
|
|
fae9118c28 | ||
|
|
c24ba59bb5 | ||
|
|
90c4f66770 | ||
|
|
b9dd6bb7c5 | ||
|
|
dff64b62f4 | ||
|
|
8eeed87bac | ||
|
|
e171bdd97f | ||
|
|
0709575cd9 | ||
|
|
be0447ae15 | ||
|
|
1b8cd7f0d3 | ||
|
|
2fd82cc259 | ||
|
|
00814a3807 | ||
|
|
08705c9d3b | ||
|
|
1c9cf788cb | ||
|
|
1ab5d9c7e6 | ||
|
|
2b13866c0b | ||
|
|
e812884802 | ||
|
|
e2ae7e21ad | ||
|
|
3fc1ec42be | ||
|
|
c4992501bd | ||
|
|
acffc03c79 | ||
|
|
763def2de0 | ||
|
|
d0a67af684 | ||
|
|
be8b56bdde | ||
|
|
9f2c0c7699 | ||
|
|
c1fa9d7442 | ||
|
|
0d5da763b3 | ||
|
|
d1b4863a27 | ||
|
|
dce9a8a58c | ||
|
|
fbf048cb00 | ||
|
|
48a8f69cca | ||
|
|
ecdf65da3e | ||
|
|
5585b9e222 | ||
|
|
97710a90c4 | ||
|
|
0ab7643299 | ||
|
|
d2aa73b85e | ||
|
|
8828964e53 | ||
|
|
b188da8595 | ||
|
|
2e7ba468c9 | ||
|
|
d26c9180ed | ||
|
|
ca96aa725d | ||
|
|
614917ba76 | ||
|
|
1fed8ca4d9 | ||
|
|
232a202275 | ||
|
|
791c493a6a | ||
|
|
f9d4ce2128 | ||
|
|
8abfc923a0 | ||
|
|
e6598a79bb | ||
|
|
d2a43a99c8 | ||
|
|
db8f340e2d | ||
|
|
2e81b190a4 | ||
|
|
81cd60d2da | ||
|
|
3ffc8c3d9e | ||
|
|
18db24c011 | ||
|
|
cd698ac367 | ||
|
|
58b640b087 | ||
|
|
1f28aaeaed | ||
|
|
35b835172b | ||
|
|
1c85bcecc5 | ||
|
|
c3483ae340 | ||
|
|
a5051c7225 | ||
|
|
12f6cfa42d | ||
|
|
4c94f2b816 | ||
|
|
c81aa5859d | ||
|
|
21e213fb19 | ||
|
|
f67bf3eeac | ||
|
|
9d13038599 | ||
|
|
0c8c2d4804 | ||
|
|
f05a4b523c | ||
|
|
b7369abcca | ||
|
|
b40cb13740 | ||
|
|
6f8cf9853b | ||
|
|
75906037ac | ||
|
|
b58e041e8d | ||
|
|
93d585c0d5 | ||
|
|
0dbf56d357 | ||
|
|
a3293e6788 | ||
|
|
a99f5985f5 | ||
|
|
91243005bc | ||
|
|
a76b5a4300 | ||
|
|
c1f9df963c | ||
|
|
954aa4eea5 | ||
|
|
33bd0de604 | ||
|
|
881d891d70 | ||
|
|
d9db455dab | ||
|
|
e0dc3fbf1a | ||
|
|
efddf4cacf | ||
|
|
732222ab98 | ||
|
|
5b994c4f8f | ||
|
|
5d2ebdf680 | ||
|
|
f1bc18ef65 | ||
|
|
8f361f9932 | ||
|
|
f532088e38 |
11
.github/workflows/centos-ci.yml
vendored
11
.github/workflows/centos-ci.yml
vendored
@@ -11,7 +11,7 @@ jobs:
|
|||||||
build:
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container:
|
container:
|
||||||
image: quay.io/centos/centos:stream8 # 使用 CentOS Stream 8 容器
|
image: almalinux:9 # maven >=3.6.3
|
||||||
|
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
@@ -28,9 +28,10 @@ jobs:
|
|||||||
|
|
||||||
- name: Reset DNF repositories
|
- name: Reset DNF repositories
|
||||||
run: |
|
run: |
|
||||||
cd /etc/yum.repos.d/
|
sed -e 's|^mirrorlist=|#mirrorlist=|g' \
|
||||||
sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
|
-e 's|^# baseurl=https://repo.almalinux.org|baseurl=https://mirrors.aliyun.com|g' \
|
||||||
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
|
/etc/yum.repos.d/almalinux*.repo
|
||||||
|
|
||||||
|
|
||||||
- name: Update DNF package index
|
- name: Update DNF package index
|
||||||
run: dnf makecache
|
run: dnf makecache
|
||||||
@@ -47,7 +48,7 @@ jobs:
|
|||||||
mvn -version
|
mvn -version
|
||||||
|
|
||||||
- name: Cache Maven packages
|
- name: Cache Maven packages
|
||||||
uses: actions/cache@v2
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.m2
|
path: ~/.m2
|
||||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||||
|
|||||||
14
.github/workflows/mac-ci.yml
vendored
14
.github/workflows/mac-ci.yml
vendored
@@ -17,21 +17,27 @@ jobs:
|
|||||||
java-version: [21] # Define the JDK versions to test
|
java-version: [21] # Define the JDK versions to test
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up JDK ${{ matrix.java-version }}
|
- name: Set up JDK ${{ matrix.java-version }}
|
||||||
uses: actions/setup-java@v2
|
uses: actions/setup-java@v3
|
||||||
with:
|
with:
|
||||||
java-version: ${{ matrix.java-version }}
|
java-version: ${{ matrix.java-version }}
|
||||||
distribution: 'adopt'
|
distribution: 'temurin'
|
||||||
|
|
||||||
- name: Cache Maven packages
|
- name: Cache Maven packages
|
||||||
uses: actions/cache@v2
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: ~/Library/Caches/Maven # macOS Maven cache path
|
path: ~/Library/Caches/Maven # macOS Maven cache path
|
||||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||||
restore-keys: ${{ runner.os }}-m2
|
restore-keys: ${{ runner.os }}-m2
|
||||||
|
|
||||||
|
- name: Install system dependencies
|
||||||
|
run: |
|
||||||
|
brew update
|
||||||
|
brew install cmake
|
||||||
|
brew install gcc
|
||||||
|
|
||||||
- name: Build with Maven
|
- name: Build with Maven
|
||||||
run: mvn -B package --file pom.xml
|
run: mvn -B package --file pom.xml
|
||||||
|
|
||||||
|
|||||||
2
.github/workflows/ubuntu-ci.yml
vendored
2
.github/workflows/ubuntu-ci.yml
vendored
@@ -26,7 +26,7 @@ jobs:
|
|||||||
distribution: 'adopt'
|
distribution: 'adopt'
|
||||||
|
|
||||||
- name: Cache Maven packages
|
- name: Cache Maven packages
|
||||||
uses: actions/cache@v2
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.m2
|
path: ~/.m2
|
||||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||||
|
|||||||
2
.github/workflows/windows-ci.yml
vendored
2
.github/workflows/windows-ci.yml
vendored
@@ -26,7 +26,7 @@ jobs:
|
|||||||
distribution: 'adopt' # You might need to change this if 'adopt' doesn't support JDK 21
|
distribution: 'adopt' # You might need to change this if 'adopt' doesn't support JDK 21
|
||||||
|
|
||||||
- name: Cache Maven packages
|
- name: Cache Maven packages
|
||||||
uses: actions/cache@v2
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~\.m2 # Windows uses a backslash for paths
|
path: ~\.m2 # Windows uses a backslash for paths
|
||||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||||
|
|||||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -20,4 +20,5 @@ chm_db/
|
|||||||
__pycache__/
|
__pycache__/
|
||||||
/dict
|
/dict
|
||||||
assembly/build/*-SNAPSHOT
|
assembly/build/*-SNAPSHOT
|
||||||
**/node_modules/
|
**/node_modules/
|
||||||
|
benchmark/res/
|
||||||
199
CHANGELOG.md
199
CHANGELOG.md
@@ -3,6 +3,205 @@
|
|||||||
- All notable changes to this project will be documented in this file.
|
- All notable changes to this project will be documented in this file.
|
||||||
- "Breaking Changes" describes any changes that may break existing functionality or cause
|
- "Breaking Changes" describes any changes that may break existing functionality or cause
|
||||||
compatibility issues with previous versions.
|
compatibility issues with previous versions.
|
||||||
|
## SuperSonic [1.0.0] - 2025-08-05
|
||||||
|
|
||||||
|
### 重大特性变更 / Major Features
|
||||||
|
|
||||||
|
#### 多数据库支持扩展 / Multi-Database Support
|
||||||
|
- **Oracle数据库支持**: 新增Oracle数据库引擎类型及适配器 ([8eeed87ba](https://github.com/tencentmusic/supersonic/commit/8eeed87ba) by supersonicbi)
|
||||||
|
- **StarRocks支持**: 支持StarRocks和多catalog功能 ([33268bf3d](https://github.com/tencentmusic/supersonic/commit/33268bf3d) by zyclove)
|
||||||
|
- **SAP HANA支持**: 新增SAP HANA数据库适配支持 ([2e28a4c7a](https://github.com/tencentmusic/supersonic/commit/2e28a4c7a) by wwsheng009)
|
||||||
|
- **DuckDB支持**: 支持DuckDB数据库 ([a058dc8b6](https://github.com/tencentmusic/supersonic/commit/a058dc8b6) by jerryjzhang)
|
||||||
|
- **Kyuubi支持**: 支持Kyuubi Presto Trino ([5e3bafb95](https://github.com/tencentmusic/supersonic/commit/5e3bafb95) by zyclove)
|
||||||
|
- **OpenSearch支持**: 新增OpenSearch支持 ([d942d35c9](https://github.com/tencentmusic/supersonic/commit/d942d35c9) by zyclove)
|
||||||
|
|
||||||
|
#### 智能问答增强 / AI-Enhanced Query Processing
|
||||||
|
- **LLM纠错器**: 新增LLM物理SQL纠错器 ([f899d23b6](https://github.com/tencentmusic/supersonic/commit/f899d23b6) by 柯慕灵)
|
||||||
|
- **记忆管理**: Agent记忆管理启用few-shot优先机制 ([fae9118c2](https://github.com/tencentmusic/supersonic/commit/fae9118c2) by feelshana)
|
||||||
|
- **结构化查询**: 支持struct查询中的offset子句 ([d2a43a99c](https://github.com/tencentmusic/supersonic/commit/d2a43a99c) by jerryjzhang)
|
||||||
|
- **向量召回优化**: 优化嵌入向量召回机制 ([8c6ae6252](https://github.com/tencentmusic/supersonic/commit/8c6ae6252) by lexluo09)
|
||||||
|
|
||||||
|
#### 权限管理系统 / Permission Management
|
||||||
|
- **Agent权限**: 支持agent级别的权限管理 ([b5aa6e046](https://github.com/tencentmusic/supersonic/commit/b5aa6e046) by jerryjzhang)
|
||||||
|
- **用户管理**: 支持用户删除功能 ([1c9cf788c](https://github.com/tencentmusic/supersonic/commit/1c9cf788c) by supersonicbi)
|
||||||
|
- **鉴权优化**: 全面优化鉴权与召回机制 ([1faf84e37](https://github.com/tencentmusic/supersonic/commit/1faf84e37), [7e6639df8](https://github.com/tencentmusic/supersonic/commit/7e6639df8) by guilinlewis)
|
||||||
|
|
||||||
|
### 架构升级 / Architecture Upgrades
|
||||||
|
|
||||||
|
#### 核心框架升级 / Core Framework Upgrades
|
||||||
|
- **SpringBoot 3升级**: 完成SpringBoot 3.x升级 ([07f6be51c](https://github.com/tencentmusic/supersonic/commit/07f6be51c) by mislayming)
|
||||||
|
- **依赖升级**: 升级依赖包并修复安全漏洞 ([232a20227](https://github.com/tencentmusic/supersonic/commit/232a20227) by beat4ocean)
|
||||||
|
- **LangChain4j更新**: 替换已废弃的LangChain4j APIs ([acffc03c7](https://github.com/tencentmusic/supersonic/commit/acffc03c7) by beat4ocean)
|
||||||
|
- **Swagger升级**: 使用SpringDoc支持Swagger在Spring 3.x ([758d170bb](https://github.com/tencentmusic/supersonic/commit/758d170bb) by jerryjzhang)
|
||||||
|
|
||||||
|
#### 许可证变更 / License Changes
|
||||||
|
- **Apache 2.0**: 从MIT更改为Apache 2.0许可证 ([0aa002882](https://github.com/tencentmusic/supersonic/commit/0aa002882) by jerryjzhang)
|
||||||
|
|
||||||
|
### 性能优化 / Performance Improvements
|
||||||
|
|
||||||
|
#### 系统性能 / System Performance
|
||||||
|
- **GC优化**: 实现Generational ZGC ([3fc1ec42b](https://github.com/tencentmusic/supersonic/commit/3fc1ec42b) by beat4ocean)
|
||||||
|
- **Docker优化**: 减少Docker镜像体积 ([614917ba7](https://github.com/tencentmusic/supersonic/commit/614917ba7) by kino)
|
||||||
|
- **并行处理**: 嵌入向量并行执行优化 ([8c6ae6252](https://github.com/tencentmusic/supersonic/commit/8c6ae6252) by lexluo09)
|
||||||
|
- **记忆评估**: 记忆评估性能优化 ([524ec38ed](https://github.com/tencentmusic/supersonic/commit/524ec38ed) by yudong)
|
||||||
|
- **多平台构建**: 支持Docker多平台构建 ([da6d28c18](https://github.com/tencentmusic/supersonic/commit/da6d28c18) by jerryjzhang)
|
||||||
|
|
||||||
|
#### 数据处理优化 / Data Processing Optimization
|
||||||
|
- **日期格式**: 支持更多日期字符串格式 ([2b13866c0](https://github.com/tencentmusic/supersonic/commit/2b13866c0) by supersonicbi)
|
||||||
|
- **SQL优化**: 优化SQL生成和执行性能 ([0ab764329](https://github.com/tencentmusic/supersonic/commit/0ab764329) by jerryjzhang)
|
||||||
|
- **模型关联**: 优化模型关联查询性能 ([47c2595fb](https://github.com/tencentmusic/supersonic/commit/47c2595fb) by Willy-J)
|
||||||
|
|
||||||
|
### 功能增强 / Feature Enhancements
|
||||||
|
|
||||||
|
#### 前端界面优化 / Frontend Improvements
|
||||||
|
- **图表导出**: 消息支持导出图表图片 ([ce9ae1c0c](https://github.com/tencentmusic/supersonic/commit/ce9ae1c0c) by pisces)
|
||||||
|
- **路由重构**: 重构语义建模路由交互 ([82c63a7f2](https://github.com/tencentmusic/supersonic/commit/82c63a7f2) by tristanliu)
|
||||||
|
- **权限界面**: 统一助理权限设置交互界面 ([46d64d78f](https://github.com/tencentmusic/supersonic/commit/46d64d78f) by tristanliu)
|
||||||
|
- **图表优化**: 优化ChatMsg图表条件 ([06fb6ba74](https://github.com/tencentmusic/supersonic/commit/06fb6ba74) by FredTsang)
|
||||||
|
- **数据格式**: 提取formatByDataFormatType()方法 ([9ffdba956](https://github.com/tencentmusic/supersonic/commit/9ffdba956) by FredTsang)
|
||||||
|
|
||||||
|
#### 开发体验 / Developer Experience
|
||||||
|
- **构建脚本**: 优化Web应用构建脚本 ([baae7f74b](https://github.com/tencentmusic/supersonic/commit/baae7f74b) by zyclove)
|
||||||
|
- **GitHub Actions**: 优化GitHub Actions镜像推送 ([6a4458a57](https://github.com/tencentmusic/supersonic/commit/6a4458a57) by lexluo09)
|
||||||
|
- **基准测试**: 改进基准测试,增加解析结果分析 ([97710a90c](https://github.com/tencentmusic/supersonic/commit/97710a90c) by Antgeek)
|
||||||
|
|
||||||
|
### Bug修复 / Bug Fixes
|
||||||
|
|
||||||
|
#### 核心功能修复 / Core Function Fixes
|
||||||
|
- **插件功能**: 修复插件功能无法调用/结果被NL2SQL覆盖问题 ([c75233e37](https://github.com/tencentmusic/supersonic/commit/c75233e37) by QJ_wonder)
|
||||||
|
- **维度别名**: 修复映射阶段维度值别名不生效问题 ([785bda6cd](https://github.com/tencentmusic/supersonic/commit/785bda6cd) by feelshana)
|
||||||
|
- **模型字段**: 修复模型字段更新问题 ([6bd897084](https://github.com/tencentmusic/supersonic/commit/6bd897084) by WDEP)
|
||||||
|
- **多轮对话**: 修复headless中字段查询及多轮对话使用问题 ([be0447ae1](https://github.com/tencentmusic/supersonic/commit/be0447ae1) by QJ_wonder)
|
||||||
|
|
||||||
|
#### NPE异常修复 / NPE Exception Fixes
|
||||||
|
- **聊天查询**: 修复EmbeddingMatchStrategy.detectByBatch() NPE异常 ([6d907b6ad](https://github.com/tencentmusic/supersonic/commit/6d907b6ad) by wangyong)
|
||||||
|
- **文件处理**: 修复FileHandlerImpl.convert2Resp() 维度值数据行首字符为空格异常 ([da172a030](https://github.com/tencentmusic/supersonic/commit/da172a030) by wangyong)
|
||||||
|
- **头部服务**: 修复多处headless NPE问题 ([79a44b27e](https://github.com/tencentmusic/supersonic/commit/79a44b27e) by jerryjzhang)
|
||||||
|
- **解析信息**: 修复getParseInfo中的NPE ([dce9a8a58](https://github.com/tencentmusic/supersonic/commit/dce9a8a58) by supersonicbi)
|
||||||
|
|
||||||
|
#### SQL兼容性修复 / SQL Compatibility Fixes
|
||||||
|
- **SQL处理**: 修复SQL前后换行符导致的语句结尾";"删除问题 ([55ac3d1aa](https://github.com/tencentmusic/supersonic/commit/55ac3d1aa) by wangyong)
|
||||||
|
- **查询别名**: DictUtils.constructQuerySqlReq针对sql query增加别名 ([042791762](https://github.com/tencentmusic/supersonic/commit/042791762) by andybj0228)
|
||||||
|
- **SQL变量**: 支持SQL脚本变量替换 ([0709575cd](https://github.com/tencentmusic/supersonic/commit/0709575cd) by wanglongqiang)
|
||||||
|
|
||||||
|
#### 前端Bug修复 / Frontend Bug Fixes
|
||||||
|
- **UI样式**: 修复问答对话右侧历史对话模块样式异常 ([c33a85b58](https://github.com/tencentmusic/supersonic/commit/c33a85b58) by wangyong)
|
||||||
|
- **推荐维度**: 修复页面不显示推荐下钻维度问题 ([62b9db679](https://github.com/tencentmusic/supersonic/commit/62b9db679) by WDEP)
|
||||||
|
- **图表显示**: 修复饼图显示条件问题 ([1b8cd7f0d](https://github.com/tencentmusic/supersonic/commit/1b8cd7f0d) by WDEP)
|
||||||
|
- **负数支持**: 支持负数显示 ([2552e2ae4](https://github.com/tencentmusic/supersonic/commit/2552e2ae4) by FredTsang)
|
||||||
|
- **百分比显示**: 支持bar图needMultiply100显示正确百分比值 ([8abfc923a](https://github.com/tencentmusic/supersonic/commit/8abfc923a) by coosir)
|
||||||
|
- **TypeScript错误**: 修复前端TypeScript错误 ([5585b9e22](https://github.com/tencentmusic/supersonic/commit/5585b9e22) by poncheen)
|
||||||
|
|
||||||
|
#### 系统兼容性修复 / System Compatibility Fixes
|
||||||
|
- **Windows脚本**: 修复Windows daemon.bat路径配置问题 ([e5a41765b](https://github.com/tencentmusic/supersonic/commit/e5a41765b) by 柯慕灵)
|
||||||
|
- **字符编码**: 将utf8编码修改为utf8mb4,解决字符问题 ([2e81b190a](https://github.com/tencentmusic/supersonic/commit/2e81b190a) by Kun Gu)
|
||||||
|
- **记忆缓存**: 修复记忆管理中因缓存无法存储的问题 ([81cd60d2d](https://github.com/tencentmusic/supersonic/commit/81cd60d2d) by guilinlewis)
|
||||||
|
- **Mac兼容**: 降级djl库以支持Mac Intel机器 ([bf3213e8f](https://github.com/tencentmusic/supersonic/commit/bf3213e8f) by jerryjzhang)
|
||||||
|
|
||||||
|
### 数据管理优化 / Data Management Improvements
|
||||||
|
|
||||||
|
#### 维度指标管理 / Dimension & Metric Management
|
||||||
|
- **维度检索**: 修复维度和指标检索及百分比显示问题 ([d8fe2ed2b](https://github.com/tencentmusic/supersonic/commit/d8fe2ed2b) by 木鱼和尚)
|
||||||
|
- **查询导出**: 基于queryColumns导出数据 ([11d1264d3](https://github.com/tencentmusic/supersonic/commit/11d1264d3) by FredTsang)
|
||||||
|
- **表格排序**: 移除表格defaultSortOrder ([32675387d](https://github.com/tencentmusic/supersonic/commit/32675387d) by FredTsang)
|
||||||
|
- **维度搜索**: 修复维度搜索带key查询范围超出问题 ([269f146c1](https://github.com/tencentmusic/supersonic/commit/269f146c1) by wangyong)
|
||||||
|
|
||||||
|
### 测试和质量保证 / Testing & Quality Assurance
|
||||||
|
|
||||||
|
#### 单元测试 / Unit Testing
|
||||||
|
- **测试修复**: 修复单元测试用例 ([91e4b51ef](https://github.com/tencentmusic/supersonic/commit/91e4b51ef) by jerryjzhang)
|
||||||
|
- **模型测试**: 修复ModelCreateForm.tsx错误 ([d2aa73b85](https://github.com/tencentmusic/supersonic/commit/d2aa73b85) by Antgeek)
|
||||||
|
|
||||||
|
### 重要变更说明 / Breaking Changes
|
||||||
|
|
||||||
|
#### 升级注意事项 / Upgrade Notes
|
||||||
|
1. **SpringBoot 3升级**: 可能需要更新依赖配置和代码适配
|
||||||
|
2. **许可证变更**: 从MIT变更为Apache 2.0,请注意法律合规
|
||||||
|
3. **API接口调整**: 部分API接口为支持新功能进行了调整
|
||||||
|
4. **数据库兼容**: 新增多种数据库支持,配置方式有所变化
|
||||||
|
|
||||||
|
### 完整提交统计 / Commit Statistics
|
||||||
|
- **总提交数**: 419个提交
|
||||||
|
- **主要贡献者**:
|
||||||
|
- jerryjzhang: 158次提交
|
||||||
|
- supersonicbi: 22次提交
|
||||||
|
- zyclove: 20次提交
|
||||||
|
- beat4ocean: 15次提交
|
||||||
|
- guilinlewis: 11次提交
|
||||||
|
- wangyong: 11次提交
|
||||||
|
- 其他贡献者: 182次提交
|
||||||
|
- **涉及模块**: headless, chat, auth, common, webapp, launcher, docker
|
||||||
|
- **时间跨度**: 2024年11月1日 - 2025年8月5日
|
||||||
|
|
||||||
|
### 致谢 / Acknowledgments
|
||||||
|
|
||||||
|
感谢所有为SuperSonic 1.0.0版本贡献代码、文档、测试和建议的开发者们!🎉
|
||||||
|
|
||||||
|
#### 核心贡献者 / Core Contributors
|
||||||
|
- **jerryjzhang** - 项目维护者,核心架构设计与实现
|
||||||
|
- **supersonicbi** - 核心功能开发,多数据库支持
|
||||||
|
- **beat4ocean** - 架构升级,依赖管理,安全优化
|
||||||
|
- **zyclove** - 数据库适配,构建优化
|
||||||
|
- **guilinlewis** - 鉴权系统,召回优化
|
||||||
|
- **wangyong** - Bug修复,NPE异常处理
|
||||||
|
|
||||||
|
#### 活跃贡献者 / Active Contributors
|
||||||
|
- **WDEP** - 前端优化,图表功能
|
||||||
|
- **FredTsang** - Chat SDK优化,数据导出
|
||||||
|
- **feelshana** - 记忆管理,向量召回
|
||||||
|
- **QJ_wonder** - 插件功能,多轮对话
|
||||||
|
- **Willy-J** - 模型关联,数据库兼容
|
||||||
|
- **iridescentpeo** - 查询优化,模型管理
|
||||||
|
- **tristanliu** - 前端路由,权限界面
|
||||||
|
- **mislayming** - SpringBoot 3升级
|
||||||
|
- **Antgeek** - 基准测试,模型修复
|
||||||
|
- **柯慕灵** - LLM纠错器,Windows脚本
|
||||||
|
- **superhero** - 项目管理,代码审查
|
||||||
|
|
||||||
|
#### 其他重要贡献者 / Other Important Contributors
|
||||||
|
- **木鱼和尚** - 维度指标检索优化
|
||||||
|
- **pisces** - 图表导出功能
|
||||||
|
- **lexluo09** - 并行处理,GitHub Actions
|
||||||
|
- **andybj0228** - SQL查询优化
|
||||||
|
- **wanglongqiang** - SQL变量支持
|
||||||
|
- **Hyman_bz** - StarRocks支持
|
||||||
|
- **wwsheng009** - SAP HANA适配
|
||||||
|
- **poncheen** - TypeScript错误修复
|
||||||
|
- **kino** - Docker镜像优化
|
||||||
|
- **coosir** - 前端百分比显示
|
||||||
|
- **Kun Gu** - 字符编码优化
|
||||||
|
- **chixiaopao** - NPE异常修复
|
||||||
|
- **naimehao** - 核心功能修复
|
||||||
|
- **yudong** - 记忆评估优化
|
||||||
|
- **mroldx** - 数据库脚本更新
|
||||||
|
- **ChPi** - 解析器性能优化
|
||||||
|
- **Hwting** - Docker配置优化
|
||||||
|
|
||||||
|
#### 特别感谢 / Special Thanks
|
||||||
|
感谢所有提交Issue、参与讨论、提供反馈的社区用户,你们的每一个建议都让SuperSonic变得更好!
|
||||||
|
|
||||||
|
#### 社区支持 / Community Support
|
||||||
|
SuperSonic是一个开源项目,我们欢迎更多开发者加入:
|
||||||
|
- 🔗 **GitHub**: https://github.com/tencentmusic/supersonic
|
||||||
|
- 📖 **文档**: 详见项目README和Wiki
|
||||||
|
- 🐛 **Issue报告**: 欢迎提交Bug和功能请求
|
||||||
|
- 🚀 **贡献代码**: 欢迎提交Pull Request
|
||||||
|
- 💬 **社区讨论**: 加入我们的技术交流群
|
||||||
|
|
||||||
|
#### 未来展望 / Future Vision
|
||||||
|
SuperSonic 1.0.0是一个重要的里程碑,但这只是开始。我们将继续:
|
||||||
|
- 🌟 **持续优化性能和稳定性**
|
||||||
|
- 🔧 **扩展更多数据库和AI模型支持**
|
||||||
|
- 🎨 **改善用户体验和界面设计**
|
||||||
|
- 📚 **完善文档和最佳实践**
|
||||||
|
- 🤝 **建设更活跃的开源社区**
|
||||||
|
|
||||||
|
**让我们一起把SuperSonic做得更好!** ✨
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*如果您在使用过程中遇到问题或有改进建议,欢迎随时与我们交流。每一份贡献都让SuperSonic更加强大!*
|
||||||
|
|
||||||
|
|
||||||
## SuperSonic [0.9.8] - 2024-11-01
|
## SuperSonic [0.9.8] - 2024-11-01
|
||||||
- Add LLM management module to reuse connection across agents.
|
- Add LLM management module to reuse connection across agents.
|
||||||
|
|||||||
2
LICENSE
2
LICENSE
@@ -14,7 +14,7 @@ code and logo.
|
|||||||
b. a commercial license must be obtained from the author if you want to develop and distribute a derivative work based
|
b. a commercial license must be obtained from the author if you want to develop and distribute a derivative work based
|
||||||
on SuperSonic.
|
on SuperSonic.
|
||||||
|
|
||||||
Please contact zhangjun2915@163.com by email to inquire about licensing matters.
|
Please contact supersonicbi@qq.com by email to inquire about licensing matters.
|
||||||
|
|
||||||
|
|
||||||
2. As a contributor, you should agree that:
|
2. As a contributor, you should agree that:
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ SuperSonic comes with sample semantic models as well as chat conversations that
|
|||||||
|
|
||||||
## Build and Development
|
## Build and Development
|
||||||
|
|
||||||
Please refer to project [Docs](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E7%BC%96%E8%AF%91%E6%9E%84%E5%BB%BA/).
|
Please refer to project [Docs](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E6%BA%90%E7%A0%81%E7%BC%96%E8%AF%91%E9%83%A8%E7%BD%B2/).
|
||||||
|
|
||||||
## WeChat Contact
|
## WeChat Contact
|
||||||
|
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ SuperSonic自带样例的语义模型和问答对话,只需以下三步即可
|
|||||||
|
|
||||||
## 如何构建和部署
|
## 如何构建和部署
|
||||||
|
|
||||||
请参考项目[文档](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E7%BC%96%E8%AF%91%E6%9E%84%E5%BB%BA/)。
|
请参考项目[文档](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E6%BA%90%E7%A0%81%E7%BC%96%E8%AF%91%E9%83%A8%E7%BD%B2/)。
|
||||||
|
|
||||||
## 微信联系方式
|
## 微信联系方式
|
||||||
|
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ SuperSonicには、サンプルのセマンティックモデルとチャット
|
|||||||
|
|
||||||
## ビルドと開発
|
## ビルドと開発
|
||||||
|
|
||||||
プロジェクト[ドキュメント](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E7%BC%96%E8%AF%91%E6%9E%84%E5%BB%BA/)を参照してください。
|
プロジェクト[ドキュメント](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E6%BA%90%E7%A0%81%E7%BC%96%E8%AF%91%E9%83%A8%E7%BD%B2/)を参照してください。
|
||||||
|
|
||||||
## WeChat連絡先
|
## WeChat連絡先
|
||||||
|
|
||||||
|
|||||||
@@ -43,10 +43,26 @@ if "%service%"=="webapp" (
|
|||||||
call mvn -f %projectDir% clean package -DskipTests -Dspotless.skip=true
|
call mvn -f %projectDir% clean package -DskipTests -Dspotless.skip=true
|
||||||
IF ERRORLEVEL 1 (
|
IF ERRORLEVEL 1 (
|
||||||
ECHO Failed to build backend Java modules.
|
ECHO Failed to build backend Java modules.
|
||||||
|
ECHO Please check Maven and Java versions are compatible.
|
||||||
|
ECHO Current Java: %JAVA_HOME%
|
||||||
|
ECHO Current Maven: %MAVEN_HOME%
|
||||||
EXIT /B 1
|
EXIT /B 1
|
||||||
)
|
)
|
||||||
|
|
||||||
|
REM extract and copy files to deployment directory
|
||||||
|
cd %projectDir%\launchers\%model_name%\target
|
||||||
|
if exist "launchers-%model_name%-%MVN_VERSION%-bin.tar.gz" (
|
||||||
|
echo "Extracting launchers-%model_name%-%MVN_VERSION%-bin.tar.gz..."
|
||||||
|
tar -xf "launchers-%model_name%-%MVN_VERSION%-bin.tar.gz"
|
||||||
|
if exist "launchers-%model_name%-%MVN_VERSION%" (
|
||||||
|
echo "Copying files to deployment directory..."
|
||||||
|
xcopy /E /Y "launchers-%model_name%-%MVN_VERSION%\*" "%buildDir%\supersonic-%model_name%-%MVN_VERSION%\"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
copy /y %projectDir%\launchers\%model_name%\target\*.tar.gz %buildDir%\
|
copy /y %projectDir%\launchers\%model_name%\target\*.tar.gz %buildDir%\
|
||||||
echo "finished building supersonic-%model_name% service"
|
echo "finished building supersonic-%model_name% service"
|
||||||
|
cd %baseDir%
|
||||||
goto :EOF
|
goto :EOF
|
||||||
|
|
||||||
|
|
||||||
@@ -72,22 +88,55 @@ if "%service%"=="webapp" (
|
|||||||
cd %buildDir%
|
cd %buildDir%
|
||||||
if exist %release_dir% rmdir /s /q %release_dir%
|
if exist %release_dir% rmdir /s /q %release_dir%
|
||||||
if exist %release_dir%.zip del %release_dir%.zip
|
if exist %release_dir%.zip del %release_dir%.zip
|
||||||
mkdir %release_dir%
|
|
||||||
rem package webapp
|
rem check if release directory already exists from buildJavaService
|
||||||
tar xvf supersonic-webapp.tar.gz
|
if exist %release_dir% (
|
||||||
move /y supersonic-webapp webapp
|
echo "Release directory already prepared by buildJavaService"
|
||||||
echo {"env": ""} > webapp\supersonic.config.json
|
) else (
|
||||||
move /y webapp %release_dir%
|
mkdir %release_dir%
|
||||||
rem package java service
|
|
||||||
tar xvf %service_name%-bin.tar.gz
|
rem package java service
|
||||||
for /d %%D in ("%service_name%\*") do (
|
tar xvf %service_name%-bin.tar.gz 2>nul
|
||||||
move "%%D" "%release_dir%"
|
if errorlevel 1 (
|
||||||
|
echo "Warning: tar command failed, trying PowerShell extraction..."
|
||||||
|
powershell -Command "Expand-Archive -Path '%service_name%-bin.tar.gz' -DestinationPath '.' -Force"
|
||||||
|
)
|
||||||
|
for /d %%D in ("%service_name%\*") do (
|
||||||
|
move "%%D" "%release_dir%"
|
||||||
|
)
|
||||||
|
rmdir /s /q %service_name% 2>nul
|
||||||
)
|
)
|
||||||
|
|
||||||
|
rem package webapp
|
||||||
|
if exist supersonic-webapp.tar.gz (
|
||||||
|
tar xvf supersonic-webapp.tar.gz 2>nul
|
||||||
|
if errorlevel 1 (
|
||||||
|
echo "Warning: tar command failed, trying PowerShell extraction..."
|
||||||
|
powershell -Command "Expand-Archive -Path 'supersonic-webapp.tar.gz' -DestinationPath '.' -Force"
|
||||||
|
)
|
||||||
|
move /y supersonic-webapp webapp
|
||||||
|
echo {"env": ""} > webapp\supersonic.config.json
|
||||||
|
move /y webapp %release_dir%
|
||||||
|
del supersonic-webapp.tar.gz 2>nul
|
||||||
|
)
|
||||||
|
|
||||||
|
rem verify deployment structure
|
||||||
|
if exist "%release_dir%\lib\launchers-%model_name%-%MVN_VERSION%.jar" (
|
||||||
|
echo "Deployment structure verified successfully"
|
||||||
|
) else (
|
||||||
|
echo "Warning: Main jar file not found in deployment structure"
|
||||||
|
echo "Expected: %release_dir%\lib\launchers-%model_name%-%MVN_VERSION%.jar"
|
||||||
|
)
|
||||||
|
|
||||||
rem generate zip file
|
rem generate zip file
|
||||||
powershell Compress-Archive -Path %release_dir% -DestinationPath %release_dir%.zip
|
powershell -Command "Compress-Archive -Path '%release_dir%' -DestinationPath '%release_dir%.zip' -Force"
|
||||||
del %service_name%-bin.tar.gz
|
if errorlevel 1 (
|
||||||
del supersonic-webapp.tar.gz
|
echo "Warning: PowerShell compression failed, release directory still available: %release_dir%"
|
||||||
rmdir /s /q %service_name%
|
) else (
|
||||||
|
echo "Successfully created release package: %release_dir%.zip"
|
||||||
|
)
|
||||||
|
|
||||||
|
del %service_name%-bin.tar.gz 2>nul
|
||||||
echo "finished packaging supersonic release"
|
echo "finished packaging supersonic release"
|
||||||
goto :EOF
|
goto :EOF
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ if "%profile%"=="" (
|
|||||||
|
|
||||||
set "model_name=%service%"
|
set "model_name=%service%"
|
||||||
|
|
||||||
cd %baseDir%
|
REM fix path configuration - point to the correct release package directory
|
||||||
|
set "releaseDir=%buildDir%\supersonic-%service%-1.0.0-SNAPSHOT"
|
||||||
|
cd %releaseDir%
|
||||||
|
|
||||||
if "%command%"=="restart" (
|
if "%command%"=="restart" (
|
||||||
call :stop
|
call :stop
|
||||||
@@ -50,20 +52,58 @@ if "%command%"=="restart" (
|
|||||||
|
|
||||||
:runJavaService
|
:runJavaService
|
||||||
echo 'java service starting, see logs in logs/'
|
echo 'java service starting, see logs in logs/'
|
||||||
set "libDir=%baseDir%\lib"
|
echo 'Using release directory: %releaseDir%'
|
||||||
set "confDir=%baseDir%\conf"
|
|
||||||
set "webDir=%baseDir%\webapp"
|
REM use release package directory as base path
|
||||||
set "logDir=%baseDir%\logs"
|
set "libDir=%releaseDir%\lib"
|
||||||
set "classpath=%baseDir%;%webDir%;%libDir%\*;%confDir%"
|
set "confDir=%releaseDir%\conf"
|
||||||
set "property=-Dfile.encoding=UTF-8 -Duser.language=Zh -Duser.region=CN -Duser.timezone=GMT+08 -Dspring.profiles.active=%profile%"
|
set "webDir=%releaseDir%\webapp"
|
||||||
set "java-command=%property% -Xms1024m -Xmx1024m -cp %CLASSPATH% %MAIN_CLASS%"
|
set "logDir=%releaseDir%\logs"
|
||||||
|
|
||||||
|
REM fix variable name matching problem
|
||||||
|
set "CLASSPATH=%releaseDir%;%webDir%;%libDir%\*;%confDir%"
|
||||||
|
set "MAIN_CLASS=%main_class%"
|
||||||
|
|
||||||
|
REM add port configuration
|
||||||
|
set "property=-Dfile.encoding=UTF-8 -Duser.language=Zh -Duser.region=CN -Duser.timezone=GMT+08 -Dspring.profiles.active=%profile% -Dserver.port=9080"
|
||||||
|
set "java_command=%property% -Xms1024m -Xmx2048m -cp "%CLASSPATH%" %MAIN_CLASS%"
|
||||||
|
|
||||||
if not exist %logDir% mkdir %logDir%
|
if not exist %logDir% mkdir %logDir%
|
||||||
start /B java %java-command% >nul 2>&1
|
|
||||||
timeout /t 10 >nul
|
REM check if the main jar file exists
|
||||||
|
if not exist "%libDir%\launchers-standalone-1.0.0-SNAPSHOT.jar" (
|
||||||
|
echo "Error: Main jar file not found in %libDir%"
|
||||||
|
echo "Please make sure the application has been built and packaged correctly."
|
||||||
|
goto :EOF
|
||||||
|
)
|
||||||
|
|
||||||
|
echo 'Main Class: %MAIN_CLASS%'
|
||||||
|
echo 'Profile: %profile%'
|
||||||
|
echo 'Starting Java service...'
|
||||||
|
|
||||||
|
REM start service and save logs
|
||||||
|
start /B java %java_command% > "%logDir%\supersonic.log" 2>&1
|
||||||
|
timeout /t 15 >nul
|
||||||
|
|
||||||
|
REM check service status
|
||||||
|
netstat -an | findstr ":9080" >nul
|
||||||
|
if errorlevel 1 (
|
||||||
|
echo "Warning: Port 9080 is not listening"
|
||||||
|
echo "Please check the log file: %logDir%\supersonic.log"
|
||||||
|
if exist "%logDir%\supersonic.log" (
|
||||||
|
echo "Recent log entries:"
|
||||||
|
powershell -Command "Get-Content '%logDir%\supersonic.log' | Select-Object -Last 10"
|
||||||
|
)
|
||||||
|
) else (
|
||||||
|
echo "Service started successfully on port 9080"
|
||||||
|
echo "You can access the application at: http://localhost:9080"
|
||||||
|
)
|
||||||
|
|
||||||
echo 'java service started'
|
echo 'java service started'
|
||||||
goto :EOF
|
goto :EOF
|
||||||
|
|
||||||
:stopJavaService
|
:stopJavaService
|
||||||
|
echo 'Stopping Java service...'
|
||||||
for /f "tokens=2" %%i in ('tasklist ^| findstr /i "java"') do (
|
for /f "tokens=2" %%i in ('tasklist ^| findstr /i "java"') do (
|
||||||
taskkill /PID %%i /F
|
taskkill /PID %%i /F
|
||||||
echo "java service (PID = %%i) is killed."
|
echo "java service (PID = %%i) is killed."
|
||||||
|
|||||||
@@ -60,7 +60,8 @@ function runJavaService {
|
|||||||
JAVA_HOME=$(ls /usr/jdk64/jdk* -d 2>/dev/null | xargs | awk '{print "'$local_app_name'"}')
|
JAVA_HOME=$(ls /usr/jdk64/jdk* -d 2>/dev/null | xargs | awk '{print "'$local_app_name'"}')
|
||||||
fi
|
fi
|
||||||
export PATH=$JAVA_HOME/bin:$PATH
|
export PATH=$JAVA_HOME/bin:$PATH
|
||||||
command="-Dfile.encoding=UTF-8 -Duser.language=Zh -Duser.region=CN -Duser.timezone=GMT+08 -Dapp_name=${local_app_name} -Xms1024m -Xmx1024m $main_class"
|
command="-Dfile.encoding=UTF-8 -Duser.language=Zh -Duser.region=CN -Duser.timezone=GMT+08
|
||||||
|
-Dapp_name=${local_app_name} -Xms1024m -Xmx2048m -XX:+UseZGC -XX:+ZGenerational $main_class"
|
||||||
|
|
||||||
mkdir -p $javaRunDir/logs
|
mkdir -p $javaRunDir/logs
|
||||||
java -Dspring.profiles.active="$profile" $command >/dev/null 2>$javaRunDir/logs/error.log &
|
java -Dspring.profiles.active="$profile" $command >/dev/null 2>$javaRunDir/logs/error.log &
|
||||||
|
|||||||
5
assembly/bin/supersonic-docker-compose.sh
Normal file
5
assembly/bin/supersonic-docker-compose.sh
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
#!/usr/bin/env sh
|
||||||
|
|
||||||
|
export SUPERSONIC_VERSION=latest
|
||||||
|
|
||||||
|
docker-compose -f docker-compose.yml -p supersonic up
|
||||||
23
assembly/bin/supersonic-docker-run.sh
Normal file
23
assembly/bin/supersonic-docker-run.sh
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env sh
|
||||||
|
|
||||||
|
export SUPERSONIC_VERSION=latest
|
||||||
|
|
||||||
|
#### Set below DB configs to connect to your own database
|
||||||
|
# Supported DB_TYPE: h2, mysql, postgres
|
||||||
|
export S2_DB_TYPE=h2
|
||||||
|
export S2_DB_HOST=
|
||||||
|
export S2_DB_PORT=
|
||||||
|
export S2_DB_USER=
|
||||||
|
export S2_DB_PASSWORD=
|
||||||
|
export S2_DB_DATABASE=
|
||||||
|
|
||||||
|
docker run --rm -it -d \
|
||||||
|
--name supersonic_standalone \
|
||||||
|
-p 9080:9080 \
|
||||||
|
-e S2_DB_TYPE=${S2_DB_TYPE} \
|
||||||
|
-e S2_DB_HOST=${S2_DB_HOST} \
|
||||||
|
-e S2_DB_PORT=${S2_DB_PORT} \
|
||||||
|
-e S2_DB_USER=${S2_DB_USER} \
|
||||||
|
-e S2_DB_PASSWORD=${S2_DB_PASSWORD} \
|
||||||
|
-e S2_DB_DATABASE=${S2_DB_DATABASE} \
|
||||||
|
supersonicbi/supersonic:${SUPERSONIC_VERSION}
|
||||||
@@ -1,10 +1,11 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
#### Set below DB configs to connect to your own database
|
#### Set below DB configs to connect to your own database
|
||||||
|
# Comment out below exports to config your DB connection
|
||||||
# Supported DB_TYPE: h2, mysql, postgres
|
# Supported DB_TYPE: h2, mysql, postgres
|
||||||
export S2_DB_TYPE=h2
|
#export S2_DB_TYPE=h2
|
||||||
export S2_DB_HOST=
|
#export S2_DB_HOST=
|
||||||
export S2_DB_PORT=
|
#export S2_DB_PORT=
|
||||||
export S2_DB_USER=
|
#export S2_DB_USER=
|
||||||
export S2_DB_PASSWORD=
|
#export S2_DB_PASSWORD=
|
||||||
export S2_DB_DATABASE=
|
#export S2_DB_DATABASE=
|
||||||
@@ -34,8 +34,8 @@
|
|||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
<maven.compiler.source>8</maven.compiler.source>
|
<maven.compiler.source>21</maven.compiler.source>
|
||||||
<maven.compiler.target>8</maven.compiler.target>
|
<maven.compiler.target>21</maven.compiler.target>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
@@ -21,6 +21,8 @@ public interface UserAdaptor {
|
|||||||
|
|
||||||
void register(UserReq userReq);
|
void register(UserReq userReq);
|
||||||
|
|
||||||
|
void deleteUser(long userId);
|
||||||
|
|
||||||
String login(UserReq userReq, HttpServletRequest request);
|
String login(UserReq userReq, HttpServletRequest request);
|
||||||
|
|
||||||
String login(UserReq userReq, String appKey);
|
String login(UserReq userReq, String appKey);
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ public class UserWithPassword extends User {
|
|||||||
|
|
||||||
public UserWithPassword(Long id, String name, String displayName, String email, String password,
|
public UserWithPassword(Long id, String name, String displayName, String email, String password,
|
||||||
Integer isAdmin) {
|
Integer isAdmin) {
|
||||||
super(id, name, displayName, email, isAdmin);
|
super(id, name, displayName, email, isAdmin, null);
|
||||||
this.password = password;
|
this.password = password;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ public interface UserService {
|
|||||||
|
|
||||||
void register(UserReq userCmd);
|
void register(UserReq userCmd);
|
||||||
|
|
||||||
|
void deleteUser(long userId);
|
||||||
|
|
||||||
String login(UserReq userCmd, HttpServletRequest request);
|
String login(UserReq userCmd, HttpServletRequest request);
|
||||||
|
|
||||||
String login(UserReq userCmd, String appKey);
|
String login(UserReq userCmd, String appKey);
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ import jakarta.servlet.http.HttpServletRequest;
|
|||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
|
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@@ -90,6 +92,12 @@ public class DefaultUserAdaptor implements UserAdaptor {
|
|||||||
userRepository.addUser(userDO);
|
userRepository.addUser(userDO);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteUser(long userId) {
|
||||||
|
UserRepository userRepository = ContextUtils.getBean(UserRepository.class);
|
||||||
|
userRepository.deleteUser(userId);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String login(UserReq userReq, HttpServletRequest request) {
|
public String login(UserReq userReq, HttpServletRequest request) {
|
||||||
TokenService tokenService = ContextUtils.getBean(TokenService.class);
|
TokenService tokenService = ContextUtils.getBean(TokenService.class);
|
||||||
@@ -102,7 +110,9 @@ public class DefaultUserAdaptor implements UserAdaptor {
|
|||||||
TokenService tokenService = ContextUtils.getBean(TokenService.class);
|
TokenService tokenService = ContextUtils.getBean(TokenService.class);
|
||||||
try {
|
try {
|
||||||
UserWithPassword user = getUserWithPassword(userReq);
|
UserWithPassword user = getUserWithPassword(userReq);
|
||||||
return tokenService.generateToken(UserWithPassword.convert(user), appKey);
|
String token = tokenService.generateToken(UserWithPassword.convert(user), appKey);
|
||||||
|
updateLastLogin(userReq.getName());
|
||||||
|
return token;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("", e);
|
log.error("", e);
|
||||||
throw new RuntimeException("password encrypt error, please try again");
|
throw new RuntimeException("password encrypt error, please try again");
|
||||||
@@ -213,8 +223,9 @@ public class DefaultUserAdaptor implements UserAdaptor {
|
|||||||
new UserWithPassword(userDO.getId(), userDO.getName(), userDO.getDisplayName(),
|
new UserWithPassword(userDO.getId(), userDO.getName(), userDO.getDisplayName(),
|
||||||
userDO.getEmail(), userDO.getPassword(), userDO.getIsAdmin());
|
userDO.getEmail(), userDO.getPassword(), userDO.getIsAdmin());
|
||||||
|
|
||||||
String token =
|
// 使用令牌名称作为生成key ,这样可以区分正常请求和api 请求,api 的令牌失效时间很长,需考虑令牌泄露的情况
|
||||||
tokenService.generateToken(UserWithPassword.convert(userWithPassword), expireTime);
|
String token = tokenService.generateToken(UserWithPassword.convert(userWithPassword),
|
||||||
|
"SysDbToken:" + name, (new Date().getTime() + expireTime));
|
||||||
UserTokenDO userTokenDO = saveUserToken(name, userName, token, expireTime);
|
UserTokenDO userTokenDO = saveUserToken(name, userName, token, expireTime);
|
||||||
return convertUserToken(userTokenDO);
|
return convertUserToken(userTokenDO);
|
||||||
}
|
}
|
||||||
@@ -267,4 +278,11 @@ public class DefaultUserAdaptor implements UserAdaptor {
|
|||||||
userToken.setExpireDate(userTokenDO.getExpireDateTime());
|
userToken.setExpireDate(userTokenDO.getExpireDateTime());
|
||||||
return userToken;
|
return userToken;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void updateLastLogin(String userName) {
|
||||||
|
UserRepository userRepository = ContextUtils.getBean(UserRepository.class);
|
||||||
|
UserDO userDO = userRepository.getUser(userName);
|
||||||
|
userDO.setLastLogin(new Timestamp(System.currentTimeMillis()));
|
||||||
|
userRepository.updateUser(userDO);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.auth.authentication.interceptor;
|
package com.tencent.supersonic.auth.authentication.interceptor;
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||||
import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
|
import com.tencent.supersonic.auth.api.authentication.service.UserService;
|
||||||
import com.tencent.supersonic.auth.authentication.utils.TokenService;
|
import com.tencent.supersonic.auth.authentication.utils.TokenService;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
@@ -16,7 +16,7 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor {
|
|||||||
|
|
||||||
protected AuthenticationConfig authenticationConfig;
|
protected AuthenticationConfig authenticationConfig;
|
||||||
|
|
||||||
protected UserServiceImpl userServiceImpl;
|
protected UserService userService;
|
||||||
|
|
||||||
protected TokenService tokenService;
|
protected TokenService tokenService;
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.auth.authentication.interceptor;
|
|||||||
import com.tencent.supersonic.auth.api.authentication.annotation.AuthenticationIgnore;
|
import com.tencent.supersonic.auth.api.authentication.annotation.AuthenticationIgnore;
|
||||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
|
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
|
||||||
import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
|
import com.tencent.supersonic.auth.api.authentication.service.UserService;
|
||||||
import com.tencent.supersonic.auth.authentication.utils.TokenService;
|
import com.tencent.supersonic.auth.authentication.utils.TokenService;
|
||||||
import com.tencent.supersonic.common.pojo.exception.AccessException;
|
import com.tencent.supersonic.common.pojo.exception.AccessException;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
@@ -16,12 +16,7 @@ import org.springframework.web.method.HandlerMethod;
|
|||||||
import java.lang.reflect.Method;
|
import java.lang.reflect.Method;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
|
||||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_IS_ADMIN;
|
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.*;
|
||||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_DISPLAY_NAME;
|
|
||||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_EMAIL;
|
|
||||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_ID;
|
|
||||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_NAME;
|
|
||||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_PASSWORD;
|
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor {
|
public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor {
|
||||||
@@ -30,7 +25,7 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor
|
|||||||
public boolean preHandle(HttpServletRequest request, HttpServletResponse response,
|
public boolean preHandle(HttpServletRequest request, HttpServletResponse response,
|
||||||
Object handler) throws AccessException {
|
Object handler) throws AccessException {
|
||||||
authenticationConfig = ContextUtils.getBean(AuthenticationConfig.class);
|
authenticationConfig = ContextUtils.getBean(AuthenticationConfig.class);
|
||||||
userServiceImpl = ContextUtils.getBean(UserServiceImpl.class);
|
userService = ContextUtils.getBean(UserService.class);
|
||||||
tokenService = ContextUtils.getBean(TokenService.class);
|
tokenService = ContextUtils.getBean(TokenService.class);
|
||||||
if (!authenticationConfig.isEnabled()) {
|
if (!authenticationConfig.isEnabled()) {
|
||||||
return true;
|
return true;
|
||||||
|
|||||||
@@ -3,7 +3,11 @@ package com.tencent.supersonic.auth.authentication.persistence.dataobject;
|
|||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
import com.baomidou.mybatisplus.annotation.TableName;
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
|
||||||
|
@Data
|
||||||
@TableName("s2_user")
|
@TableName("s2_user")
|
||||||
public class UserDO {
|
public class UserDO {
|
||||||
|
|
||||||
@@ -27,71 +31,25 @@ public class UserDO {
|
|||||||
/** */
|
/** */
|
||||||
private Integer isAdmin;
|
private Integer isAdmin;
|
||||||
|
|
||||||
/** @return id */
|
private Timestamp lastLogin;
|
||||||
public Long getId() {
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param id */
|
|
||||||
public void setId(Long id) {
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return name */
|
|
||||||
public String getName() {
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param name */
|
/** @param name */
|
||||||
public void setName(String name) {
|
public void setName(String name) {
|
||||||
this.name = name == null ? null : name.trim();
|
this.name = name == null ? null : name.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return password */
|
|
||||||
public String getPassword() {
|
|
||||||
return password;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param password */
|
/** @param password */
|
||||||
public void setPassword(String password) {
|
public void setPassword(String password) {
|
||||||
this.password = password == null ? null : password.trim();
|
this.password = password == null ? null : password.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getSalt() {
|
|
||||||
return salt;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSalt(String salt) {
|
public void setSalt(String salt) {
|
||||||
this.salt = salt == null ? null : salt.trim();
|
this.salt = salt == null ? null : salt.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return display_name */
|
|
||||||
public String getDisplayName() {
|
|
||||||
return displayName;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param displayName */
|
|
||||||
public void setDisplayName(String displayName) {
|
|
||||||
this.displayName = displayName == null ? null : displayName.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return email */
|
|
||||||
public String getEmail() {
|
|
||||||
return email;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param email */
|
/** @param email */
|
||||||
public void setEmail(String email) {
|
public void setEmail(String email) {
|
||||||
this.email = email == null ? null : email.trim();
|
this.email = email == null ? null : email.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return is_admin */
|
|
||||||
public Integer getIsAdmin() {
|
|
||||||
return isAdmin;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param isAdmin */
|
|
||||||
public void setIsAdmin(Integer isAdmin) {
|
|
||||||
this.isAdmin = isAdmin;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,11 @@ public interface UserRepository {
|
|||||||
|
|
||||||
UserTokenDO getUserToken(Long tokenId);
|
UserTokenDO getUserToken(Long tokenId);
|
||||||
|
|
||||||
|
UserTokenDO getUserTokenByName(String tokenName);
|
||||||
|
|
||||||
void deleteUserTokenByName(String userName);
|
void deleteUserTokenByName(String userName);
|
||||||
|
|
||||||
void deleteUserToken(Long tokenId);
|
void deleteUserToken(Long tokenId);
|
||||||
|
|
||||||
|
void deleteUser(long userId);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,6 +65,13 @@ public class UserRepositoryImpl implements UserRepository {
|
|||||||
return userTokenDOMapper.selectById(tokenId);
|
return userTokenDOMapper.selectById(tokenId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public UserTokenDO getUserTokenByName(String tokenName) {
|
||||||
|
QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
|
||||||
|
queryWrapper.lambda().eq(UserTokenDO::getName, tokenName);
|
||||||
|
return userTokenDOMapper.selectOne(queryWrapper);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void deleteUserTokenByName(String userName) {
|
public void deleteUserTokenByName(String userName) {
|
||||||
QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
|
QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
|
||||||
@@ -76,4 +83,9 @@ public class UserRepositoryImpl implements UserRepository {
|
|||||||
public void deleteUserToken(Long tokenId) {
|
public void deleteUserToken(Long tokenId) {
|
||||||
userTokenDOMapper.deleteById(tokenId);
|
userTokenDOMapper.deleteById(tokenId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteUser(long userId) {
|
||||||
|
userDOMapper.deleteById(userId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,13 +9,7 @@ import com.tencent.supersonic.common.pojo.User;
|
|||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import jakarta.servlet.http.HttpServletResponse;
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.*;
|
||||||
import org.springframework.web.bind.annotation.PathVariable;
|
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestBody;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@@ -67,6 +61,16 @@ public class UserController {
|
|||||||
userService.register(userCmd);
|
userService.register(userCmd);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@DeleteMapping("/delete/{userId}")
|
||||||
|
public void delete(@PathVariable("userId") long userId, HttpServletRequest httpServletRequest,
|
||||||
|
HttpServletResponse httpServletResponse) throws IllegalAccessException {
|
||||||
|
User user = userService.getCurrentUser(httpServletRequest, httpServletResponse);
|
||||||
|
if (user.getIsAdmin() != 1) {
|
||||||
|
throw new IllegalAccessException("only admin can delete user");
|
||||||
|
}
|
||||||
|
userService.deleteUser(userId);
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/login")
|
@PostMapping("/login")
|
||||||
public String login(@RequestBody UserReq userCmd, HttpServletRequest request) {
|
public String login(@RequestBody UserReq userCmd, HttpServletRequest request) {
|
||||||
return userService.login(userCmd, request);
|
return userService.login(userCmd, request);
|
||||||
|
|||||||
@@ -70,6 +70,11 @@ public class UserServiceImpl implements UserService {
|
|||||||
ComponentFactory.getUserAdaptor().register(userReq);
|
ComponentFactory.getUserAdaptor().register(userReq);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteUser(long userId) {
|
||||||
|
ComponentFactory.getUserAdaptor().deleteUser(userId);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String login(UserReq userReq, HttpServletRequest request) {
|
public String login(UserReq userReq, HttpServletRequest request) {
|
||||||
return ComponentFactory.getUserAdaptor().login(userReq, request);
|
return ComponentFactory.getUserAdaptor().login(userReq, request);
|
||||||
|
|||||||
@@ -6,7 +6,10 @@ import javax.crypto.spec.SecretKeySpec;
|
|||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
|
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
|
||||||
|
import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserTokenDO;
|
||||||
|
import com.tencent.supersonic.auth.authentication.persistence.repository.UserRepository;
|
||||||
import com.tencent.supersonic.common.pojo.exception.AccessException;
|
import com.tencent.supersonic.common.pojo.exception.AccessException;
|
||||||
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import io.jsonwebtoken.Claims;
|
import io.jsonwebtoken.Claims;
|
||||||
import io.jsonwebtoken.Jwts;
|
import io.jsonwebtoken.Jwts;
|
||||||
import io.jsonwebtoken.SignatureAlgorithm;
|
import io.jsonwebtoken.SignatureAlgorithm;
|
||||||
@@ -71,6 +74,7 @@ public class TokenService {
|
|||||||
return generateToken(UserWithPassword.convert(appUser), request);
|
return generateToken(UserWithPassword.convert(appUser), request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public Optional<Claims> getClaims(HttpServletRequest request) {
|
public Optional<Claims> getClaims(HttpServletRequest request) {
|
||||||
String token = request.getHeader(authenticationConfig.getTokenHttpHeaderKey());
|
String token = request.getHeader(authenticationConfig.getTokenHttpHeaderKey());
|
||||||
String appKey = getAppKey(request);
|
String appKey = getAppKey(request);
|
||||||
@@ -90,6 +94,14 @@ public class TokenService {
|
|||||||
|
|
||||||
public Optional<Claims> getClaims(String token, String appKey) {
|
public Optional<Claims> getClaims(String token, String appKey) {
|
||||||
try {
|
try {
|
||||||
|
if (StringUtils.isNotBlank(appKey) && appKey.startsWith("SysDbToken:")) {// 如果是配置的长期令牌,需校验数据库是否存在该配置
|
||||||
|
UserRepository userRepository = ContextUtils.getBean(UserRepository.class);
|
||||||
|
UserTokenDO dbToken =
|
||||||
|
userRepository.getUserTokenByName(appKey.substring("SysDbToken:".length()));
|
||||||
|
if (dbToken == null || !dbToken.getToken().equals(token.replace("Bearer ", ""))) {
|
||||||
|
throw new AccessException("Token does not exist :" + appKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
String tokenSecret = getTokenSecret(appKey);
|
String tokenSecret = getTokenSecret(appKey);
|
||||||
Claims claims =
|
Claims claims =
|
||||||
Jwts.parser().setSigningKey(tokenSecret.getBytes(StandardCharsets.UTF_8))
|
Jwts.parser().setSigningKey(tokenSecret.getBytes(StandardCharsets.UTF_8))
|
||||||
@@ -122,6 +134,16 @@ public class TokenService {
|
|||||||
Map<String, String> appKeyToSecretMap = authenticationConfig.getAppKeyToSecretMap();
|
Map<String, String> appKeyToSecretMap = authenticationConfig.getAppKeyToSecretMap();
|
||||||
String secret = appKeyToSecretMap.get(appKey);
|
String secret = appKeyToSecretMap.get(appKey);
|
||||||
if (StringUtils.isBlank(secret)) {
|
if (StringUtils.isBlank(secret)) {
|
||||||
|
if (StringUtils.isNotBlank(appKey) && appKey.startsWith("SysDbToken:")) { // 是配置的长期令牌
|
||||||
|
String realAppKey = appKey.substring("SysDbToken:".length());
|
||||||
|
String tmp =
|
||||||
|
"WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ==";
|
||||||
|
if (tmp.length() <= realAppKey.length()) {
|
||||||
|
return realAppKey;
|
||||||
|
} else {
|
||||||
|
return realAppKey + tmp.substring(realAppKey.length());
|
||||||
|
}
|
||||||
|
}
|
||||||
throw new AccessException("get secret from appKey failed :" + appKey);
|
throw new AccessException("get secret from appKey failed :" + appKey);
|
||||||
}
|
}
|
||||||
return secret;
|
return secret;
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
<result column="display_name" jdbcType="VARCHAR" property="displayName" />
|
<result column="display_name" jdbcType="VARCHAR" property="displayName" />
|
||||||
<result column="email" jdbcType="VARCHAR" property="email" />
|
<result column="email" jdbcType="VARCHAR" property="email" />
|
||||||
<result column="is_admin" jdbcType="INTEGER" property="isAdmin" />
|
<result column="is_admin" jdbcType="INTEGER" property="isAdmin" />
|
||||||
|
<result column="last_login" jdbcType="TIMESTAMP" property="lastLogin" />
|
||||||
</resultMap>
|
</resultMap>
|
||||||
<sql id="Example_Where_Clause">
|
<sql id="Example_Where_Clause">
|
||||||
<where>
|
<where>
|
||||||
@@ -40,7 +41,7 @@
|
|||||||
</where>
|
</where>
|
||||||
</sql>
|
</sql>
|
||||||
<sql id="Base_Column_List">
|
<sql id="Base_Column_List">
|
||||||
id, name, password, salt, display_name, email, is_admin
|
id, name, password, salt, display_name, email, is_admin, last_login
|
||||||
</sql>
|
</sql>
|
||||||
<select id="selectByExample" parameterType="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample" resultMap="BaseResultMap">
|
<select id="selectByExample" parameterType="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample" resultMap="BaseResultMap">
|
||||||
select
|
select
|
||||||
@@ -136,6 +137,9 @@
|
|||||||
<if test="isAdmin != null">
|
<if test="isAdmin != null">
|
||||||
is_admin = #{isAdmin,jdbcType=INTEGER},
|
is_admin = #{isAdmin,jdbcType=INTEGER},
|
||||||
</if>
|
</if>
|
||||||
|
<if test="lastLogin != null">
|
||||||
|
last_login = #{lastLogin,jdbcType=TIMESTAMP},
|
||||||
|
</if>
|
||||||
</set>
|
</set>
|
||||||
where id = #{id,jdbcType=BIGINT}
|
where id = #{id,jdbcType=BIGINT}
|
||||||
</update>
|
</update>
|
||||||
|
|||||||
@@ -15,6 +15,68 @@ import requests
|
|||||||
import time
|
import time
|
||||||
import jwt
|
import jwt
|
||||||
import traceback
|
import traceback
|
||||||
|
import os
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
class DataFrameAppender:
|
||||||
|
def __init__(self,file_name = "output"):
|
||||||
|
# 定义表头
|
||||||
|
columns = ['问题', '解析状态', '解析耗时', '执行状态', '执行耗时', '总耗时']
|
||||||
|
# 创建只有表头的 DataFrame
|
||||||
|
self.df = pd.DataFrame(columns=columns)
|
||||||
|
self.file_name = file_name
|
||||||
|
|
||||||
|
def append_data(self, new_data):
|
||||||
|
# 假设 new_data 是一维数组,将其转换为字典
|
||||||
|
columns = ['问题', '解析状态', '解析耗时', '执行状态', '执行耗时', '总耗时']
|
||||||
|
new_dict = dict(zip(columns, new_data))
|
||||||
|
# 使用 loc 方法追加数据
|
||||||
|
self.df.loc[len(self.df)] = new_dict
|
||||||
|
def print_analysis_result(self):
|
||||||
|
# 测试样例总数
|
||||||
|
total_samples = len(self.df)
|
||||||
|
|
||||||
|
# 解析成功数量
|
||||||
|
parse_success_count = (self.df['解析状态'] == '解析成功').sum()
|
||||||
|
|
||||||
|
# 执行成功数量
|
||||||
|
execute_success_count = (self.df['执行状态'] == '执行成功').sum()
|
||||||
|
|
||||||
|
# 解析平均耗时,保留两位小数
|
||||||
|
avg_parse_time = round(self.df['解析耗时'].mean(), 2)
|
||||||
|
|
||||||
|
# 执行平均耗时,保留两位小数
|
||||||
|
avg_execute_time = round(self.df['执行耗时'].mean(), 2)
|
||||||
|
|
||||||
|
# 总平均耗时,保留两位小数
|
||||||
|
avg_total_time = round(self.df['总耗时'].mean(), 2)
|
||||||
|
|
||||||
|
# 最长耗时,保留两位小数
|
||||||
|
max_time = round(self.df['总耗时'].max(), 2)
|
||||||
|
|
||||||
|
# 最短耗时,保留两位小数
|
||||||
|
min_time = round(self.df['总耗时'].min(), 2)
|
||||||
|
|
||||||
|
print(f"测试样例总数 : {total_samples}")
|
||||||
|
print(f"解析成功数量 : {parse_success_count}")
|
||||||
|
print(f"执行成功数量 : {execute_success_count}")
|
||||||
|
print(f"解析平均耗时 : {avg_parse_time} 秒")
|
||||||
|
print(f"执行平均耗时 : {avg_execute_time} 秒")
|
||||||
|
print(f"总平均耗时 : {avg_total_time} 秒")
|
||||||
|
print(f"最长耗时 : {max_time} 秒")
|
||||||
|
print(f"最短耗时 : {min_time} 秒")
|
||||||
|
|
||||||
|
def write_to_csv(self):
|
||||||
|
# 检查 data 文件夹是否存在,如果不存在则创建
|
||||||
|
if not os.path.exists('res'):
|
||||||
|
os.makedirs('res')
|
||||||
|
# 获取当前时间戳
|
||||||
|
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||||
|
# 生成带时间戳的文件名
|
||||||
|
file_path = os.path.join('res', f'{self.file_name}_{timestamp}.csv')
|
||||||
|
self.df.to_csv(file_path, index=False)
|
||||||
|
print(f"测试结果已保存到 {file_path}")
|
||||||
|
|
||||||
class BatchTest:
|
class BatchTest:
|
||||||
def __init__(self, url, agentId, chatId, userName):
|
def __init__(self, url, agentId, chatId, userName):
|
||||||
@@ -70,18 +132,35 @@ class BatchTest:
|
|||||||
def benchmark(url:str, agentId:str, chatId:str, filePath:str, userName:str):
|
def benchmark(url:str, agentId:str, chatId:str, filePath:str, userName:str):
|
||||||
batch_test = BatchTest(url, agentId, chatId, userName)
|
batch_test = BatchTest(url, agentId, chatId, userName)
|
||||||
df = batch_test.read_question_from_csv(filePath)
|
df = batch_test.read_question_from_csv(filePath)
|
||||||
|
appender = DataFrameAppender(os.path.basename(filePath))
|
||||||
for index, row in df.iterrows():
|
for index, row in df.iterrows():
|
||||||
question = row['question']
|
question = row['question']
|
||||||
print('start to ask question:', question)
|
print('start to ask question:', question)
|
||||||
# 捕获异常,防止程序中断
|
# 捕获异常,防止程序中断
|
||||||
try:
|
try:
|
||||||
parse_resp = batch_test.parse(question)
|
parse_resp = batch_test.parse(question)
|
||||||
batch_test.execute(agentId, question, parse_resp['data']['queryId'])
|
parse_status = '解析失败'
|
||||||
|
if parse_resp.get('data').get('errorMsg') is None:
|
||||||
|
parse_status = '解析成功'
|
||||||
|
parse_cost = parse_resp.get('data').get('parseTimeCost').get('parseTime')
|
||||||
|
execute_resp = batch_test.execute(agentId, question, parse_resp['data']['queryId'])
|
||||||
|
execute_status = '执行失败'
|
||||||
|
execute_cost = 0
|
||||||
|
if parse_status == '解析成功' and execute_resp.get('data').get('errorMsg') is None:
|
||||||
|
execute_status = '执行成功'
|
||||||
|
execute_cost = execute_resp.get('data').get('queryTimeCost')
|
||||||
|
res = [question.replace(',', '#'),parse_status,parse_cost/1000,execute_status,execute_cost/1000,(parse_cost+execute_cost)/1000]
|
||||||
|
appender.append_data(res)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print('error:', e)
|
print('error:', e)
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
continue
|
continue
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
# 打印分析结果
|
||||||
|
appender.print_analysis_result()
|
||||||
|
# 分析明细输出
|
||||||
|
appender.write_to_csv()
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
|||||||
@@ -18,4 +18,5 @@ public class ChatExecuteReq {
|
|||||||
private int parseId;
|
private int parseId;
|
||||||
private String queryText;
|
private String queryText;
|
||||||
private boolean saveAnswer;
|
private boolean saveAnswer;
|
||||||
|
private boolean streamingResult;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package com.tencent.supersonic.chat.api.pojo.request;
|
||||||
|
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
public class ChatMemoryDeleteReq {
|
||||||
|
|
||||||
|
private List<Long> ids;
|
||||||
|
|
||||||
|
private Integer agentId;
|
||||||
|
}
|
||||||
@@ -5,5 +5,7 @@ import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
|||||||
|
|
||||||
public interface ChatQueryExecutor {
|
public interface ChatQueryExecutor {
|
||||||
|
|
||||||
|
boolean accept(ExecuteContext executeContext);
|
||||||
|
|
||||||
QueryResult execute(ExecuteContext executeContext);
|
QueryResult execute(ExecuteContext executeContext);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,11 +37,12 @@ public class PlainTextExecutor implements ChatQueryExecutor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResult execute(ExecuteContext executeContext) {
|
public boolean accept(ExecuteContext executeContext) {
|
||||||
if (!"PLAIN_TEXT".equals(executeContext.getParseInfo().getQueryMode())) {
|
return "PLAIN_TEXT".equals(executeContext.getParseInfo().getQueryMode());
|
||||||
return null;
|
}
|
||||||
}
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryResult execute(ExecuteContext executeContext) {
|
||||||
AgentService agentService = ContextUtils.getBean(AgentService.class);
|
AgentService agentService = ContextUtils.getBean(AgentService.class);
|
||||||
Agent chatAgent = agentService.getAgent(executeContext.getAgent().getId());
|
Agent chatAgent = agentService.getAgent(executeContext.getAgent().getId());
|
||||||
ChatApp chatApp = chatAgent.getChatAppConfig().get(APP_KEY);
|
ChatApp chatApp = chatAgent.getChatAppConfig().get(APP_KEY);
|
||||||
|
|||||||
@@ -8,6 +8,11 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
|||||||
|
|
||||||
public class PluginExecutor implements ChatQueryExecutor {
|
public class PluginExecutor implements ChatQueryExecutor {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean accept(ExecuteContext executeContext) {
|
||||||
|
return PluginQueryManager.isPluginQuery(executeContext.getParseInfo().getQueryMode());
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResult execute(ExecuteContext executeContext) {
|
public QueryResult execute(ExecuteContext executeContext) {
|
||||||
SemanticParseInfo parseInfo = executeContext.getParseInfo();
|
SemanticParseInfo parseInfo = executeContext.getParseInfo();
|
||||||
|
|||||||
@@ -25,6 +25,11 @@ import java.util.Objects;
|
|||||||
|
|
||||||
public class SqlExecutor implements ChatQueryExecutor {
|
public class SqlExecutor implements ChatQueryExecutor {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean accept(ExecuteContext executeContext) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
@Override
|
@Override
|
||||||
public QueryResult execute(ExecuteContext executeContext) {
|
public QueryResult execute(ExecuteContext executeContext) {
|
||||||
@@ -70,8 +75,12 @@ public class SqlExecutor implements ChatQueryExecutor {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
QuerySqlReq sqlReq =
|
// 使用querySQL,它已经包含了所有修正(包括物理SQL修正)
|
||||||
QuerySqlReq.builder().sql(parseInfo.getSqlInfo().getCorrectedS2SQL()).build();
|
String finalSql = StringUtils.isNotBlank(parseInfo.getSqlInfo().getQuerySQL())
|
||||||
|
? parseInfo.getSqlInfo().getQuerySQL()
|
||||||
|
: parseInfo.getSqlInfo().getCorrectedS2SQL();
|
||||||
|
|
||||||
|
QuerySqlReq sqlReq = QuerySqlReq.builder().sql(finalSql).build();
|
||||||
sqlReq.setSqlInfo(parseInfo.getSqlInfo());
|
sqlReq.setSqlInfo(parseInfo.getSqlInfo());
|
||||||
sqlReq.setDataSetId(parseInfo.getDataSetId());
|
sqlReq.setDataSetId(parseInfo.getDataSetId());
|
||||||
|
|
||||||
@@ -80,12 +89,12 @@ public class SqlExecutor implements ChatQueryExecutor {
|
|||||||
queryResult.setQueryId(executeContext.getRequest().getQueryId());
|
queryResult.setQueryId(executeContext.getRequest().getQueryId());
|
||||||
queryResult.setChatContext(parseInfo);
|
queryResult.setChatContext(parseInfo);
|
||||||
queryResult.setQueryMode(parseInfo.getQueryMode());
|
queryResult.setQueryMode(parseInfo.getQueryMode());
|
||||||
queryResult.setQueryTimeCost(System.currentTimeMillis() - startTime);
|
|
||||||
SemanticQueryResp queryResp =
|
SemanticQueryResp queryResp =
|
||||||
semanticLayer.queryByReq(sqlReq, executeContext.getRequest().getUser());
|
semanticLayer.queryByReq(sqlReq, executeContext.getRequest().getUser());
|
||||||
|
queryResult.setQueryTimeCost(System.currentTimeMillis() - startTime);
|
||||||
if (queryResp != null) {
|
if (queryResp != null) {
|
||||||
queryResult.setQueryAuthorization(queryResp.getQueryAuthorization());
|
queryResult.setQueryAuthorization(queryResp.getQueryAuthorization());
|
||||||
queryResult.setQuerySql(queryResp.getSql());
|
queryResult.setQuerySql(finalSql);
|
||||||
queryResult.setQueryResults(queryResp.getResultList());
|
queryResult.setQueryResults(queryResp.getResultList());
|
||||||
queryResult.setQueryColumns(queryResp.getColumns());
|
queryResult.setQueryColumns(queryResp.getColumns());
|
||||||
queryResult.setQueryState(QueryState.SUCCESS);
|
queryResult.setQueryState(QueryState.SUCCESS);
|
||||||
|
|||||||
@@ -4,5 +4,7 @@ import com.tencent.supersonic.chat.server.pojo.ParseContext;
|
|||||||
|
|
||||||
public interface ChatQueryParser {
|
public interface ChatQueryParser {
|
||||||
|
|
||||||
|
boolean accept(ParseContext parseContext);
|
||||||
|
|
||||||
void parse(ParseContext parseContext);
|
void parse(ParseContext parseContext);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,12 +14,12 @@ public class NL2PluginParser implements ChatQueryParser {
|
|||||||
private final List<PluginRecognizer> pluginRecognizers =
|
private final List<PluginRecognizer> pluginRecognizers =
|
||||||
ComponentFactory.getPluginRecognizers();
|
ComponentFactory.getPluginRecognizers();
|
||||||
|
|
||||||
|
public boolean accept(ParseContext parseContext) {
|
||||||
|
return parseContext.getAgent().containsPluginTool();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void parse(ParseContext parseContext) {
|
public void parse(ParseContext parseContext) {
|
||||||
if (!parseContext.getAgent().containsPluginTool()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
pluginRecognizers.forEach(pluginRecognizer -> {
|
pluginRecognizers.forEach(pluginRecognizer -> {
|
||||||
pluginRecognizer.recognize(parseContext);
|
pluginRecognizer.recognize(parseContext);
|
||||||
log.info("{} recallResult:{}", pluginRecognizer.getClass().getSimpleName(),
|
log.info("{} recallResult:{}", pluginRecognizer.getClass().getSimpleName(),
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ import dev.langchain4j.model.input.PromptTemplate;
|
|||||||
import dev.langchain4j.model.output.Response;
|
import dev.langchain4j.model.output.Response;
|
||||||
import dev.langchain4j.provider.ModelProvider;
|
import dev.langchain4j.provider.ModelProvider;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
@@ -73,12 +74,12 @@ public class NL2SQLParser implements ChatQueryParser {
|
|||||||
.build());
|
.build());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean accept(ParseContext parseContext) {
|
||||||
|
return parseContext.enableNL2SQL();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void parse(ParseContext parseContext) {
|
public void parse(ParseContext parseContext) {
|
||||||
if (!parseContext.enableNL2SQL()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// first go with rule-based parsers unless the user has already selected one parse.
|
// first go with rule-based parsers unless the user has already selected one parse.
|
||||||
if (Objects.isNull(parseContext.getRequest().getSelectedParse())) {
|
if (Objects.isNull(parseContext.getRequest().getSelectedParse())) {
|
||||||
QueryNLReq queryNLReq = QueryReqConverter.buildQueryNLReq(parseContext);
|
QueryNLReq queryNLReq = QueryReqConverter.buildQueryNLReq(parseContext);
|
||||||
@@ -171,10 +172,6 @@ public class NL2SQLParser implements ChatQueryParser {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// derive mapping result of current question and parsing result of last question.
|
|
||||||
ChatLayerService chatLayerService = ContextUtils.getBean(ChatLayerService.class);
|
|
||||||
MapResp currentMapResult = chatLayerService.map(queryNLReq);
|
|
||||||
|
|
||||||
List<QueryResp> historyQueries =
|
List<QueryResp> historyQueries =
|
||||||
getHistoryQueries(parseContext.getRequest().getChatId(), 1);
|
getHistoryQueries(parseContext.getRequest().getChatId(), 1);
|
||||||
if (historyQueries.isEmpty()) {
|
if (historyQueries.isEmpty()) {
|
||||||
@@ -182,12 +179,18 @@ public class NL2SQLParser implements ChatQueryParser {
|
|||||||
}
|
}
|
||||||
QueryResp lastQuery = historyQueries.get(0);
|
QueryResp lastQuery = historyQueries.get(0);
|
||||||
SemanticParseInfo lastParseInfo = lastQuery.getParseInfos().get(0);
|
SemanticParseInfo lastParseInfo = lastQuery.getParseInfos().get(0);
|
||||||
Long dataId = lastParseInfo.getDataSetId();
|
String histSQL = lastParseInfo.getSqlInfo().getCorrectedS2SQL();
|
||||||
|
if (StringUtils.isBlank(histSQL)) // 优化性能,如果问答不是chat bi 则无需重写,因为数据都不全
|
||||||
|
return;
|
||||||
|
|
||||||
|
// derive mapping result of current question and parsing result of last question.
|
||||||
|
ChatLayerService chatLayerService = ContextUtils.getBean(ChatLayerService.class);
|
||||||
|
MapResp currentMapResult = chatLayerService.map(queryNLReq); // 优化性能 ,只有满足条件才mapping
|
||||||
|
|
||||||
|
Long dataId = lastParseInfo.getDataSetId();
|
||||||
String curtMapStr =
|
String curtMapStr =
|
||||||
generateSchemaPrompt(currentMapResult.getMapInfo().getMatchedElements(dataId));
|
generateSchemaPrompt(currentMapResult.getMapInfo().getMatchedElements(dataId));
|
||||||
String histMapStr = generateSchemaPrompt(lastParseInfo.getElementMatches());
|
String histMapStr = generateSchemaPrompt(lastParseInfo.getElementMatches());
|
||||||
String histSQL = lastParseInfo.getSqlInfo().getCorrectedS2SQL();
|
|
||||||
|
|
||||||
Map<String, Object> variables = new HashMap<>();
|
Map<String, Object> variables = new HashMap<>();
|
||||||
variables.put("current_question", currentMapResult.getQueryText());
|
variables.put("current_question", currentMapResult.getQueryText());
|
||||||
|
|||||||
@@ -6,12 +6,12 @@ import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
|||||||
|
|
||||||
public class PlainTextParser implements ChatQueryParser {
|
public class PlainTextParser implements ChatQueryParser {
|
||||||
|
|
||||||
|
public boolean accept(ParseContext parseContext) {
|
||||||
|
return !parseContext.getAgent().containsAnyTool();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void parse(ParseContext parseContext) {
|
public void parse(ParseContext parseContext) {
|
||||||
if (parseContext.getAgent().containsAnyTool()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
SemanticParseInfo parseInfo = new SemanticParseInfo();
|
SemanticParseInfo parseInfo = new SemanticParseInfo();
|
||||||
parseInfo.setQueryMode("PLAIN_TEXT");
|
parseInfo.setQueryMode("PLAIN_TEXT");
|
||||||
parseInfo.setId(1);
|
parseInfo.setId(1);
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
package com.tencent.supersonic.chat.server.persistence.dataobject;
|
package com.tencent.supersonic.chat.server.persistence.dataobject;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
|
@TableName("s2_chat")
|
||||||
public class ChatDO {
|
public class ChatDO {
|
||||||
|
|
||||||
private long chatId;
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long chatId;
|
||||||
private Integer agentId;
|
private Integer agentId;
|
||||||
private String chatName;
|
private String chatName;
|
||||||
private String createTime;
|
private String createTime;
|
||||||
|
|||||||
@@ -1,20 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.server.persistence.dataobject;
|
|
||||||
|
|
||||||
import lombok.Data;
|
|
||||||
|
|
||||||
import java.util.Date;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
public class DictConfDO {
|
|
||||||
|
|
||||||
private Long id;
|
|
||||||
|
|
||||||
private Long modelId;
|
|
||||||
|
|
||||||
private String dimValueInfos;
|
|
||||||
|
|
||||||
private String createdBy;
|
|
||||||
private String updatedBy;
|
|
||||||
private Date createdAt;
|
|
||||||
private Date updatedAt;
|
|
||||||
}
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.server.persistence.dataobject;
|
|
||||||
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.ToString;
|
|
||||||
import org.apache.commons.codec.digest.DigestUtils;
|
|
||||||
|
|
||||||
import java.util.Date;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
@ToString
|
|
||||||
public class DictTaskDO {
|
|
||||||
|
|
||||||
private Long id;
|
|
||||||
|
|
||||||
private String name;
|
|
||||||
|
|
||||||
private String description;
|
|
||||||
|
|
||||||
private String command;
|
|
||||||
|
|
||||||
private String commandMd5;
|
|
||||||
|
|
||||||
private String dimIds;
|
|
||||||
|
|
||||||
private Integer status;
|
|
||||||
|
|
||||||
private String createdBy;
|
|
||||||
|
|
||||||
private Date createdAt;
|
|
||||||
|
|
||||||
private Double progress;
|
|
||||||
|
|
||||||
private Long elapsedMs;
|
|
||||||
|
|
||||||
public String getCommandMd5() {
|
|
||||||
return DigestUtils.md5Hex(command);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -35,9 +35,7 @@ public class ChatMemoryRepositoryImpl implements ChatMemoryRepository {
|
|||||||
if (CollectionUtils.isEmpty(ids)) {
|
if (CollectionUtils.isEmpty(ids)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
for (Long id : ids) {
|
chatMemoryMapper.deleteByIds(ids);
|
||||||
chatMemoryMapper.deleteById(id);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
|
|||||||
chatQueryDO.setUserName(chatParseReq.getUser().getName());
|
chatQueryDO.setUserName(chatParseReq.getUser().getName());
|
||||||
chatQueryDO.setQueryText(chatParseReq.getQueryText());
|
chatQueryDO.setQueryText(chatParseReq.getQueryText());
|
||||||
chatQueryDO.setAgentId(chatParseReq.getAgentId());
|
chatQueryDO.setAgentId(chatParseReq.getAgentId());
|
||||||
chatQueryDO.setQueryResult("");
|
chatQueryDO.setQueryResult("{}");
|
||||||
chatQueryDO.setQueryState(1);
|
chatQueryDO.setQueryState(1);
|
||||||
try {
|
try {
|
||||||
chatQueryDOMapper.insert(chatQueryDO);
|
chatQueryDOMapper.insert(chatQueryDO);
|
||||||
|
|||||||
@@ -88,10 +88,10 @@ public class WebServiceQuery extends PluginSemanticQuery {
|
|||||||
restTemplate = ContextUtils.getBean(RestTemplate.class);
|
restTemplate = ContextUtils.getBean(RestTemplate.class);
|
||||||
try {
|
try {
|
||||||
responseEntity =
|
responseEntity =
|
||||||
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, Object.class);
|
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, String.class);
|
||||||
objectResponse = responseEntity.getBody();
|
objectResponse = responseEntity.getBody();
|
||||||
log.info("objectResponse:{}", objectResponse);
|
log.info("objectResponse:{}", objectResponse);
|
||||||
Map<String, Object> response = JsonUtil.objectToMap(objectResponse);
|
Map<String, Object> response = JSON.parseObject(objectResponse.toString());
|
||||||
webServiceResponse.setResult(response);
|
webServiceResponse.setResult(response);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.info("Exception:{}", e.getMessage());
|
log.info("Exception:{}", e.getMessage());
|
||||||
|
|||||||
@@ -19,7 +19,8 @@ public class ParseContext {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public boolean enableNL2SQL() {
|
public boolean enableNL2SQL() {
|
||||||
return Objects.nonNull(agent) && agent.containsDatasetTool();
|
return Objects.nonNull(agent) && agent.containsDatasetTool()
|
||||||
|
&& response.getSelectedParses().size() == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean enableLLM() {
|
public boolean enableLLM() {
|
||||||
|
|||||||
@@ -1,13 +1,20 @@
|
|||||||
package com.tencent.supersonic.chat.server.processor.execute;
|
package com.tencent.supersonic.chat.server.processor.execute;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSON;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||||
|
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
|
||||||
|
import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryRepository;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
||||||
import com.tencent.supersonic.common.pojo.ChatApp;
|
import com.tencent.supersonic.common.pojo.ChatApp;
|
||||||
import com.tencent.supersonic.common.pojo.enums.AppModule;
|
import com.tencent.supersonic.common.pojo.enums.AppModule;
|
||||||
import com.tencent.supersonic.common.util.ChatAppManager;
|
import com.tencent.supersonic.common.util.ChatAppManager;
|
||||||
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import dev.langchain4j.data.message.AiMessage;
|
import dev.langchain4j.data.message.AiMessage;
|
||||||
|
import dev.langchain4j.model.StreamingResponseHandler;
|
||||||
import dev.langchain4j.model.chat.ChatLanguageModel;
|
import dev.langchain4j.model.chat.ChatLanguageModel;
|
||||||
|
import dev.langchain4j.model.chat.StreamingChatLanguageModel;
|
||||||
import dev.langchain4j.model.input.Prompt;
|
import dev.langchain4j.model.input.Prompt;
|
||||||
import dev.langchain4j.model.input.PromptTemplate;
|
import dev.langchain4j.model.input.PromptTemplate;
|
||||||
import dev.langchain4j.model.output.Response;
|
import dev.langchain4j.model.output.Response;
|
||||||
@@ -24,9 +31,11 @@ import java.util.Objects;
|
|||||||
* DataInterpretProcessor interprets query result to make it more readable to the users.
|
* DataInterpretProcessor interprets query result to make it more readable to the users.
|
||||||
*/
|
*/
|
||||||
public class DataInterpretProcessor implements ExecuteResultProcessor {
|
public class DataInterpretProcessor implements ExecuteResultProcessor {
|
||||||
|
public static String tip = "AI 回答中...\r\n";
|
||||||
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
|
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
|
||||||
|
|
||||||
|
private static Map<Long, StringBuffer> resultCache = new HashMap<>();
|
||||||
|
|
||||||
public static final String APP_KEY = "DATA_INTERPRETER";
|
public static final String APP_KEY = "DATA_INTERPRETER";
|
||||||
private static final String INSTRUCTION = ""
|
private static final String INSTRUCTION = ""
|
||||||
+ "#Role: You are a data expert who communicates with business users everyday."
|
+ "#Role: You are a data expert who communicates with business users everyday."
|
||||||
@@ -41,12 +50,24 @@ public class DataInterpretProcessor implements ExecuteResultProcessor {
|
|||||||
.appModule(AppModule.CHAT).description("通过大模型对结果数据做提炼总结").enable(false).build());
|
.appModule(AppModule.CHAT).description("通过大模型对结果数据做提炼总结").enable(false).build());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String getTextSummary(Long queryId) {
|
||||||
|
if (resultCache.get(queryId) != null) {
|
||||||
|
return resultCache.get(queryId).toString();
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Map<Long, StringBuffer> getResultCache() {
|
||||||
|
return resultCache;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean accept(ExecuteContext executeContext) {
|
public boolean accept(ExecuteContext executeContext) {
|
||||||
Agent agent = executeContext.getAgent();
|
Agent agent = executeContext.getAgent();
|
||||||
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
|
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
|
||||||
return Objects.nonNull(chatApp) && chatApp.isEnable();
|
return Objects.nonNull(chatApp) && chatApp.isEnable()
|
||||||
|
&& StringUtils.isNotBlank(executeContext.getResponse().getTextResult()) // 如果都没结果,则无法处理
|
||||||
|
&& StringUtils.isBlank(executeContext.getResponse().getTextSummary()); // 如果已经有汇总的结果了,无法再次处理
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -56,18 +77,62 @@ public class DataInterpretProcessor implements ExecuteResultProcessor {
|
|||||||
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
|
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
|
||||||
|
|
||||||
Map<String, Object> variable = new HashMap<>();
|
Map<String, Object> variable = new HashMap<>();
|
||||||
variable.put("question", executeContext.getRequest().getQueryText());
|
String question = executeContext.getResponse().getTextResult();// 结果解析应该用改写的问题,因为改写的内容信息量更大
|
||||||
|
if (executeContext.getParseInfo().getProperties() != null
|
||||||
|
&& executeContext.getParseInfo().getProperties().containsKey("CONTEXT")) {
|
||||||
|
Map<String, Object> context = (Map<String, Object>) executeContext.getParseInfo()
|
||||||
|
.getProperties().get("CONTEXT");
|
||||||
|
if (context.get("queryText") != null && "".equals(context.get("queryText"))) {
|
||||||
|
question = context.get("queryText").toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
variable.put("question", question);
|
||||||
variable.put("data", queryResult.getTextResult());
|
variable.put("data", queryResult.getTextResult());
|
||||||
|
|
||||||
Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variable);
|
Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variable);
|
||||||
ChatLanguageModel chatLanguageModel =
|
if (executeContext.getRequest().isStreamingResult()) {
|
||||||
ModelProvider.getChatModel(chatApp.getChatModelConfig());
|
StreamingChatLanguageModel chatLanguageModel =
|
||||||
Response<AiMessage> response = chatLanguageModel.generate(prompt.toUserMessage());
|
ModelProvider.getChatStreamingModel(chatApp.getChatModelConfig());
|
||||||
String anwser = response.content().text();
|
final Long queryId = executeContext.getRequest().getQueryId();
|
||||||
keyPipelineLog.info("DataInterpretProcessor modelReq:\n{} \nmodelResp:\n{}", prompt.text(),
|
resultCache.put(queryId, new StringBuffer(tip));
|
||||||
anwser);
|
chatLanguageModel.generate(prompt.toUserMessage(),
|
||||||
if (StringUtils.isNotBlank(anwser)) {
|
new StreamingResponseHandler<AiMessage>() {
|
||||||
queryResult.setTextSummary(anwser);
|
@Override
|
||||||
|
public void onNext(String token) {
|
||||||
|
resultCache.get(queryId).append(token);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onComplete(Response<AiMessage> response) {
|
||||||
|
ChatQueryRepository chatQueryRepository =
|
||||||
|
ContextUtils.getBean(ChatQueryRepository.class);
|
||||||
|
ChatQueryDO chatQueryDO = chatQueryRepository.getChatQueryDO(queryId);
|
||||||
|
JSONObject queryResult = JSON.parseObject(chatQueryDO.getQueryResult());
|
||||||
|
queryResult.put("textSummary",
|
||||||
|
resultCache.get(queryId).toString().substring(tip.length()));
|
||||||
|
chatQueryDO.setQueryResult(queryResult.toJSONString());
|
||||||
|
chatQueryRepository.updateChatQuery(chatQueryDO);
|
||||||
|
resultCache.remove(queryId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onError(Throwable error) {
|
||||||
|
error.printStackTrace();
|
||||||
|
resultCache.remove(queryId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
ChatLanguageModel chatLanguageModel =
|
||||||
|
ModelProvider.getChatModel(chatApp.getChatModelConfig());
|
||||||
|
Response<AiMessage> response = chatLanguageModel.generate(prompt.toUserMessage());
|
||||||
|
String anwser = response.content().text();
|
||||||
|
keyPipelineLog.info("DataInterpretProcessor modelReq:\n{} \nmodelResp:\n{}",
|
||||||
|
prompt.text(), anwser);
|
||||||
|
if (StringUtils.isNotBlank(anwser)) {
|
||||||
|
queryResult.setTextSummary(anwser);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -66,8 +66,10 @@ public class ChatConfigController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/getDomainDataSetTree")
|
@GetMapping("/getDomainDataSetTree")
|
||||||
public List<ItemResp> getDomainDataSetTree() {
|
public List<ItemResp> getDomainDataSetTree(HttpServletRequest request,
|
||||||
return semanticLayerService.getDomainDataSetTree();
|
HttpServletResponse response) {
|
||||||
|
User user = UserHolder.findUser(request, response);
|
||||||
|
return semanticLayerService.getDomainDataSetTree(user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/getDataSetSchema/{id}")
|
@GetMapping("/getDataSetSchema/{id}")
|
||||||
|
|||||||
@@ -22,11 +22,10 @@ public class ChatController {
|
|||||||
private ChatManageService chatService;
|
private ChatManageService chatService;
|
||||||
|
|
||||||
@PostMapping("/save")
|
@PostMapping("/save")
|
||||||
public Boolean save(@RequestParam(value = "chatName") String chatName,
|
public Long save(@RequestParam(value = "chatName") String chatName,
|
||||||
@RequestParam(value = "agentId", required = false) Integer agentId,
|
@RequestParam(value = "agentId", required = false) Integer agentId,
|
||||||
HttpServletRequest request, HttpServletResponse response) {
|
HttpServletRequest request, HttpServletResponse response) {
|
||||||
chatService.addChat(UserHolder.findUser(request, response), chatName, agentId);
|
return chatService.addChat(UserHolder.findUser(request, response), chatName, agentId);
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/getAll")
|
@GetMapping("/getAll")
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
|||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatQueryDataReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.ChatParseResp;
|
import com.tencent.supersonic.chat.api.pojo.response.ChatParseResp;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||||
import com.tencent.supersonic.chat.server.service.ChatQueryService;
|
import com.tencent.supersonic.chat.server.service.ChatQueryService;
|
||||||
import com.tencent.supersonic.common.pojo.User;
|
import com.tencent.supersonic.common.pojo.User;
|
||||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||||
@@ -50,6 +51,14 @@ public class ChatQueryController {
|
|||||||
return chatQueryService.execute(chatExecuteReq);
|
return chatQueryService.execute(chatExecuteReq);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@PostMapping("getExecuteSummary")
|
||||||
|
public Object getExecuteSummary(@RequestBody ChatExecuteReq chatExecuteReq,
|
||||||
|
HttpServletRequest request, HttpServletResponse response) {
|
||||||
|
chatExecuteReq.setUser(UserHolder.findUser(request, response));
|
||||||
|
QueryResult res = chatQueryService.getTextSummary(chatExecuteReq);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
@PostMapping("/")
|
@PostMapping("/")
|
||||||
public Object query(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request,
|
public Object query(@RequestBody ChatParseReq chatParseReq, HttpServletRequest request,
|
||||||
HttpServletResponse response) throws Exception {
|
HttpServletResponse response) throws Exception {
|
||||||
|
|||||||
@@ -4,12 +4,12 @@ import com.github.pagehelper.PageInfo;
|
|||||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryCreateReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryCreateReq;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryDeleteReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
||||||
import com.tencent.supersonic.chat.server.service.MemoryService;
|
import com.tencent.supersonic.chat.server.service.MemoryService;
|
||||||
import com.tencent.supersonic.common.pojo.User;
|
import com.tencent.supersonic.common.pojo.User;
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import jakarta.servlet.http.HttpServletResponse;
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
@@ -53,8 +53,10 @@ public class MemoryController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("batchDelete")
|
@PostMapping("batchDelete")
|
||||||
public Boolean batchDelete(@RequestBody MetaBatchReq metaBatchReq) {
|
public Boolean deleteMemory(@RequestBody ChatMemoryDeleteReq chatMemoryDeleteReq,
|
||||||
memoryService.batchDelete(metaBatchReq.getIds());
|
HttpServletRequest request, HttpServletResponse response) {
|
||||||
|
User user = UserHolder.findUser(request, response);
|
||||||
|
memoryService.batchDelete(chatMemoryDeleteReq, user);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,6 +35,8 @@ public interface ChatManageService {
|
|||||||
|
|
||||||
QueryResp getChatQuery(Long queryId);
|
QueryResp getChatQuery(Long queryId);
|
||||||
|
|
||||||
|
ChatQueryDO getChatQueryDO(Long queryId);
|
||||||
|
|
||||||
List<QueryResp> getChatQueries(Integer chatId);
|
List<QueryResp> getChatQueries(Integer chatId);
|
||||||
|
|
||||||
ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId);
|
ShowCaseResp queryShowCase(PageQueryInfoReq pageQueryInfoReq, int agentId);
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ public interface ChatQueryService {
|
|||||||
|
|
||||||
QueryResult execute(ChatExecuteReq chatExecuteReq) throws Exception;
|
QueryResult execute(ChatExecuteReq chatExecuteReq) throws Exception;
|
||||||
|
|
||||||
|
QueryResult getTextSummary(ChatExecuteReq chatExecuteReq);
|
||||||
|
|
||||||
QueryResult parseAndExecute(ChatParseReq chatParseReq);
|
QueryResult parseAndExecute(ChatParseReq chatParseReq);
|
||||||
|
|
||||||
Object queryData(ChatQueryDataReq chatQueryDataReq, User user) throws Exception;
|
Object queryData(ChatQueryDataReq chatQueryDataReq, User user) throws Exception;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.server.service;
|
package com.tencent.supersonic.chat.server.service;
|
||||||
|
|
||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryDeleteReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
||||||
@@ -14,7 +15,7 @@ public interface MemoryService {
|
|||||||
|
|
||||||
void updateMemory(ChatMemoryUpdateReq chatMemoryUpdateReq, User user);
|
void updateMemory(ChatMemoryUpdateReq chatMemoryUpdateReq, User user);
|
||||||
|
|
||||||
void batchDelete(List<Long> ids);
|
void batchDelete(ChatMemoryDeleteReq chatMemoryDeleteReq, User user);
|
||||||
|
|
||||||
PageInfo<ChatMemory> pageMemories(PageMemoryReq pageMemoryReq);
|
PageInfo<ChatMemory> pageMemories(PageMemoryReq pageMemoryReq);
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.beans.factory.annotation.Qualifier;
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
|
import org.springframework.context.annotation.Lazy;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@@ -39,6 +40,7 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
|
|||||||
private MemoryService memoryService;
|
private MemoryService memoryService;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
|
@Lazy
|
||||||
private ChatQueryService chatQueryService;
|
private ChatQueryService chatQueryService;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
@@ -161,9 +163,11 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
|
|||||||
JsonUtil.toMap(agentDO.getChatModelConfig(), String.class, ChatApp.class));
|
JsonUtil.toMap(agentDO.getChatModelConfig(), String.class, ChatApp.class));
|
||||||
agent.setVisualConfig(JsonUtil.toObject(agentDO.getVisualConfig(), VisualConfig.class));
|
agent.setVisualConfig(JsonUtil.toObject(agentDO.getVisualConfig(), VisualConfig.class));
|
||||||
agent.getChatAppConfig().values().forEach(c -> {
|
agent.getChatAppConfig().values().forEach(c -> {
|
||||||
ChatModel chatModel = chatModelService.getChatModel(c.getChatModelId());
|
if (c.isEnable()) {// 优化,减少访问数据库的次数
|
||||||
if (Objects.nonNull(chatModel)) {
|
ChatModel chatModel = chatModelService.getChatModel(c.getChatModelId());
|
||||||
c.setChatModelConfig(chatModelService.getChatModel(c.getChatModelId()).getConfig());
|
if (Objects.nonNull(chatModel)) {
|
||||||
|
c.setChatModelConfig(chatModel.getConfig());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
agent.setAdmins(JsonUtil.toList(agentDO.getAdmin(), String.class));
|
agent.setAdmins(JsonUtil.toList(agentDO.getAdmin(), String.class));
|
||||||
|
|||||||
@@ -123,6 +123,11 @@ public class ChatManageServiceImpl implements ChatManageService {
|
|||||||
return chatQueryRepository.getChatQuery(queryId);
|
return chatQueryRepository.getChatQuery(queryId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ChatQueryDO getChatQueryDO(Long queryId) {
|
||||||
|
return chatQueryRepository.getChatQueryDO(queryId);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<QueryResp> getChatQueries(Integer chatId) {
|
public List<QueryResp> getChatQueries(Integer chatId) {
|
||||||
List<QueryResp> queries = chatQueryRepository.getChatQueries(chatId);
|
List<QueryResp> queries = chatQueryRepository.getChatQueries(chatId);
|
||||||
@@ -233,6 +238,10 @@ public class ChatManageServiceImpl implements ChatManageService {
|
|||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(Long questionId, int parseId) {
|
public SemanticParseInfo getParseInfo(Long questionId, int parseId) {
|
||||||
ChatParseDO chatParseDO = chatQueryRepository.getParseInfo(questionId, parseId);
|
ChatParseDO chatParseDO = chatQueryRepository.getParseInfo(questionId, parseId);
|
||||||
return JSONObject.parseObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
|
if (chatParseDO == null) {
|
||||||
|
return null;
|
||||||
|
} else {
|
||||||
|
return JSONObject.parseObject(chatParseDO.getParseInfo(), SemanticParseInfo.class);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.server.service.impl;
|
package com.tencent.supersonic.chat.server.service.impl;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
||||||
@@ -9,8 +11,10 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
|||||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||||
import com.tencent.supersonic.chat.server.executor.ChatQueryExecutor;
|
import com.tencent.supersonic.chat.server.executor.ChatQueryExecutor;
|
||||||
import com.tencent.supersonic.chat.server.parser.ChatQueryParser;
|
import com.tencent.supersonic.chat.server.parser.ChatQueryParser;
|
||||||
|
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
||||||
import com.tencent.supersonic.chat.server.pojo.ParseContext;
|
import com.tencent.supersonic.chat.server.pojo.ParseContext;
|
||||||
|
import com.tencent.supersonic.chat.server.processor.execute.DataInterpretProcessor;
|
||||||
import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor;
|
import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor;
|
||||||
import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor;
|
import com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor;
|
||||||
import com.tencent.supersonic.chat.server.service.AgentService;
|
import com.tencent.supersonic.chat.server.service.AgentService;
|
||||||
@@ -49,6 +53,7 @@ import net.sf.jsqlparser.schema.Column;
|
|||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.commons.lang3.tuple.Pair;
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.context.annotation.Lazy;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@@ -66,6 +71,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private SemanticLayerService semanticLayerService;
|
private SemanticLayerService semanticLayerService;
|
||||||
@Autowired
|
@Autowired
|
||||||
|
@Lazy
|
||||||
private AgentService agentService;
|
private AgentService agentService;
|
||||||
|
|
||||||
private final List<ChatQueryParser> chatQueryParsers = ComponentFactory.getChatParsers();
|
private final List<ChatQueryParser> chatQueryParsers = ComponentFactory.getChatParsers();
|
||||||
@@ -95,7 +101,11 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ParseContext parseContext = buildParseContext(chatParseReq, new ChatParseResp(queryId));
|
ParseContext parseContext = buildParseContext(chatParseReq, new ChatParseResp(queryId));
|
||||||
chatQueryParsers.forEach(p -> p.parse(parseContext));
|
for (ChatQueryParser parser : chatQueryParsers) {
|
||||||
|
if (parser.accept(parseContext)) {
|
||||||
|
parser.parse(parseContext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (ParseResultProcessor processor : parseResultProcessors) {
|
for (ParseResultProcessor processor : parseResultProcessors) {
|
||||||
if (processor.accept(parseContext)) {
|
if (processor.accept(parseContext)) {
|
||||||
@@ -116,9 +126,11 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
QueryResult queryResult = new QueryResult();
|
QueryResult queryResult = new QueryResult();
|
||||||
ExecuteContext executeContext = buildExecuteContext(chatExecuteReq);
|
ExecuteContext executeContext = buildExecuteContext(chatExecuteReq);
|
||||||
for (ChatQueryExecutor chatQueryExecutor : chatQueryExecutors) {
|
for (ChatQueryExecutor chatQueryExecutor : chatQueryExecutors) {
|
||||||
queryResult = chatQueryExecutor.execute(executeContext);
|
if (chatQueryExecutor.accept(executeContext)) {
|
||||||
if (queryResult != null) {
|
queryResult = chatQueryExecutor.execute(executeContext);
|
||||||
break;
|
if (queryResult != null) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,6 +147,21 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
|||||||
return queryResult;
|
return queryResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryResult getTextSummary(ChatExecuteReq chatExecuteReq) {
|
||||||
|
String text = DataInterpretProcessor.getTextSummary(chatExecuteReq.getQueryId());
|
||||||
|
if (StringUtils.isNotBlank(text)) {
|
||||||
|
QueryResult res = new QueryResult();
|
||||||
|
res.setTextSummary(text);
|
||||||
|
res.setQueryId(chatExecuteReq.getQueryId());
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
ChatQueryDO chatQueryDo = chatManageService.getChatQueryDO(chatExecuteReq.getQueryId());
|
||||||
|
QueryResult res = JSON.parseObject(chatQueryDo.getQueryResult(), QueryResult.class);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResult parseAndExecute(ChatParseReq chatParseReq) {
|
public QueryResult parseAndExecute(ChatParseReq chatParseReq) {
|
||||||
ChatParseResp parseResp = parse(chatParseReq);
|
ChatParseResp parseResp = parse(chatParseReq);
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.github.pagehelper.PageHelper;
|
|||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryDeleteReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
||||||
@@ -18,19 +19,23 @@ import com.tencent.supersonic.common.config.EmbeddingConfig;
|
|||||||
import com.tencent.supersonic.common.pojo.Text2SQLExemplar;
|
import com.tencent.supersonic.common.pojo.Text2SQLExemplar;
|
||||||
import com.tencent.supersonic.common.pojo.User;
|
import com.tencent.supersonic.common.pojo.User;
|
||||||
import com.tencent.supersonic.common.service.ExemplarService;
|
import com.tencent.supersonic.common.service.ExemplarService;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.boot.CommandLineRunner;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class MemoryServiceImpl implements MemoryService {
|
@Slf4j
|
||||||
|
public class MemoryServiceImpl implements MemoryService, CommandLineRunner {
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private ChatMemoryRepository chatMemoryRepository;
|
private ChatMemoryRepository chatMemoryRepository;
|
||||||
@@ -61,12 +66,17 @@ public class MemoryServiceImpl implements MemoryService {
|
|||||||
ChatMemoryDO chatMemoryDO = chatMemoryRepository.getMemory(chatMemoryUpdateReq.getId());
|
ChatMemoryDO chatMemoryDO = chatMemoryRepository.getMemory(chatMemoryUpdateReq.getId());
|
||||||
boolean hadEnabled =
|
boolean hadEnabled =
|
||||||
MemoryStatus.ENABLED.toString().equals(chatMemoryDO.getStatus().trim());
|
MemoryStatus.ENABLED.toString().equals(chatMemoryDO.getStatus().trim());
|
||||||
if (MemoryStatus.ENABLED.equals(chatMemoryUpdateReq.getStatus()) && !hadEnabled) {
|
|
||||||
|
if (MemoryStatus.ENABLED.equals(chatMemoryUpdateReq.getStatus())) {
|
||||||
|
// Update the latest SQL/Schema to vector DB once memory is enabled
|
||||||
|
chatMemoryDO.setS2sql(chatMemoryUpdateReq.getS2sql());
|
||||||
|
chatMemoryDO.setDbSchema(chatMemoryUpdateReq.getDbSchema());
|
||||||
enableMemory(chatMemoryDO);
|
enableMemory(chatMemoryDO);
|
||||||
} else if (MemoryStatus.DISABLED.equals(chatMemoryUpdateReq.getStatus()) && hadEnabled) {
|
} else if ((MemoryStatus.DISABLED.equals(chatMemoryUpdateReq.getStatus())
|
||||||
|
|| MemoryStatus.PENDING.equals(chatMemoryUpdateReq.getStatus())) && hadEnabled) {
|
||||||
|
// Remove from vector DB when transitioning: launched→disabled OR enabled→pending
|
||||||
disableMemory(chatMemoryDO);
|
disableMemory(chatMemoryDO);
|
||||||
}
|
}
|
||||||
|
|
||||||
LambdaUpdateWrapper<ChatMemoryDO> updateWrapper = new LambdaUpdateWrapper<>();
|
LambdaUpdateWrapper<ChatMemoryDO> updateWrapper = new LambdaUpdateWrapper<>();
|
||||||
updateWrapper.eq(ChatMemoryDO::getId, chatMemoryDO.getId());
|
updateWrapper.eq(ChatMemoryDO::getId, chatMemoryDO.getId());
|
||||||
if (Objects.nonNull(chatMemoryUpdateReq.getStatus())) {
|
if (Objects.nonNull(chatMemoryUpdateReq.getStatus())) {
|
||||||
@@ -87,6 +97,12 @@ public class MemoryServiceImpl implements MemoryService {
|
|||||||
updateWrapper.set(ChatMemoryDO::getHumanReviewCmt,
|
updateWrapper.set(ChatMemoryDO::getHumanReviewCmt,
|
||||||
chatMemoryUpdateReq.getHumanReviewCmt());
|
chatMemoryUpdateReq.getHumanReviewCmt());
|
||||||
}
|
}
|
||||||
|
if (Objects.nonNull(chatMemoryUpdateReq.getDbSchema())) {
|
||||||
|
updateWrapper.set(ChatMemoryDO::getDbSchema, chatMemoryUpdateReq.getDbSchema());
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(chatMemoryUpdateReq.getS2sql())) {
|
||||||
|
updateWrapper.set(ChatMemoryDO::getS2sql, chatMemoryUpdateReq.getS2sql());
|
||||||
|
}
|
||||||
updateWrapper.set(ChatMemoryDO::getUpdatedAt, new Date());
|
updateWrapper.set(ChatMemoryDO::getUpdatedAt, new Date());
|
||||||
updateWrapper.set(ChatMemoryDO::getUpdatedBy, user.getName());
|
updateWrapper.set(ChatMemoryDO::getUpdatedBy, user.getName());
|
||||||
|
|
||||||
@@ -94,7 +110,22 @@ public class MemoryServiceImpl implements MemoryService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void batchDelete(List<Long> ids) {
|
public void batchDelete(ChatMemoryDeleteReq chatMemoryDeleteReq, User user) {
|
||||||
|
QueryWrapper<ChatMemoryDO> queryWrapper = new QueryWrapper<>();
|
||||||
|
if (!CollectionUtils.isEmpty(chatMemoryDeleteReq.getIds())) {
|
||||||
|
queryWrapper.lambda().in(ChatMemoryDO::getId, chatMemoryDeleteReq.getIds());
|
||||||
|
}
|
||||||
|
if (chatMemoryDeleteReq.getAgentId() != null) {
|
||||||
|
queryWrapper.lambda().eq(ChatMemoryDO::getAgentId, chatMemoryDeleteReq.getAgentId());
|
||||||
|
}
|
||||||
|
List<ChatMemoryDO> chatMemoryDOS = chatMemoryRepository.getMemories(queryWrapper);
|
||||||
|
List<Long> ids = new ArrayList<>();
|
||||||
|
chatMemoryDOS.forEach(chatMemoryDO -> {
|
||||||
|
if (MemoryStatus.ENABLED.toString().equals(chatMemoryDO.getStatus().trim())) {
|
||||||
|
disableMemory(chatMemoryDO);
|
||||||
|
}
|
||||||
|
ids.add(chatMemoryDO.getId());
|
||||||
|
});
|
||||||
chatMemoryRepository.batchDelete(ids);
|
chatMemoryRepository.batchDelete(ids);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -187,4 +218,25 @@ public class MemoryServiceImpl implements MemoryService {
|
|||||||
return memory;
|
return memory;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run(String... args) { // 优化,启动时检查,向量数据,将记忆放到向量数据库
|
||||||
|
loadSysExemplars();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void loadSysExemplars() {
|
||||||
|
try {
|
||||||
|
List<ChatMemory> memories = this
|
||||||
|
.getMemories(ChatMemoryFilter.builder().status(MemoryStatus.ENABLED).build());
|
||||||
|
for (ChatMemory memory : memories) {
|
||||||
|
exemplarService.storeExemplar(
|
||||||
|
embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
|
||||||
|
Text2SQLExemplar.builder().question(memory.getQuestion())
|
||||||
|
.sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema())
|
||||||
|
.sql(memory.getS2sql()).build());
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("Failed to load system exemplars", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -108,6 +108,7 @@ public class PluginServiceImpl implements PluginService {
|
|||||||
if (StringUtils.isNotBlank(pluginQueryReq.getCreatedBy())) {
|
if (StringUtils.isNotBlank(pluginQueryReq.getCreatedBy())) {
|
||||||
queryWrapper.lambda().eq(PluginDO::getCreatedBy, pluginQueryReq.getCreatedBy());
|
queryWrapper.lambda().eq(PluginDO::getCreatedBy, pluginQueryReq.getCreatedBy());
|
||||||
}
|
}
|
||||||
|
queryWrapper.orderByAsc("name");
|
||||||
List<PluginDO> pluginDOS = pluginRepository.query(queryWrapper);
|
List<PluginDO> pluginDOS = pluginRepository.query(queryWrapper);
|
||||||
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
|
if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) {
|
||||||
pluginDOS = pluginDOS.stream()
|
pluginDOS = pluginDOS.stream()
|
||||||
|
|||||||
@@ -21,7 +21,10 @@
|
|||||||
<groupId>org.springframework.boot</groupId>
|
<groupId>org.springframework.boot</groupId>
|
||||||
<artifactId>spring-boot-starter-validation</artifactId>
|
<artifactId>spring-boot-starter-validation</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-autoconfigure-processor</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.slf4j</groupId>
|
<groupId>org.slf4j</groupId>
|
||||||
@@ -33,7 +36,7 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.httpcomponents.client5</groupId>
|
<groupId>org.apache.httpcomponents.client5</groupId>
|
||||||
<artifactId>httpclient5</artifactId>
|
<artifactId>httpclient5</artifactId>
|
||||||
<version>${httpclient5.version}</version> <!-- 请确认使用最新稳定版本 -->
|
<version>${httpclient5.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<!-- <dependency>-->
|
<!-- <dependency>-->
|
||||||
<!-- <groupId>org.apache.httpcomponents</groupId>-->
|
<!-- <groupId>org.apache.httpcomponents</groupId>-->
|
||||||
@@ -182,10 +185,6 @@
|
|||||||
<groupId>dev.langchain4j</groupId>
|
<groupId>dev.langchain4j</groupId>
|
||||||
<artifactId>langchain4j-pgvector</artifactId>
|
<artifactId>langchain4j-pgvector</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<groupId>dev.langchain4j</groupId>
|
|
||||||
<artifactId>langchain4j-azure-open-ai</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>dev.langchain4j</groupId>
|
<groupId>dev.langchain4j</groupId>
|
||||||
<artifactId>langchain4j-embeddings-bge-small-zh</artifactId>
|
<artifactId>langchain4j-embeddings-bge-small-zh</artifactId>
|
||||||
@@ -198,34 +197,6 @@
|
|||||||
<groupId>dev.langchain4j</groupId>
|
<groupId>dev.langchain4j</groupId>
|
||||||
<artifactId>langchain4j-embeddings-all-minilm-l6-v2-q</artifactId>
|
<artifactId>langchain4j-embeddings-all-minilm-l6-v2-q</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<groupId>dev.langchain4j</groupId>
|
|
||||||
<artifactId>langchain4j-qianfan</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>dev.langchain4j</groupId>
|
|
||||||
<artifactId>langchain4j-zhipu-ai</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>dev.langchain4j</groupId>
|
|
||||||
<artifactId>langchain4j-dashscope</artifactId>
|
|
||||||
<exclusions>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.slf4j</groupId>
|
|
||||||
<artifactId>slf4j-simple</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
</exclusions>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>dev.langchain4j</groupId>
|
|
||||||
<artifactId>langchain4j-chatglm</artifactId>
|
|
||||||
<exclusions>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>org.slf4j</groupId>
|
|
||||||
<artifactId>slf4j-simple</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
</exclusions>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>dev.langchain4j</groupId>
|
<groupId>dev.langchain4j</groupId>
|
||||||
<artifactId>langchain4j-ollama</artifactId>
|
<artifactId>langchain4j-ollama</artifactId>
|
||||||
@@ -237,11 +208,6 @@
|
|||||||
<version>${hanlp.version}</version>
|
<version>${hanlp.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework.boot</groupId>
|
|
||||||
<artifactId>spring-boot-autoconfigure-processor</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.google.code.gson</groupId>
|
<groupId>com.google.code.gson</groupId>
|
||||||
<artifactId>gson</artifactId>
|
<artifactId>gson</artifactId>
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ public class LoadRemoveService {
|
|||||||
List<String> resultList = new ArrayList<>(value);
|
List<String> resultList = new ArrayList<>(value);
|
||||||
if (!CollectionUtils.isEmpty(modelIdOrDataSetIds)) {
|
if (!CollectionUtils.isEmpty(modelIdOrDataSetIds)) {
|
||||||
resultList.removeIf(nature -> {
|
resultList.removeIf(nature -> {
|
||||||
if (Objects.isNull(nature)) {
|
if (Objects.isNull(nature) || !nature.startsWith("_")) { // 系统的字典是以 _ 开头的,
|
||||||
|
// 过滤因引用外部字典导致的异常
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
Long id = getId(nature);
|
Long id = getId(nature);
|
||||||
|
|||||||
@@ -77,11 +77,6 @@ public class SemanticSqlConformance implements SqlConformance {
|
|||||||
return SqlConformanceEnum.BIG_QUERY.isMinusAllowed();
|
return SqlConformanceEnum.BIG_QUERY.isMinusAllowed();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean isRegexReplaceCaptureGroupDollarIndexed() {
|
|
||||||
return SqlConformanceEnum.BIG_QUERY.isRegexReplaceCaptureGroupDollarIndexed();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isApplyAllowed() {
|
public boolean isApplyAllowed() {
|
||||||
return SqlConformanceEnum.BIG_QUERY.isApplyAllowed();
|
return SqlConformanceEnum.BIG_QUERY.isApplyAllowed();
|
||||||
|
|||||||
@@ -26,6 +26,16 @@ public class SqlDialectFactory {
|
|||||||
.withLiteralQuoteString("'").withIdentifierQuoteString("\"")
|
.withLiteralQuoteString("'").withIdentifierQuoteString("\"")
|
||||||
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||||
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
||||||
|
public static final Context PRESTO_CONTEXT =
|
||||||
|
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.PRESTO)
|
||||||
|
.withLiteralQuoteString("'").withIdentifierQuoteString("\"")
|
||||||
|
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||||
|
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
||||||
|
public static final Context KYUUBI_CONTEXT =
|
||||||
|
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
|
||||||
|
.withLiteralQuoteString("'").withIdentifierQuoteString("`")
|
||||||
|
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||||
|
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(false);
|
||||||
private static Map<EngineType, SemanticSqlDialect> sqlDialectMap;
|
private static Map<EngineType, SemanticSqlDialect> sqlDialectMap;
|
||||||
|
|
||||||
static {
|
static {
|
||||||
@@ -35,6 +45,10 @@ public class SqlDialectFactory {
|
|||||||
sqlDialectMap.put(EngineType.H2, new SemanticSqlDialect(DEFAULT_CONTEXT));
|
sqlDialectMap.put(EngineType.H2, new SemanticSqlDialect(DEFAULT_CONTEXT));
|
||||||
sqlDialectMap.put(EngineType.POSTGRESQL, new SemanticSqlDialect(POSTGRESQL_CONTEXT));
|
sqlDialectMap.put(EngineType.POSTGRESQL, new SemanticSqlDialect(POSTGRESQL_CONTEXT));
|
||||||
sqlDialectMap.put(EngineType.HANADB, new SemanticSqlDialect(HANADB_CONTEXT));
|
sqlDialectMap.put(EngineType.HANADB, new SemanticSqlDialect(HANADB_CONTEXT));
|
||||||
|
sqlDialectMap.put(EngineType.STARROCKS, new SemanticSqlDialect(DEFAULT_CONTEXT));
|
||||||
|
sqlDialectMap.put(EngineType.KYUUBI, new SemanticSqlDialect(KYUUBI_CONTEXT));
|
||||||
|
sqlDialectMap.put(EngineType.PRESTO, new SemanticSqlDialect(PRESTO_CONTEXT));
|
||||||
|
sqlDialectMap.put(EngineType.TRINO, new SemanticSqlDialect(PRESTO_CONTEXT));
|
||||||
}
|
}
|
||||||
|
|
||||||
public static SemanticSqlDialect getSqlDialect(EngineType engineType) {
|
public static SemanticSqlDialect getSqlDialect(EngineType engineType) {
|
||||||
|
|||||||
@@ -2,19 +2,11 @@ package com.tencent.supersonic.common.calcite;
|
|||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.EngineType;
|
import com.tencent.supersonic.common.pojo.enums.EngineType;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.calcite.sql.SqlIdentifier;
|
import net.sf.jsqlparser.expression.Alias;
|
||||||
import org.apache.calcite.sql.SqlLiteral;
|
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
||||||
import org.apache.calcite.sql.SqlNode;
|
import net.sf.jsqlparser.statement.select.ParenthesedSelect;
|
||||||
import org.apache.calcite.sql.SqlNodeList;
|
import net.sf.jsqlparser.statement.select.Select;
|
||||||
import org.apache.calcite.sql.SqlOrderBy;
|
import net.sf.jsqlparser.statement.select.WithItem;
|
||||||
import org.apache.calcite.sql.SqlSelect;
|
|
||||||
import org.apache.calcite.sql.SqlWith;
|
|
||||||
import org.apache.calcite.sql.SqlWithItem;
|
|
||||||
import org.apache.calcite.sql.SqlWriterConfig;
|
|
||||||
import org.apache.calcite.sql.parser.SqlParseException;
|
|
||||||
import org.apache.calcite.sql.parser.SqlParser;
|
|
||||||
import org.apache.calcite.sql.parser.SqlParserPos;
|
|
||||||
import org.apache.calcite.sql.pretty.SqlPrettyWriter;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -22,85 +14,37 @@ import java.util.List;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
public class SqlMergeWithUtils {
|
public class SqlMergeWithUtils {
|
||||||
public static String mergeWith(EngineType engineType, String sql, List<String> parentSqlList,
|
public static String mergeWith(EngineType engineType, String sql, List<String> parentSqlList,
|
||||||
List<String> parentWithNameList) throws SqlParseException {
|
List<String> parentWithNameList) throws Exception {
|
||||||
SqlParser.Config parserConfig = Configuration.getParserConfig(engineType);
|
|
||||||
|
|
||||||
// Parse the main SQL statement
|
Select selectStatement = (Select) CCJSqlParserUtil.parse(sql);
|
||||||
SqlParser parser = SqlParser.create(sql, parserConfig);
|
List<WithItem> withItemList = new ArrayList<>();
|
||||||
SqlNode sqlNode1 = parser.parseQuery();
|
|
||||||
|
|
||||||
// List to hold all WITH items
|
|
||||||
List<SqlNode> withItemList = new ArrayList<>();
|
|
||||||
|
|
||||||
// Iterate over each parentSql and parentWithName pair
|
|
||||||
for (int i = 0; i < parentSqlList.size(); i++) {
|
for (int i = 0; i < parentSqlList.size(); i++) {
|
||||||
String parentSql = parentSqlList.get(i);
|
String parentSql = parentSqlList.get(i);
|
||||||
String parentWithName = parentWithNameList.get(i);
|
String parentWithName = parentWithNameList.get(i);
|
||||||
|
|
||||||
// Parse the parent SQL statement
|
Select parentSelect = (Select) CCJSqlParserUtil.parse(parentSql);
|
||||||
parser = SqlParser.create(parentSql, parserConfig);
|
ParenthesedSelect select = new ParenthesedSelect();
|
||||||
SqlNode sqlNode2 = parser.parseQuery();
|
select.setSelect(parentSelect);
|
||||||
|
|
||||||
// Create a new WITH item for parentWithName without quotes
|
// Create a new WITH item for parentWithName without quotes
|
||||||
SqlWithItem withItem = new SqlWithItem(SqlParserPos.ZERO,
|
WithItem withItem = new WithItem();
|
||||||
new SqlIdentifier(parentWithName, SqlParserPos.ZERO), null, sqlNode2,
|
withItem.setAlias(new Alias(parentWithName));
|
||||||
SqlLiteral.createBoolean(false, SqlParserPos.ZERO));
|
withItem.setSelect(select);
|
||||||
|
|
||||||
// Add the new WITH item to the list
|
// Add the new WITH item to the list
|
||||||
withItemList.add(withItem);
|
withItemList.add(withItem);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if the main SQL node contains an ORDER BY or LIMIT clause
|
// Extract existing WITH items from mainSelectBody if it has any
|
||||||
SqlNode limitNode = null;
|
if (selectStatement.getWithItemsList() != null) {
|
||||||
SqlNodeList orderByList = null;
|
withItemList.addAll(selectStatement.getWithItemsList());
|
||||||
if (sqlNode1 instanceof SqlOrderBy) {
|
|
||||||
SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode1;
|
|
||||||
limitNode = sqlOrderBy.fetch;
|
|
||||||
orderByList = sqlOrderBy.orderList;
|
|
||||||
sqlNode1 = sqlOrderBy.query;
|
|
||||||
} else if (sqlNode1 instanceof SqlSelect) {
|
|
||||||
SqlSelect sqlSelect = (SqlSelect) sqlNode1;
|
|
||||||
limitNode = sqlSelect.getFetch();
|
|
||||||
sqlSelect.setFetch(null);
|
|
||||||
sqlNode1 = sqlSelect;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract existing WITH items from sqlNode1 if it is a SqlWith
|
// Set the new WITH items list to the main select body
|
||||||
if (sqlNode1 instanceof SqlWith) {
|
selectStatement.setWithItemsList(withItemList);
|
||||||
SqlWith sqlWith = (SqlWith) sqlNode1;
|
|
||||||
withItemList.addAll(sqlWith.withList.getList());
|
|
||||||
sqlNode1 = sqlWith.body;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a new SqlWith node
|
|
||||||
SqlWith finalSqlNode = new SqlWith(SqlParserPos.ZERO,
|
|
||||||
new SqlNodeList(withItemList, SqlParserPos.ZERO), sqlNode1);
|
|
||||||
|
|
||||||
// If there was an ORDER BY or LIMIT clause, wrap the finalSqlNode in a SqlOrderBy
|
|
||||||
SqlNode resultNode = finalSqlNode;
|
|
||||||
if (orderByList != null || limitNode != null) {
|
|
||||||
resultNode = new SqlOrderBy(SqlParserPos.ZERO, finalSqlNode,
|
|
||||||
orderByList != null ? orderByList : SqlNodeList.EMPTY, null, limitNode);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Custom SqlPrettyWriter configuration to avoid quoting identifiers
|
|
||||||
SqlWriterConfig config = Configuration.getSqlWriterConfig(engineType);
|
|
||||||
// Pretty print the final SQL
|
// Pretty print the final SQL
|
||||||
SqlPrettyWriter writer = new SqlPrettyWriter(config);
|
return selectStatement.toString();
|
||||||
return writer.format(resultNode);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static boolean hasWith(EngineType engineType, String sql) throws SqlParseException {
|
|
||||||
SqlParser.Config parserConfig = Configuration.getParserConfig(engineType);
|
|
||||||
SqlParser parser = SqlParser.create(sql, parserConfig);
|
|
||||||
SqlNode sqlNode = parser.parseQuery();
|
|
||||||
SqlNode sqlSelect = sqlNode;
|
|
||||||
if (sqlNode instanceof SqlOrderBy) {
|
|
||||||
SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode;
|
|
||||||
sqlSelect = sqlOrderBy.query;
|
|
||||||
} else if (sqlNode instanceof SqlSelect) {
|
|
||||||
sqlSelect = (SqlSelect) sqlNode;
|
|
||||||
}
|
|
||||||
return sqlSelect instanceof SqlWith;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
package com.tencent.supersonic.common.config;
|
package com.tencent.supersonic.common.config;
|
||||||
|
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.common.pojo.ChatModelConfig;
|
import com.tencent.supersonic.common.pojo.ChatModelConfig;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public class ChatModel {
|
public class ChatModel {
|
||||||
@@ -25,5 +27,11 @@ public class ChatModel {
|
|||||||
|
|
||||||
private String admin;
|
private String admin;
|
||||||
|
|
||||||
private String viewer;
|
private List<String> viewers = Lists.newArrayList();
|
||||||
|
|
||||||
|
private Integer isOpen = 0;
|
||||||
|
|
||||||
|
public boolean isPublic() {
|
||||||
|
return isOpen != null && isOpen == 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,14 +4,10 @@ import com.google.common.collect.ImmutableMap;
|
|||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.common.pojo.EmbeddingModelConfig;
|
import com.tencent.supersonic.common.pojo.EmbeddingModelConfig;
|
||||||
import com.tencent.supersonic.common.pojo.Parameter;
|
import com.tencent.supersonic.common.pojo.Parameter;
|
||||||
import dev.langchain4j.provider.AzureModelFactory;
|
|
||||||
import dev.langchain4j.provider.DashscopeModelFactory;
|
|
||||||
import dev.langchain4j.provider.EmbeddingModelConstant;
|
import dev.langchain4j.provider.EmbeddingModelConstant;
|
||||||
import dev.langchain4j.provider.InMemoryModelFactory;
|
import dev.langchain4j.provider.InMemoryModelFactory;
|
||||||
import dev.langchain4j.provider.OllamaModelFactory;
|
import dev.langchain4j.provider.OllamaModelFactory;
|
||||||
import dev.langchain4j.provider.OpenAiModelFactory;
|
import dev.langchain4j.provider.OpenAiModelFactory;
|
||||||
import dev.langchain4j.provider.QianfanModelFactory;
|
|
||||||
import dev.langchain4j.provider.ZhipuModelFactory;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -70,52 +66,31 @@ public class EmbeddingModelParameterConfig extends ParameterConfig {
|
|||||||
|
|
||||||
private static ArrayList<String> getCandidateValues() {
|
private static ArrayList<String> getCandidateValues() {
|
||||||
return Lists.newArrayList(InMemoryModelFactory.PROVIDER, OpenAiModelFactory.PROVIDER,
|
return Lists.newArrayList(InMemoryModelFactory.PROVIDER, OpenAiModelFactory.PROVIDER,
|
||||||
OllamaModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER,
|
OllamaModelFactory.PROVIDER);
|
||||||
QianfanModelFactory.PROVIDER, ZhipuModelFactory.PROVIDER,
|
|
||||||
AzureModelFactory.PROVIDER);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getBaseUrlDependency() {
|
private static List<Parameter.Dependency> getBaseUrlDependency() {
|
||||||
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
|
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
|
||||||
Lists.newArrayList(OpenAiModelFactory.PROVIDER, OllamaModelFactory.PROVIDER,
|
Lists.newArrayList(OpenAiModelFactory.PROVIDER, OllamaModelFactory.PROVIDER),
|
||||||
AzureModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER,
|
|
||||||
QianfanModelFactory.PROVIDER, ZhipuModelFactory.PROVIDER),
|
|
||||||
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL,
|
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL,
|
||||||
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_BASE_URL,
|
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_BASE_URL));
|
||||||
AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_BASE_URL,
|
|
||||||
DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_BASE_URL,
|
|
||||||
QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_BASE_URL,
|
|
||||||
ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_BASE_URL));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getApiKeyDependency() {
|
private static List<Parameter.Dependency> getApiKeyDependency() {
|
||||||
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
|
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
|
||||||
Lists.newArrayList(OpenAiModelFactory.PROVIDER, AzureModelFactory.PROVIDER,
|
Lists.newArrayList(OpenAiModelFactory.PROVIDER),
|
||||||
DashscopeModelFactory.PROVIDER, QianfanModelFactory.PROVIDER,
|
ImmutableMap.of(OpenAiModelFactory.PROVIDER, DEMO));
|
||||||
ZhipuModelFactory.PROVIDER),
|
|
||||||
ImmutableMap.of(OpenAiModelFactory.PROVIDER, DEMO, AzureModelFactory.PROVIDER, DEMO,
|
|
||||||
DashscopeModelFactory.PROVIDER, DEMO, QianfanModelFactory.PROVIDER, DEMO,
|
|
||||||
ZhipuModelFactory.PROVIDER, DEMO));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getModelNameDependency() {
|
private static List<Parameter.Dependency> getModelNameDependency() {
|
||||||
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
|
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
|
||||||
Lists.newArrayList(InMemoryModelFactory.PROVIDER, OpenAiModelFactory.PROVIDER,
|
Lists.newArrayList(InMemoryModelFactory.PROVIDER, OpenAiModelFactory.PROVIDER,
|
||||||
OllamaModelFactory.PROVIDER, AzureModelFactory.PROVIDER,
|
OllamaModelFactory.PROVIDER),
|
||||||
DashscopeModelFactory.PROVIDER, QianfanModelFactory.PROVIDER,
|
|
||||||
ZhipuModelFactory.PROVIDER),
|
|
||||||
ImmutableMap.of(InMemoryModelFactory.PROVIDER, EmbeddingModelConstant.BGE_SMALL_ZH,
|
ImmutableMap.of(InMemoryModelFactory.PROVIDER, EmbeddingModelConstant.BGE_SMALL_ZH,
|
||||||
OpenAiModelFactory.PROVIDER,
|
OpenAiModelFactory.PROVIDER,
|
||||||
OpenAiModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
|
OpenAiModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
|
||||||
OllamaModelFactory.PROVIDER,
|
OllamaModelFactory.PROVIDER,
|
||||||
OllamaModelFactory.DEFAULT_EMBEDDING_MODEL_NAME, AzureModelFactory.PROVIDER,
|
OllamaModelFactory.DEFAULT_EMBEDDING_MODEL_NAME));
|
||||||
AzureModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
|
|
||||||
DashscopeModelFactory.PROVIDER,
|
|
||||||
DashscopeModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
|
|
||||||
QianfanModelFactory.PROVIDER,
|
|
||||||
QianfanModelFactory.DEFAULT_EMBEDDING_MODEL_NAME,
|
|
||||||
ZhipuModelFactory.PROVIDER,
|
|
||||||
ZhipuModelFactory.DEFAULT_EMBEDDING_MODEL_NAME));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getModelPathDependency() {
|
private static List<Parameter.Dependency> getModelPathDependency() {
|
||||||
@@ -126,7 +101,7 @@ public class EmbeddingModelParameterConfig extends ParameterConfig {
|
|||||||
|
|
||||||
private static List<Parameter.Dependency> getSecretKeyDependency() {
|
private static List<Parameter.Dependency> getSecretKeyDependency() {
|
||||||
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
|
return getDependency(EMBEDDING_MODEL_PROVIDER.getName(),
|
||||||
Lists.newArrayList(QianfanModelFactory.PROVIDER),
|
Lists.newArrayList(OpenAiModelFactory.PROVIDER),
|
||||||
ImmutableMap.of(QianfanModelFactory.PROVIDER, DEMO));
|
ImmutableMap.of(OpenAiModelFactory.PROVIDER, DEMO));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,6 +46,62 @@ public class FieldValueReplaceVisitor extends ExpressionVisitorAdapter {
|
|||||||
replaceComparisonExpression(expr);
|
replaceComparisonExpression(expr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void visit(LikeExpression expr) {
|
||||||
|
Expression leftExpression = expr.getLeftExpression();
|
||||||
|
Expression rightExpression = expr.getRightExpression();
|
||||||
|
|
||||||
|
if (!(leftExpression instanceof Column)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (CollectionUtils.isEmpty(filedNameToValueMap)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (Objects.isNull(rightExpression) || Objects.isNull(leftExpression)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Column column = (Column) leftExpression;
|
||||||
|
String columnName = column.getColumnName();
|
||||||
|
if (StringUtils.isEmpty(columnName)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Map<String, String> valueMap = filedNameToValueMap.get(columnName);
|
||||||
|
if (Objects.isNull(valueMap) || valueMap.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (rightExpression instanceof StringValue) {
|
||||||
|
StringValue rightStringValue = (StringValue) rightExpression;
|
||||||
|
String value = rightStringValue.getValue();
|
||||||
|
|
||||||
|
// 使用split处理方式,按通配符分割字符串,对每个片段进行转换
|
||||||
|
String[] parts = value.split("%", -1);
|
||||||
|
boolean changed = false;
|
||||||
|
|
||||||
|
// 处理每个部分
|
||||||
|
for (int i = 0; i < parts.length; i++) {
|
||||||
|
if (!parts[i].isEmpty()) {
|
||||||
|
String replaceValue = getReplaceValue(valueMap, parts[i]);
|
||||||
|
if (StringUtils.isNotEmpty(replaceValue) && !parts[i].equals(replaceValue)) {
|
||||||
|
parts[i] = replaceValue;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果有任何部分发生变化,则重新构建字符串
|
||||||
|
if (changed) {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (int i = 0; i < parts.length; i++) {
|
||||||
|
sb.append(parts[i]);
|
||||||
|
// 除了最后一个部分,其他部分后面都需要加上"%"
|
||||||
|
if (i < parts.length - 1) {
|
||||||
|
sb.append("%");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rightStringValue.setValue(sb.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void visit(InExpression inExpression) {
|
public void visit(InExpression inExpression) {
|
||||||
if (!(inExpression.getLeftExpression() instanceof Column)) {
|
if (!(inExpression.getLeftExpression() instanceof Column)) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -16,16 +16,21 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
|||||||
import net.sf.jsqlparser.schema.Column;
|
import net.sf.jsqlparser.schema.Column;
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.*;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
|
public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
|
||||||
|
|
||||||
private List<Expression> waitingForAdds = new ArrayList<>();
|
private List<Expression> waitingForAdds = new ArrayList<>();
|
||||||
private Set<String> fieldNames;
|
private Set<String> fieldNames;
|
||||||
|
private Map<String, String> fieldNameMap = new HashMap<>();
|
||||||
|
|
||||||
|
private static Set<String> HAVING_AGG_TYPES = Set.of("SUM", "AVG", "MAX", "MIN", "COUNT");
|
||||||
|
|
||||||
|
public FiledFilterReplaceVisitor(Map<String, String> fieldNameMap) {
|
||||||
|
this.fieldNameMap = fieldNameMap;
|
||||||
|
this.fieldNames = fieldNameMap.keySet();
|
||||||
|
}
|
||||||
|
|
||||||
public FiledFilterReplaceVisitor(Set<String> fieldNames) {
|
public FiledFilterReplaceVisitor(Set<String> fieldNames) {
|
||||||
this.fieldNames = fieldNames;
|
this.fieldNames = fieldNames;
|
||||||
@@ -82,7 +87,22 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
|
|||||||
Expression leftExpression = comparisonOperator.getLeftExpression();
|
Expression leftExpression = comparisonOperator.getLeftExpression();
|
||||||
|
|
||||||
if (!(leftExpression instanceof Function)) {
|
if (!(leftExpression instanceof Function)) {
|
||||||
return result;
|
if (leftExpression instanceof Column) {
|
||||||
|
Column leftColumn = (Column) leftExpression;
|
||||||
|
String agg = fieldNameMap.get(leftColumn.getColumnName());
|
||||||
|
if (agg != null && HAVING_AGG_TYPES.contains(agg.toUpperCase())) {
|
||||||
|
Expression expression = parseCondExpression(comparisonOperator, condExpr);
|
||||||
|
if (Objects.nonNull(expression)) {
|
||||||
|
result.add(expression);
|
||||||
|
return result;
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} else {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Function leftFunction = (Function) leftExpression;
|
Function leftFunction = (Function) leftExpression;
|
||||||
@@ -91,7 +111,8 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ExpressionList<?> leftFunctionParams = leftFunction.getParameters();
|
ExpressionList<?> leftFunctionParams = leftFunction.getParameters();
|
||||||
if (CollectionUtils.isEmpty(leftFunctionParams)) {
|
if (CollectionUtils.isEmpty(leftFunctionParams)
|
||||||
|
|| !(leftFunctionParams.get(0) instanceof Column)) {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,14 +122,24 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Expression expression = parseCondExpression(comparisonOperator, condExpr);
|
||||||
|
if (Objects.nonNull(expression)) {
|
||||||
|
result.add(expression);
|
||||||
|
return result;
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Expression parseCondExpression(ComparisonOperator comparisonOperator, String condExpr) {
|
||||||
try {
|
try {
|
||||||
|
String comparisonOperatorStr = comparisonOperator.toString();
|
||||||
ComparisonOperator parsedExpression =
|
ComparisonOperator parsedExpression =
|
||||||
(ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
|
(ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
|
||||||
comparisonOperator.setLeftExpression(parsedExpression.getLeftExpression());
|
comparisonOperator.setLeftExpression(parsedExpression.getLeftExpression());
|
||||||
comparisonOperator.setRightExpression(parsedExpression.getRightExpression());
|
comparisonOperator.setRightExpression(parsedExpression.getRightExpression());
|
||||||
comparisonOperator.setASTNode(parsedExpression.getASTNode());
|
comparisonOperator.setASTNode(parsedExpression.getASTNode());
|
||||||
result.add(CCJSqlParserUtil.parseCondExpression(comparisonOperatorStr));
|
return CCJSqlParserUtil.parseCondExpression(comparisonOperatorStr);
|
||||||
return result;
|
|
||||||
} catch (JSQLParserException e) {
|
} catch (JSQLParserException e) {
|
||||||
log.error("JSQLParserException", e);
|
log.error("JSQLParserException", e);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -309,7 +309,7 @@ public class SqlAddHelper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String addHaving(String sql, Set<String> fieldNames) {
|
public static String addHaving(String sql, Map<String, String> fieldNames) {
|
||||||
Select selectStatement = SqlSelectHelper.getSelect(sql);
|
Select selectStatement = SqlSelectHelper.getSelect(sql);
|
||||||
|
|
||||||
if (!(selectStatement instanceof PlainSelect)) {
|
if (!(selectStatement instanceof PlainSelect)) {
|
||||||
|
|||||||
@@ -146,6 +146,10 @@ public class SqlReplaceHelper {
|
|||||||
public static String replaceFields(String sql, Map<String, String> fieldNameMap,
|
public static String replaceFields(String sql, Map<String, String> fieldNameMap,
|
||||||
boolean exactReplace) {
|
boolean exactReplace) {
|
||||||
Select selectStatement = SqlSelectHelper.getSelect(sql);
|
Select selectStatement = SqlSelectHelper.getSelect(sql);
|
||||||
|
// alias field should not be replaced
|
||||||
|
Set<String> aliases = SqlSelectHelper.getAliasFields(sql);
|
||||||
|
aliases.forEach(alias -> fieldNameMap.put(alias, alias));
|
||||||
|
|
||||||
Set<Select> plainSelectList = SqlSelectHelper.getAllSelect(selectStatement);
|
Set<Select> plainSelectList = SqlSelectHelper.getAllSelect(selectStatement);
|
||||||
for (Select plainSelect : plainSelectList) {
|
for (Select plainSelect : plainSelectList) {
|
||||||
if (plainSelect instanceof PlainSelect) {
|
if (plainSelect instanceof PlainSelect) {
|
||||||
@@ -723,7 +727,7 @@ public class SqlReplaceHelper {
|
|||||||
List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
|
List<PlainSelect> plainSelects = SqlSelectHelper.getPlainSelects(plainSelectList);
|
||||||
for (PlainSelect plainSelect : plainSelects) {
|
for (PlainSelect plainSelect : plainSelects) {
|
||||||
if (Objects.nonNull(plainSelect.getFromItem())) {
|
if (Objects.nonNull(plainSelect.getFromItem())) {
|
||||||
Table table = (Table) plainSelect.getFromItem();
|
Table table = SqlSelectHelper.getTable(plainSelect.getFromItem());
|
||||||
if (table.getName().equals(tableName)) {
|
if (table.getName().equals(tableName)) {
|
||||||
replacePlainSelectByExpr(plainSelect, replace);
|
replacePlainSelectByExpr(plainSelect, replace);
|
||||||
if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
|
if (SqlSelectHelper.hasAggregateFunction(plainSelect)) {
|
||||||
|
|||||||
@@ -723,6 +723,44 @@ public class SqlSelectHelper {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Table getTable(FromItem fromItem) {
|
||||||
|
Table table = null;
|
||||||
|
if (fromItem instanceof Table) {
|
||||||
|
table = (Table) fromItem;
|
||||||
|
} else if (fromItem instanceof ParenthesedSelect) {
|
||||||
|
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) fromItem;
|
||||||
|
if (parenthesedSelect.getSelect() instanceof PlainSelect) {
|
||||||
|
PlainSelect subSelect = (PlainSelect) parenthesedSelect.getSelect();
|
||||||
|
table = getTable(subSelect.getSelectBody());
|
||||||
|
} else if (parenthesedSelect.getSelect() instanceof SetOperationList) {
|
||||||
|
table = getTable(parenthesedSelect.getSelect());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Table getTable(Select select) {
|
||||||
|
if (select == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
List<PlainSelect> plainSelectList = getWithItem(select);
|
||||||
|
if (!CollectionUtils.isEmpty(plainSelectList)) {
|
||||||
|
List<PlainSelect> selectList = new ArrayList<>(plainSelectList);
|
||||||
|
Table table = getTable(selectList.get(0));
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
if (select instanceof PlainSelect) {
|
||||||
|
PlainSelect plainSelect = (PlainSelect) select;
|
||||||
|
return getTable(plainSelect.getFromItem());
|
||||||
|
} else if (select instanceof SetOperationList) {
|
||||||
|
SetOperationList setOperationList = (SetOperationList) select;
|
||||||
|
if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
|
||||||
|
return getTable(setOperationList.getSelects().get(0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
public static String getDbTableName(String sql) {
|
public static String getDbTableName(String sql) {
|
||||||
Table table = getTable(sql);
|
Table table = getTable(sql);
|
||||||
return table.getFullyQualifiedName();
|
return table.getFullyQualifiedName();
|
||||||
@@ -989,6 +1027,15 @@ public class SqlSelectHelper {
|
|||||||
for (SelectItem selectItem : selectItems) {
|
for (SelectItem selectItem : selectItems) {
|
||||||
selectItem.accept(visitor);
|
selectItem.accept(visitor);
|
||||||
}
|
}
|
||||||
|
if (plainSelect.getHaving() != null) {
|
||||||
|
plainSelect.getHaving().accept(visitor);
|
||||||
|
}
|
||||||
|
if (!CollectionUtils.isEmpty(plainSelect.getOrderByElements())) {
|
||||||
|
for (OrderByElement orderByElement : plainSelect.getOrderByElements()) {
|
||||||
|
orderByElement.getExpression().accept(visitor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return !visitor.getFunctionNames().isEmpty();
|
return !visitor.getFunctionNames().isEmpty();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -30,4 +30,6 @@ public class ChatModelDO {
|
|||||||
private String admin;
|
private String admin;
|
||||||
|
|
||||||
private String viewer;
|
private String viewer;
|
||||||
|
|
||||||
|
private Integer isOpen;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ public class ChatModelConfig implements Serializable {
|
|||||||
private Boolean logRequests = false;
|
private Boolean logRequests = false;
|
||||||
private Boolean logResponses = false;
|
private Boolean logResponses = false;
|
||||||
private Boolean enableSearch = false;
|
private Boolean enableSearch = false;
|
||||||
|
private Boolean jsonFormat = false;
|
||||||
|
private String jsonFormatType = "json_schema";
|
||||||
|
|
||||||
public String keyDecrypt() {
|
public String keyDecrypt() {
|
||||||
return AESEncryptionUtil.aesDecryptECB(getApiKey());
|
return AESEncryptionUtil.aesDecryptECB(getApiKey());
|
||||||
|
|||||||
@@ -2,15 +2,7 @@ package com.tencent.supersonic.common.pojo;
|
|||||||
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import dev.langchain4j.provider.AzureModelFactory;
|
import dev.langchain4j.provider.*;
|
||||||
import dev.langchain4j.provider.DashscopeModelFactory;
|
|
||||||
import dev.langchain4j.provider.DifyModelFactory;
|
|
||||||
import dev.langchain4j.provider.LocalAiModelFactory;
|
|
||||||
import dev.langchain4j.provider.ModelProvider;
|
|
||||||
import dev.langchain4j.provider.OllamaModelFactory;
|
|
||||||
import dev.langchain4j.provider.OpenAiModelFactory;
|
|
||||||
import dev.langchain4j.provider.QianfanModelFactory;
|
|
||||||
import dev.langchain4j.provider.ZhipuModelFactory;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -21,7 +13,7 @@ public class ChatModelParameters {
|
|||||||
|
|
||||||
public static final Parameter CHAT_MODEL_PROVIDER =
|
public static final Parameter CHAT_MODEL_PROVIDER =
|
||||||
new Parameter("provider", ModelProvider.DEMO_CHAT_MODEL.getProvider(), "接口协议", "",
|
new Parameter("provider", ModelProvider.DEMO_CHAT_MODEL.getProvider(), "接口协议", "",
|
||||||
"list", MODULE_NAME, getCandidateValues());
|
"list", MODULE_NAME, getCandidateProviders());
|
||||||
|
|
||||||
public static final Parameter CHAT_MODEL_BASE_URL =
|
public static final Parameter CHAT_MODEL_BASE_URL =
|
||||||
new Parameter("baseUrl", ModelProvider.DEMO_CHAT_MODEL.getBaseUrl(), "BaseUrl", "",
|
new Parameter("baseUrl", ModelProvider.DEMO_CHAT_MODEL.getBaseUrl(), "BaseUrl", "",
|
||||||
@@ -37,15 +29,6 @@ public class ChatModelParameters {
|
|||||||
public static final Parameter CHAT_MODEL_API_VERSION = new Parameter("apiVersion", "2024-02-01",
|
public static final Parameter CHAT_MODEL_API_VERSION = new Parameter("apiVersion", "2024-02-01",
|
||||||
"ApiVersion", "", "string", MODULE_NAME, null, getApiVersionDependency());
|
"ApiVersion", "", "string", MODULE_NAME, null, getApiVersionDependency());
|
||||||
|
|
||||||
public static final Parameter CHAT_MODEL_ENDPOINT = new Parameter("endpoint", "llama_2_70b",
|
|
||||||
"Endpoint", "", "string", MODULE_NAME, null, getEndpointDependency());
|
|
||||||
|
|
||||||
public static final Parameter CHAT_MODEL_SECRET_KEY = new Parameter("secretKey", "demo",
|
|
||||||
"SecretKey", "", "password", MODULE_NAME, null, getSecretKeyDependency());
|
|
||||||
|
|
||||||
public static final Parameter CHAT_MODEL_ENABLE_SEARCH = new Parameter("enableSearch", "false",
|
|
||||||
"是否启用搜索增强功能,设为false表示不启用", "", "bool", MODULE_NAME, null, getEnableSearchDependency());
|
|
||||||
|
|
||||||
public static final Parameter CHAT_MODEL_TEMPERATURE =
|
public static final Parameter CHAT_MODEL_TEMPERATURE =
|
||||||
new Parameter("temperature", "0.0", "Temperature", "", "slider", MODULE_NAME);
|
new Parameter("temperature", "0.0", "Temperature", "", "slider", MODULE_NAME);
|
||||||
|
|
||||||
@@ -53,42 +36,27 @@ public class ChatModelParameters {
|
|||||||
new Parameter("timeOut", "60", "超时时间(秒)", "", "number", MODULE_NAME);
|
new Parameter("timeOut", "60", "超时时间(秒)", "", "number", MODULE_NAME);
|
||||||
|
|
||||||
public static List<Parameter> getParameters() {
|
public static List<Parameter> getParameters() {
|
||||||
return Lists.newArrayList(CHAT_MODEL_PROVIDER, CHAT_MODEL_BASE_URL, CHAT_MODEL_ENDPOINT,
|
return Lists.newArrayList(CHAT_MODEL_PROVIDER, CHAT_MODEL_BASE_URL, CHAT_MODEL_API_KEY,
|
||||||
CHAT_MODEL_API_KEY, CHAT_MODEL_SECRET_KEY, CHAT_MODEL_NAME, CHAT_MODEL_API_VERSION,
|
CHAT_MODEL_NAME, CHAT_MODEL_API_VERSION, CHAT_MODEL_TEMPERATURE,
|
||||||
CHAT_MODEL_ENABLE_SEARCH, CHAT_MODEL_TEMPERATURE, CHAT_MODEL_TIMEOUT);
|
CHAT_MODEL_TIMEOUT);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<String> getCandidateValues() {
|
private static List<String> getCandidateProviders() {
|
||||||
return Lists.newArrayList(OpenAiModelFactory.PROVIDER, OllamaModelFactory.PROVIDER,
|
return Lists.newArrayList(OpenAiModelFactory.PROVIDER, OllamaModelFactory.PROVIDER,
|
||||||
QianfanModelFactory.PROVIDER, ZhipuModelFactory.PROVIDER,
|
DifyModelFactory.PROVIDER);
|
||||||
LocalAiModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER,
|
|
||||||
AzureModelFactory.PROVIDER, DifyModelFactory.PROVIDER);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getBaseUrlDependency() {
|
private static List<Parameter.Dependency> getBaseUrlDependency() {
|
||||||
return getDependency(CHAT_MODEL_PROVIDER.getName(), getCandidateValues(),
|
return getDependency(CHAT_MODEL_PROVIDER.getName(), getCandidateProviders(),
|
||||||
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL,
|
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_BASE_URL,
|
||||||
AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_BASE_URL,
|
|
||||||
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_BASE_URL,
|
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_BASE_URL,
|
||||||
QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_BASE_URL,
|
|
||||||
ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_BASE_URL,
|
|
||||||
LocalAiModelFactory.PROVIDER, LocalAiModelFactory.DEFAULT_BASE_URL,
|
|
||||||
DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_BASE_URL,
|
|
||||||
DifyModelFactory.PROVIDER, DifyModelFactory.DEFAULT_BASE_URL));
|
DifyModelFactory.PROVIDER, DifyModelFactory.DEFAULT_BASE_URL));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getApiKeyDependency() {
|
private static List<Parameter.Dependency> getApiKeyDependency() {
|
||||||
return getDependency(CHAT_MODEL_PROVIDER.getName(),
|
return getDependency(CHAT_MODEL_PROVIDER.getName(),
|
||||||
Lists.newArrayList(OpenAiModelFactory.PROVIDER, QianfanModelFactory.PROVIDER,
|
Lists.newArrayList(OpenAiModelFactory.PROVIDER, DifyModelFactory.PROVIDER),
|
||||||
ZhipuModelFactory.PROVIDER, LocalAiModelFactory.PROVIDER,
|
|
||||||
AzureModelFactory.PROVIDER, DashscopeModelFactory.PROVIDER,
|
|
||||||
DifyModelFactory.PROVIDER),
|
|
||||||
ImmutableMap.of(OpenAiModelFactory.PROVIDER,
|
ImmutableMap.of(OpenAiModelFactory.PROVIDER,
|
||||||
ModelProvider.DEMO_CHAT_MODEL.getApiKey(), QianfanModelFactory.PROVIDER,
|
|
||||||
ModelProvider.DEMO_CHAT_MODEL.getApiKey(), ZhipuModelFactory.PROVIDER,
|
|
||||||
ModelProvider.DEMO_CHAT_MODEL.getApiKey(), LocalAiModelFactory.PROVIDER,
|
|
||||||
ModelProvider.DEMO_CHAT_MODEL.getApiKey(), AzureModelFactory.PROVIDER,
|
|
||||||
ModelProvider.DEMO_CHAT_MODEL.getApiKey(), DashscopeModelFactory.PROVIDER,
|
|
||||||
ModelProvider.DEMO_CHAT_MODEL.getApiKey(), DifyModelFactory.PROVIDER,
|
ModelProvider.DEMO_CHAT_MODEL.getApiKey(), DifyModelFactory.PROVIDER,
|
||||||
ModelProvider.DEMO_CHAT_MODEL.getApiKey()));
|
ModelProvider.DEMO_CHAT_MODEL.getApiKey()));
|
||||||
}
|
}
|
||||||
@@ -100,33 +68,28 @@ public class ChatModelParameters {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getModelNameDependency() {
|
private static List<Parameter.Dependency> getModelNameDependency() {
|
||||||
return getDependency(CHAT_MODEL_PROVIDER.getName(), getCandidateValues(),
|
return getDependency(CHAT_MODEL_PROVIDER.getName(), getCandidateProviders(),
|
||||||
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_MODEL_NAME,
|
ImmutableMap.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_MODEL_NAME,
|
||||||
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_MODEL_NAME,
|
OllamaModelFactory.PROVIDER, OllamaModelFactory.DEFAULT_MODEL_NAME,
|
||||||
QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_MODEL_NAME,
|
|
||||||
ZhipuModelFactory.PROVIDER, ZhipuModelFactory.DEFAULT_MODEL_NAME,
|
|
||||||
LocalAiModelFactory.PROVIDER, LocalAiModelFactory.DEFAULT_MODEL_NAME,
|
|
||||||
AzureModelFactory.PROVIDER, AzureModelFactory.DEFAULT_MODEL_NAME,
|
|
||||||
DashscopeModelFactory.PROVIDER, DashscopeModelFactory.DEFAULT_MODEL_NAME,
|
|
||||||
DifyModelFactory.PROVIDER, DifyModelFactory.DEFAULT_MODEL_NAME));
|
DifyModelFactory.PROVIDER, DifyModelFactory.DEFAULT_MODEL_NAME));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getEndpointDependency() {
|
private static List<Parameter.Dependency> getEndpointDependency() {
|
||||||
return getDependency(CHAT_MODEL_PROVIDER.getName(),
|
return getDependency(CHAT_MODEL_PROVIDER.getName(),
|
||||||
Lists.newArrayList(QianfanModelFactory.PROVIDER), ImmutableMap
|
Lists.newArrayList(OpenAiModelFactory.PROVIDER), ImmutableMap
|
||||||
.of(QianfanModelFactory.PROVIDER, QianfanModelFactory.DEFAULT_ENDPOINT));
|
.of(OpenAiModelFactory.PROVIDER, OpenAiModelFactory.DEFAULT_MODEL_NAME));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getEnableSearchDependency() {
|
private static List<Parameter.Dependency> getEnableSearchDependency() {
|
||||||
return getDependency(CHAT_MODEL_PROVIDER.getName(),
|
return getDependency(CHAT_MODEL_PROVIDER.getName(),
|
||||||
Lists.newArrayList(DashscopeModelFactory.PROVIDER),
|
Lists.newArrayList(OpenAiModelFactory.PROVIDER),
|
||||||
ImmutableMap.of(DashscopeModelFactory.PROVIDER, "false"));
|
ImmutableMap.of(OpenAiModelFactory.PROVIDER, "false"));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getSecretKeyDependency() {
|
private static List<Parameter.Dependency> getSecretKeyDependency() {
|
||||||
return getDependency(CHAT_MODEL_PROVIDER.getName(),
|
return getDependency(CHAT_MODEL_PROVIDER.getName(),
|
||||||
Lists.newArrayList(QianfanModelFactory.PROVIDER), ImmutableMap.of(
|
Lists.newArrayList(OpenAiModelFactory.PROVIDER), ImmutableMap.of(
|
||||||
QianfanModelFactory.PROVIDER, ModelProvider.DEMO_CHAT_MODEL.getApiKey()));
|
OpenAiModelFactory.PROVIDER, ModelProvider.DEMO_CHAT_MODEL.getApiKey()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Parameter.Dependency> getDependency(String dependencyParameterName,
|
private static List<Parameter.Dependency> getDependency(String dependencyParameterName,
|
||||||
|
|||||||
@@ -1,27 +1,26 @@
|
|||||||
package com.tencent.supersonic.common.pojo;
|
package com.tencent.supersonic.common.pojo;
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.EventType;
|
import com.tencent.supersonic.common.pojo.enums.EventType;
|
||||||
|
import lombok.Getter;
|
||||||
import org.springframework.context.ApplicationEvent;
|
import org.springframework.context.ApplicationEvent;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
@Getter
|
||||||
public class DataEvent extends ApplicationEvent {
|
public class DataEvent extends ApplicationEvent {
|
||||||
|
|
||||||
private List<DataItem> dataItems;
|
private final List<DataItem> dataItems;
|
||||||
|
|
||||||
private EventType eventType;
|
private final EventType eventType;
|
||||||
|
|
||||||
public DataEvent(Object source, List<DataItem> dataItems, EventType eventType) {
|
private final String userName;
|
||||||
|
|
||||||
|
public DataEvent(Object source, List<DataItem> dataItems, EventType eventType,
|
||||||
|
String userName) {
|
||||||
super(source);
|
super(source);
|
||||||
this.dataItems = dataItems;
|
this.dataItems = dataItems;
|
||||||
this.eventType = eventType;
|
this.eventType = eventType;
|
||||||
|
this.userName = userName;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<DataItem> getDataItems() {
|
|
||||||
return dataItems;
|
|
||||||
}
|
|
||||||
|
|
||||||
public EventType getEventType() {
|
|
||||||
return eventType;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,4 +5,6 @@ public class DimensionConstants {
|
|||||||
public static final String DIMENSION_TIME_FORMAT = "time_format";
|
public static final String DIMENSION_TIME_FORMAT = "time_format";
|
||||||
|
|
||||||
public static final String DIMENSION_TYPE = "dimension_type";
|
public static final String DIMENSION_TYPE = "dimension_type";
|
||||||
|
|
||||||
|
public static final String DIMENSION_DATA_TYPE = "dimension_data_type";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package com.tencent.supersonic.common.pojo;
|
|||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import jakarta.validation.constraints.NotBlank;
|
import jakarta.validation.constraints.NotBlank;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.ASC_UPPER;
|
import static com.tencent.supersonic.common.pojo.Constants.ASC_UPPER;
|
||||||
|
|||||||
@@ -22,4 +22,6 @@ public class Text2SQLExemplar implements Serializable {
|
|||||||
private String dbSchema;
|
private String dbSchema;
|
||||||
|
|
||||||
private String sql;
|
private String sql;
|
||||||
|
|
||||||
|
protected double similarity; // 传递相似度,可以作为样本筛选的依据
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import lombok.NoArgsConstructor;
|
|||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@NoArgsConstructor
|
@NoArgsConstructor
|
||||||
@@ -22,26 +23,28 @@ public class User implements Serializable {
|
|||||||
|
|
||||||
private Integer isAdmin;
|
private Integer isAdmin;
|
||||||
|
|
||||||
|
private Timestamp lastLogin;
|
||||||
|
|
||||||
public static User get(Long id, String name, String displayName, String email,
|
public static User get(Long id, String name, String displayName, String email,
|
||||||
Integer isAdmin) {
|
Integer isAdmin) {
|
||||||
return new User(id, name, displayName, email, isAdmin);
|
return new User(id, name, displayName, email, isAdmin, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static User get(Long id, String name) {
|
public static User get(Long id, String name) {
|
||||||
return new User(id, name, name, name, 0);
|
return new User(id, name, name, name, 0, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static User getDefaultUser() {
|
public static User getDefaultUser() {
|
||||||
return new User(1L, "admin", "admin", "admin@email", 1);
|
return new User(1L, "admin", "admin", "admin@email", 1, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static User getVisitUser() {
|
public static User getVisitUser() {
|
||||||
return new User(1L, "visit", "visit", "visit@email", 0);
|
return new User(1L, "visit", "visit", "visit@email", 0, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static User getAppUser(int appId) {
|
public static User getAppUser(int appId) {
|
||||||
String name = String.format("app_%s", appId);
|
String name = String.format("app_%s", appId);
|
||||||
return new User(1L, name, name, "", 1);
|
return new User(1L, name, name, "", 1, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getDisplayName() {
|
public String getDisplayName() {
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ public enum EngineType {
|
|||||||
STARROCKS(10, "STARROCKS"),
|
STARROCKS(10, "STARROCKS"),
|
||||||
KYUUBI(11, "KYUUBI"),
|
KYUUBI(11, "KYUUBI"),
|
||||||
PRESTO(12, "PRESTO"),
|
PRESTO(12, "PRESTO"),
|
||||||
TRINO(13, "TRINO"),;
|
TRINO(13, "TRINO"),
|
||||||
|
ORACLE(14, "ORACLE");
|
||||||
|
|
||||||
private Integer code;
|
private Integer code;
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
package com.tencent.supersonic.common.pojo.enums;
|
package com.tencent.supersonic.common.pojo.enums;
|
||||||
|
|
||||||
public enum TypeEnums {
|
public enum TypeEnums {
|
||||||
METRIC, DIMENSION, TAG, DOMAIN, DATASET, MODEL, UNKNOWN
|
METRIC, DIMENSION, VALUE, TAG, DOMAIN, DATASET, MODEL, UNKNOWN
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import com.tencent.supersonic.common.pojo.User;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public interface ChatModelService {
|
public interface ChatModelService {
|
||||||
List<ChatModel> getChatModels();
|
List<ChatModel> getChatModels(User user);
|
||||||
|
|
||||||
ChatModel getChatModel(Integer id);
|
ChatModel getChatModel(Integer id);
|
||||||
|
|
||||||
@@ -15,5 +15,5 @@ public interface ChatModelService {
|
|||||||
|
|
||||||
ChatModel updateChatModel(ChatModel chatModel, User user);
|
ChatModel updateChatModel(ChatModel chatModel, User user);
|
||||||
|
|
||||||
void deleteChatModel(Integer id);
|
void deleteChatModel(Integer id, User user);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import org.apache.commons.lang3.StringUtils;
|
|||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.util.Comparator;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
@@ -23,8 +24,15 @@ import java.util.stream.Collectors;
|
|||||||
public class ChatModelServiceImpl extends ServiceImpl<ChatModelMapper, ChatModelDO>
|
public class ChatModelServiceImpl extends ServiceImpl<ChatModelMapper, ChatModelDO>
|
||||||
implements ChatModelService {
|
implements ChatModelService {
|
||||||
@Override
|
@Override
|
||||||
public List<ChatModel> getChatModels() {
|
public List<ChatModel> getChatModels(User user) {
|
||||||
return list().stream().map(this::convert).collect(Collectors.toList());
|
return list().stream().map(this::convert).filter(chatModel -> {
|
||||||
|
if (chatModel.isPublic() || user.isSuperAdmin()
|
||||||
|
|| chatModel.getCreatedBy().equals(user.getName())
|
||||||
|
|| chatModel.getViewers().contains(user.getName())) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}).sorted(Comparator.comparingLong(ChatModel::getId)).collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -41,10 +49,14 @@ public class ChatModelServiceImpl extends ServiceImpl<ChatModelMapper, ChatModel
|
|||||||
chatModelDO.setCreatedBy(user.getName());
|
chatModelDO.setCreatedBy(user.getName());
|
||||||
chatModelDO.setCreatedAt(new Date());
|
chatModelDO.setCreatedAt(new Date());
|
||||||
chatModelDO.setUpdatedBy(user.getName());
|
chatModelDO.setUpdatedBy(user.getName());
|
||||||
chatModelDO.setUpdatedAt(new Date());
|
chatModelDO.setUpdatedAt(chatModelDO.getCreatedAt());
|
||||||
|
chatModelDO.setIsOpen(chatModel.getIsOpen());
|
||||||
if (StringUtils.isBlank(chatModel.getAdmin())) {
|
if (StringUtils.isBlank(chatModel.getAdmin())) {
|
||||||
chatModelDO.setAdmin(user.getName());
|
chatModelDO.setAdmin(user.getName());
|
||||||
}
|
}
|
||||||
|
if (!chatModel.getViewers().isEmpty()) {
|
||||||
|
chatModelDO.setViewer(JsonUtil.toString(chatModel.getViewers()));
|
||||||
|
}
|
||||||
save(chatModelDO);
|
save(chatModelDO);
|
||||||
chatModel.setId(chatModelDO.getId());
|
chatModel.setId(chatModelDO.getId());
|
||||||
return chatModel;
|
return chatModel;
|
||||||
@@ -55,15 +67,24 @@ public class ChatModelServiceImpl extends ServiceImpl<ChatModelMapper, ChatModel
|
|||||||
ChatModelDO chatModelDO = convert(chatModel);
|
ChatModelDO chatModelDO = convert(chatModel);
|
||||||
chatModelDO.setUpdatedBy(user.getName());
|
chatModelDO.setUpdatedBy(user.getName());
|
||||||
chatModelDO.setUpdatedAt(new Date());
|
chatModelDO.setUpdatedAt(new Date());
|
||||||
|
chatModelDO.setIsOpen(chatModel.getIsOpen());
|
||||||
if (StringUtils.isBlank(chatModel.getAdmin())) {
|
if (StringUtils.isBlank(chatModel.getAdmin())) {
|
||||||
chatModel.setAdmin(user.getName());
|
chatModel.setAdmin(user.getName());
|
||||||
}
|
}
|
||||||
|
if (!chatModel.getViewers().isEmpty()) {
|
||||||
|
chatModelDO.setViewer(JsonUtil.toString(chatModel.getViewers()));
|
||||||
|
}
|
||||||
updateById(chatModelDO);
|
updateById(chatModelDO);
|
||||||
return chatModel;
|
return chatModel;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void deleteChatModel(Integer id) {
|
public void deleteChatModel(Integer id, User user) {
|
||||||
|
ChatModel chatModel = getChatModel(id);
|
||||||
|
if (!checkAdminPermission(user, chatModel)) {
|
||||||
|
throw new RuntimeException("没有权限删除该大模型");
|
||||||
|
}
|
||||||
|
|
||||||
removeById(id);
|
removeById(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,6 +95,7 @@ public class ChatModelServiceImpl extends ServiceImpl<ChatModelMapper, ChatModel
|
|||||||
ChatModel chatModel = new ChatModel();
|
ChatModel chatModel = new ChatModel();
|
||||||
BeanUtils.copyProperties(chatModelDO, chatModel);
|
BeanUtils.copyProperties(chatModelDO, chatModel);
|
||||||
chatModel.setConfig(JsonUtil.toObject(chatModelDO.getConfig(), ChatModelConfig.class));
|
chatModel.setConfig(JsonUtil.toObject(chatModelDO.getConfig(), ChatModelConfig.class));
|
||||||
|
chatModel.setViewers(JsonUtil.toList(chatModelDO.getViewer(), String.class));
|
||||||
return chatModel;
|
return chatModel;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -86,4 +108,13 @@ public class ChatModelServiceImpl extends ServiceImpl<ChatModelMapper, ChatModel
|
|||||||
chatModelDO.setConfig(JsonUtil.toString(chatModel.getConfig()));
|
chatModelDO.setConfig(JsonUtil.toString(chatModel.getConfig()));
|
||||||
return chatModelDO;
|
return chatModelDO;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private boolean checkAdminPermission(User user, ChatModel chatModel) {
|
||||||
|
String admin = chatModel.getAdmin();
|
||||||
|
if (user.isSuperAdmin()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return admin != null && admin.equals(user.getName())
|
||||||
|
|| chatModel.getCreatedBy().equals(user.getName());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,10 +49,10 @@ public class EmbeddingServiceImpl implements EmbeddingService {
|
|||||||
try {
|
try {
|
||||||
EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel();
|
EmbeddingModel embeddingModel = ModelProvider.getEmbeddingModel();
|
||||||
Embedding embedding = embeddingModel.embed(question).content();
|
Embedding embedding = embeddingModel.embed(question).content();
|
||||||
boolean existSegment = existSegment(embeddingStore, query, embedding);
|
MetadataFilterBuilder filterBuilder =
|
||||||
if (existSegment) {
|
new MetadataFilterBuilder(TextSegmentConvert.QUERY_ID);
|
||||||
continue;
|
Filter filter = filterBuilder.isEqualTo(TextSegmentConvert.getQueryId(query));
|
||||||
}
|
embeddingStore.removeAll(filter);
|
||||||
embeddingStore.add(embedding, query);
|
embeddingStore.add(embedding, query);
|
||||||
cache.put(TextSegmentConvert.getQueryId(query), true);
|
cache.put(TextSegmentConvert.getQueryId(query), true);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@@ -62,14 +62,14 @@ public class EmbeddingServiceImpl implements EmbeddingService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean existSegment(EmbeddingStore embeddingStore, TextSegment query,
|
private boolean existSegment(String collectionName, EmbeddingStore embeddingStore,
|
||||||
Embedding embedding) {
|
TextSegment query, Embedding embedding) {
|
||||||
String queryId = TextSegmentConvert.getQueryId(query);
|
String queryId = TextSegmentConvert.getQueryId(query);
|
||||||
if (queryId == null) {
|
if (queryId == null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
// Check cache first
|
// Check cache first
|
||||||
Boolean cachedResult = cache.getIfPresent(queryId);
|
Boolean cachedResult = cache.getIfPresent(collectionName + queryId);
|
||||||
if (cachedResult != null) {
|
if (cachedResult != null) {
|
||||||
return cachedResult;
|
return cachedResult;
|
||||||
}
|
}
|
||||||
@@ -82,7 +82,7 @@ public class EmbeddingServiceImpl implements EmbeddingService {
|
|||||||
EmbeddingSearchResult result = embeddingStore.search(request);
|
EmbeddingSearchResult result = embeddingStore.search(request);
|
||||||
List<EmbeddingMatch<TextSegment>> relevant = result.matches();
|
List<EmbeddingMatch<TextSegment>> relevant = result.matches();
|
||||||
boolean exists = CollectionUtils.isNotEmpty(relevant);
|
boolean exists = CollectionUtils.isNotEmpty(relevant);
|
||||||
cache.put(queryId, exists);
|
cache.put(collectionName + queryId, exists);
|
||||||
return exists;
|
return exists;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -72,7 +72,10 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner {
|
|||||||
embeddingService.retrieveQuery(collection, retrieveQuery, num);
|
embeddingService.retrieveQuery(collection, retrieveQuery, num);
|
||||||
results.forEach(ret -> {
|
results.forEach(ret -> {
|
||||||
ret.getRetrieval().forEach(r -> {
|
ret.getRetrieval().forEach(r -> {
|
||||||
exemplars.add(JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class));
|
Text2SQLExemplar tmp = // 传递相似度,可以作为样本筛选的依据
|
||||||
|
JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class);
|
||||||
|
tmp.setSimilarity(r.getSimilarity());
|
||||||
|
exemplars.add(tmp);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import org.springframework.util.CollectionUtils;
|
|||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class SystemConfigServiceImpl extends ServiceImpl<SystemConfigMapper, SystemConfigDO>
|
public class SystemConfigServiceImpl extends ServiceImpl<SystemConfigMapper, SystemConfigDO>
|
||||||
@@ -38,8 +39,8 @@ public class SystemConfigServiceImpl extends ServiceImpl<SystemConfigMapper, Sys
|
|||||||
return systemConfigDb;
|
return systemConfigDb;
|
||||||
}
|
}
|
||||||
|
|
||||||
private SystemConfig getSystemConfigFromDB() {
|
private SystemConfig getSystemConfigFromDB() { // 加上id ,如果有多条记录,会出错
|
||||||
List<SystemConfigDO> list = list();
|
List<SystemConfigDO> list = this.lambdaQuery().eq(SystemConfigDO::getId, 1).list();
|
||||||
if (CollectionUtils.isEmpty(list)) {
|
if (CollectionUtils.isEmpty(list)) {
|
||||||
SystemConfig systemConfig = new SystemConfig();
|
SystemConfig systemConfig = new SystemConfig();
|
||||||
systemConfig.setId(1);
|
systemConfig.setId(1);
|
||||||
|
|||||||
@@ -242,10 +242,8 @@ public class DateModeUtils {
|
|||||||
return String.format("%s >= '%s' and %s <= '%s'", dateField,
|
return String.format("%s >= '%s' and %s <= '%s'", dateField,
|
||||||
dateInfo.getStartDate(), dateField, dateInfo.getEndDate());
|
dateInfo.getStartDate(), dateField, dateInfo.getEndDate());
|
||||||
}
|
}
|
||||||
LocalDate endData =
|
LocalDate endData = DateUtils.parseDate(dateInfo.getEndDate());
|
||||||
LocalDate.parse(dateInfo.getEndDate(), DateTimeFormatter.ofPattern(DAY_FORMAT));
|
LocalDate startData = DateUtils.parseDate(dateInfo.getStartDate());
|
||||||
LocalDate startData = LocalDate.parse(dateInfo.getStartDate(),
|
|
||||||
DateTimeFormatter.ofPattern(DAY_FORMAT));
|
|
||||||
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(MONTH_FORMAT);
|
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(MONTH_FORMAT);
|
||||||
return String.format("%s >= '%s' and %s <= '%s'", dateField,
|
return String.format("%s >= '%s' and %s <= '%s'", dateField,
|
||||||
startData.format(formatter), dateField, endData.format(formatter));
|
startData.format(formatter), dateField, endData.format(formatter));
|
||||||
@@ -320,7 +318,7 @@ public class DateModeUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public String getDateWhereStr(DateConf dateInfo, ItemDateResp dateDate) {
|
public String getDateWhereStr(DateConf dateInfo, ItemDateResp dateDate) {
|
||||||
if (Objects.isNull(dateInfo)) {
|
if (Objects.isNull(dateInfo) || Objects.isNull(dateInfo.getDateField())) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
String dateStr = "";
|
String dateStr = "";
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ public class DateUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static String getBeforeDate(String currentDate, DatePeriodEnum datePeriodEnum) {
|
public static String getBeforeDate(String currentDate, DatePeriodEnum datePeriodEnum) {
|
||||||
LocalDate specifiedDate = LocalDate.parse(currentDate, DEFAULT_DATE_FORMATTER2);
|
LocalDate specifiedDate = parseDate(currentDate);
|
||||||
LocalDate startDate;
|
LocalDate startDate;
|
||||||
switch (datePeriodEnum) {
|
switch (datePeriodEnum) {
|
||||||
case MONTH:
|
case MONTH:
|
||||||
@@ -93,7 +93,7 @@ public class DateUtils {
|
|||||||
|
|
||||||
public static String getBeforeDate(String currentDate, int intervalDay,
|
public static String getBeforeDate(String currentDate, int intervalDay,
|
||||||
DatePeriodEnum datePeriodEnum) {
|
DatePeriodEnum datePeriodEnum) {
|
||||||
LocalDate specifiedDate = LocalDate.parse(currentDate, DEFAULT_DATE_FORMATTER2);
|
LocalDate specifiedDate = parseDate(currentDate);
|
||||||
LocalDate result = null;
|
LocalDate result = null;
|
||||||
switch (datePeriodEnum) {
|
switch (datePeriodEnum) {
|
||||||
case DAY:
|
case DAY:
|
||||||
@@ -161,11 +161,25 @@ public class DateUtils {
|
|||||||
return !timeString.equals("00:00:00");
|
return !timeString.equals("00:00:00");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static LocalDate parseDate(String timeString) {
|
||||||
|
DateTimeFormatter[] dateFormatters =
|
||||||
|
{DateTimeFormatter.ofPattern("yyyyMMdd"), DateTimeFormatter.ofPattern("yyyy-MM-dd"),
|
||||||
|
DateTimeFormatter.ofPattern("yyyy/MM/dd"),
|
||||||
|
DateTimeFormatter.ofPattern("yyyy-MM")};
|
||||||
|
for (DateTimeFormatter formatter : dateFormatters) {
|
||||||
|
try {
|
||||||
|
return LocalDate.parse(timeString, formatter);
|
||||||
|
} catch (DateTimeParseException ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
public static List<String> getDateList(String startDateStr, String endDateStr,
|
public static List<String> getDateList(String startDateStr, String endDateStr,
|
||||||
DatePeriodEnum period) {
|
DatePeriodEnum period) {
|
||||||
try {
|
try {
|
||||||
LocalDate startDate = LocalDate.parse(startDateStr);
|
LocalDate startDate = parseDate(startDateStr);
|
||||||
LocalDate endDate = LocalDate.parse(endDateStr);
|
LocalDate endDate = parseDate(endDateStr);
|
||||||
List<String> datesInRange = new ArrayList<>();
|
List<String> datesInRange = new ArrayList<>();
|
||||||
LocalDate currentDate = startDate;
|
LocalDate currentDate = startDate;
|
||||||
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM");
|
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM");
|
||||||
@@ -189,7 +203,7 @@ public class DateUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static boolean isAnyDateString(String value) {
|
public static boolean isAnyDateString(String value) {
|
||||||
List<String> formats = Arrays.asList("yyyy-MM-dd", "yyyy-MM", "yyyy/MM/dd");
|
List<String> formats = Arrays.asList("yyyy-MM-dd", "yyyy-MM", "yyyy/MM/dd", "yyyyMMdd");
|
||||||
return isAnyDateString(value, formats);
|
return isAnyDateString(value, formats);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
package dev.langchain4j.dashscope.spring;
|
|
||||||
|
|
||||||
import lombok.Getter;
|
|
||||||
import lombok.Setter;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Getter
|
|
||||||
@Setter
|
|
||||||
class ChatModelProperties {
|
|
||||||
|
|
||||||
String baseUrl;
|
|
||||||
String apiKey;
|
|
||||||
String modelName;
|
|
||||||
Double topP;
|
|
||||||
Integer topK;
|
|
||||||
Boolean enableSearch;
|
|
||||||
Integer seed;
|
|
||||||
Float repetitionPenalty;
|
|
||||||
Float temperature;
|
|
||||||
List<String> stops;
|
|
||||||
Integer maxTokens;
|
|
||||||
}
|
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
package dev.langchain4j.dashscope.spring;
|
|
||||||
|
|
||||||
import dev.langchain4j.model.dashscope.QwenChatModel;
|
|
||||||
import dev.langchain4j.model.dashscope.QwenEmbeddingModel;
|
|
||||||
import dev.langchain4j.model.dashscope.QwenLanguageModel;
|
|
||||||
import dev.langchain4j.model.dashscope.QwenStreamingChatModel;
|
|
||||||
import dev.langchain4j.model.dashscope.QwenStreamingLanguageModel;
|
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
|
||||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
|
||||||
import org.springframework.context.annotation.Bean;
|
|
||||||
import org.springframework.context.annotation.Configuration;
|
|
||||||
|
|
||||||
import static dev.langchain4j.dashscope.spring.Properties.PREFIX;
|
|
||||||
|
|
||||||
@Configuration
|
|
||||||
@EnableConfigurationProperties(Properties.class)
|
|
||||||
public class DashscopeAutoConfig {
|
|
||||||
|
|
||||||
@Bean
|
|
||||||
@ConditionalOnProperty(PREFIX + ".chat-model.api-key")
|
|
||||||
QwenChatModel qwenChatModel(Properties properties) {
|
|
||||||
ChatModelProperties chatModelProperties = properties.getChatModel();
|
|
||||||
return QwenChatModel.builder().baseUrl(chatModelProperties.getBaseUrl())
|
|
||||||
.apiKey(chatModelProperties.getApiKey())
|
|
||||||
.modelName(chatModelProperties.getModelName()).topP(chatModelProperties.getTopP())
|
|
||||||
.topK(chatModelProperties.getTopK())
|
|
||||||
.enableSearch(chatModelProperties.getEnableSearch())
|
|
||||||
.seed(chatModelProperties.getSeed())
|
|
||||||
.repetitionPenalty(chatModelProperties.getRepetitionPenalty())
|
|
||||||
.temperature(chatModelProperties.getTemperature())
|
|
||||||
.stops(chatModelProperties.getStops()).maxTokens(chatModelProperties.getMaxTokens())
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Bean
|
|
||||||
@ConditionalOnProperty(PREFIX + ".streaming-chat-model.api-key")
|
|
||||||
QwenStreamingChatModel qwenStreamingChatModel(Properties properties) {
|
|
||||||
ChatModelProperties chatModelProperties = properties.getStreamingChatModel();
|
|
||||||
return QwenStreamingChatModel.builder().baseUrl(chatModelProperties.getBaseUrl())
|
|
||||||
.apiKey(chatModelProperties.getApiKey())
|
|
||||||
.modelName(chatModelProperties.getModelName()).topP(chatModelProperties.getTopP())
|
|
||||||
.topK(chatModelProperties.getTopK())
|
|
||||||
.enableSearch(chatModelProperties.getEnableSearch())
|
|
||||||
.seed(chatModelProperties.getSeed())
|
|
||||||
.repetitionPenalty(chatModelProperties.getRepetitionPenalty())
|
|
||||||
.temperature(chatModelProperties.getTemperature())
|
|
||||||
.stops(chatModelProperties.getStops()).maxTokens(chatModelProperties.getMaxTokens())
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Bean
|
|
||||||
@ConditionalOnProperty(PREFIX + ".language-model.api-key")
|
|
||||||
QwenLanguageModel qwenLanguageModel(Properties properties) {
|
|
||||||
ChatModelProperties languageModel = properties.getLanguageModel();
|
|
||||||
return QwenLanguageModel.builder().baseUrl(languageModel.getBaseUrl())
|
|
||||||
.apiKey(languageModel.getApiKey()).modelName(languageModel.getModelName())
|
|
||||||
.topP(languageModel.getTopP()).topK(languageModel.getTopK())
|
|
||||||
.enableSearch(languageModel.getEnableSearch()).seed(languageModel.getSeed())
|
|
||||||
.repetitionPenalty(languageModel.getRepetitionPenalty())
|
|
||||||
.temperature(languageModel.getTemperature()).stops(languageModel.getStops())
|
|
||||||
.maxTokens(languageModel.getMaxTokens()).build();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Bean
|
|
||||||
@ConditionalOnProperty(PREFIX + ".streaming-language-model.api-key")
|
|
||||||
QwenStreamingLanguageModel qwenStreamingLanguageModel(Properties properties) {
|
|
||||||
ChatModelProperties languageModel = properties.getStreamingLanguageModel();
|
|
||||||
return QwenStreamingLanguageModel.builder().baseUrl(languageModel.getBaseUrl())
|
|
||||||
.apiKey(languageModel.getApiKey()).modelName(languageModel.getModelName())
|
|
||||||
.topP(languageModel.getTopP()).topK(languageModel.getTopK())
|
|
||||||
.enableSearch(languageModel.getEnableSearch()).seed(languageModel.getSeed())
|
|
||||||
.repetitionPenalty(languageModel.getRepetitionPenalty())
|
|
||||||
.temperature(languageModel.getTemperature()).stops(languageModel.getStops())
|
|
||||||
.maxTokens(languageModel.getMaxTokens()).build();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Bean
|
|
||||||
@ConditionalOnProperty(PREFIX + ".embedding-model.api-key")
|
|
||||||
QwenEmbeddingModel qwenEmbeddingModel(Properties properties) {
|
|
||||||
EmbeddingModelProperties embeddingModelProperties = properties.getEmbeddingModel();
|
|
||||||
return QwenEmbeddingModel.builder().apiKey(embeddingModelProperties.getApiKey())
|
|
||||||
.modelName(embeddingModelProperties.getModelName()).build();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
package dev.langchain4j.dashscope.spring;
|
|
||||||
|
|
||||||
import lombok.Getter;
|
|
||||||
import lombok.Setter;
|
|
||||||
|
|
||||||
@Getter
|
|
||||||
@Setter
|
|
||||||
class EmbeddingModelProperties {
|
|
||||||
|
|
||||||
private String apiKey;
|
|
||||||
private String modelName;
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package dev.langchain4j.dashscope.spring;
|
|
||||||
|
|
||||||
import lombok.Getter;
|
|
||||||
import lombok.Setter;
|
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
|
||||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
|
||||||
|
|
||||||
@Getter
|
|
||||||
@Setter
|
|
||||||
@ConfigurationProperties(prefix = Properties.PREFIX)
|
|
||||||
public class Properties {
|
|
||||||
|
|
||||||
static final String PREFIX = "langchain4j.dashscope";
|
|
||||||
|
|
||||||
@NestedConfigurationProperty
|
|
||||||
ChatModelProperties chatModel;
|
|
||||||
|
|
||||||
@NestedConfigurationProperty
|
|
||||||
ChatModelProperties streamingChatModel;
|
|
||||||
|
|
||||||
@NestedConfigurationProperty
|
|
||||||
ChatModelProperties languageModel;
|
|
||||||
|
|
||||||
@NestedConfigurationProperty
|
|
||||||
ChatModelProperties streamingLanguageModel;
|
|
||||||
|
|
||||||
@NestedConfigurationProperty
|
|
||||||
EmbeddingModelProperties embeddingModel;
|
|
||||||
}
|
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
package dev.langchain4j.inmemory.spring;
|
package dev.langchain4j.inmemory.spring;
|
||||||
|
|
||||||
import dev.langchain4j.model.embedding.AllMiniLmL6V2QuantizedEmbeddingModel;
|
|
||||||
import dev.langchain4j.model.embedding.BgeSmallZhEmbeddingModel;
|
|
||||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||||
import dev.langchain4j.model.embedding.S2OnnxEmbeddingModel;
|
import dev.langchain4j.model.embedding.S2OnnxEmbeddingModel;
|
||||||
|
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
|
||||||
|
import dev.langchain4j.model.embedding.onnx.bgesmallzh.BgeSmallZhEmbeddingModel;
|
||||||
import dev.langchain4j.provider.EmbeddingModelConstant;
|
import dev.langchain4j.provider.EmbeddingModelConstant;
|
||||||
import dev.langchain4j.store.embedding.EmbeddingStoreFactory;
|
import dev.langchain4j.store.embedding.EmbeddingStoreFactory;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import dev.langchain4j.data.message.ChatMessage;
|
|||||||
import dev.langchain4j.model.chat.ChatLanguageModel;
|
import dev.langchain4j.model.chat.ChatLanguageModel;
|
||||||
import dev.langchain4j.model.output.Response;
|
import dev.langchain4j.model.output.Response;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@@ -32,6 +33,7 @@ public class DifyAiChatModel implements ChatLanguageModel {
|
|||||||
private final Double temperature;
|
private final Double temperature;
|
||||||
private final Long timeOut;
|
private final Long timeOut;
|
||||||
|
|
||||||
|
@Setter
|
||||||
private String userName;
|
private String userName;
|
||||||
|
|
||||||
@Builder
|
@Builder
|
||||||
@@ -54,7 +56,7 @@ public class DifyAiChatModel implements ChatLanguageModel {
|
|||||||
@Override
|
@Override
|
||||||
public String generate(String message) {
|
public String generate(String message) {
|
||||||
DifyResult difyResult = this.difyClient.generate(message, this.getUserName());
|
DifyResult difyResult = this.difyClient.generate(message, this.getUserName());
|
||||||
return difyResult.getAnswer().toString();
|
return difyResult.getAnswer();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -67,7 +69,7 @@ public class DifyAiChatModel implements ChatLanguageModel {
|
|||||||
List<ToolSpecification> toolSpecifications) {
|
List<ToolSpecification> toolSpecifications) {
|
||||||
ensureNotEmpty(messages, "messages");
|
ensureNotEmpty(messages, "messages");
|
||||||
DifyResult difyResult =
|
DifyResult difyResult =
|
||||||
this.difyClient.generate(messages.get(0).text(), this.getUserName());
|
this.difyClient.generate(messages.get(0).toString(), this.getUserName());
|
||||||
System.out.println(difyResult.toString());
|
System.out.println(difyResult.toString());
|
||||||
|
|
||||||
if (!isNullOrEmpty(toolSpecifications)) {
|
if (!isNullOrEmpty(toolSpecifications)) {
|
||||||
@@ -84,12 +86,8 @@ public class DifyAiChatModel implements ChatLanguageModel {
|
|||||||
toolSpecification != null ? singletonList(toolSpecification) : null);
|
toolSpecification != null ? singletonList(toolSpecification) : null);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setUserName(String userName) {
|
|
||||||
this.userName = userName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getUserName() {
|
public String getUserName() {
|
||||||
return null == userName ? "zhaodongsheng" : userName;
|
return null == userName ? "admin" : userName;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
package dev.langchain4j.model.embedding;
|
package dev.langchain4j.model.embedding;
|
||||||
|
|
||||||
|
import dev.langchain4j.model.embedding.onnx.AbstractInProcessEmbeddingModel;
|
||||||
|
import dev.langchain4j.model.embedding.onnx.OnnxBertBiEncoder;
|
||||||
|
import dev.langchain4j.model.embedding.onnx.PoolingMode;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -9,6 +12,7 @@ import java.nio.file.Files;
|
|||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An embedding model that runs within your Java application's process. Any BERT-based model (e.g.,
|
* An embedding model that runs within your Java application's process. Any BERT-based model (e.g.,
|
||||||
@@ -25,6 +29,7 @@ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel {
|
|||||||
private static volatile String cachedVocabularyPath;
|
private static volatile String cachedVocabularyPath;
|
||||||
|
|
||||||
public S2OnnxEmbeddingModel(String pathToModel, String vocabularyPath) {
|
public S2OnnxEmbeddingModel(String pathToModel, String vocabularyPath) {
|
||||||
|
super(Executors.newSingleThreadExecutor());
|
||||||
if (shouldReloadModel(pathToModel, vocabularyPath)) {
|
if (shouldReloadModel(pathToModel, vocabularyPath)) {
|
||||||
synchronized (S2OnnxEmbeddingModel.class) {
|
synchronized (S2OnnxEmbeddingModel.class) {
|
||||||
if (shouldReloadModel(pathToModel, vocabularyPath)) {
|
if (shouldReloadModel(pathToModel, vocabularyPath)) {
|
||||||
@@ -61,8 +66,8 @@ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel {
|
|||||||
|
|
||||||
static OnnxBertBiEncoder loadFromFileSystem(Path pathToModel, URL vocabularyFile) {
|
static OnnxBertBiEncoder loadFromFileSystem(Path pathToModel, URL vocabularyFile) {
|
||||||
try {
|
try {
|
||||||
return new OnnxBertBiEncoder(Files.newInputStream(pathToModel), vocabularyFile,
|
return new OnnxBertBiEncoder(Files.newInputStream(pathToModel),
|
||||||
PoolingMode.MEAN);
|
vocabularyFile.openStream(), PoolingMode.MEAN);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ import static dev.langchain4j.model.openai.InternalOpenAiHelper.toOpenAiMessages
|
|||||||
import static dev.langchain4j.model.openai.InternalOpenAiHelper.toOpenAiResponseFormat;
|
import static dev.langchain4j.model.openai.InternalOpenAiHelper.toOpenAiResponseFormat;
|
||||||
import static dev.langchain4j.model.openai.InternalOpenAiHelper.toTools;
|
import static dev.langchain4j.model.openai.InternalOpenAiHelper.toTools;
|
||||||
import static dev.langchain4j.model.openai.InternalOpenAiHelper.tokenUsageFrom;
|
import static dev.langchain4j.model.openai.InternalOpenAiHelper.tokenUsageFrom;
|
||||||
import static dev.langchain4j.model.openai.OpenAiModelName.GPT_3_5_TURBO;
|
import static dev.langchain4j.model.openai.OpenAiChatModelName.GPT_3_5_TURBO;
|
||||||
import static dev.langchain4j.spi.ServiceHelper.loadFactories;
|
import static dev.langchain4j.spi.ServiceHelper.loadFactories;
|
||||||
import static java.time.Duration.ofSeconds;
|
import static java.time.Duration.ofSeconds;
|
||||||
import static java.util.Collections.emptyList;
|
import static java.util.Collections.emptyList;
|
||||||
@@ -66,7 +66,6 @@ import static java.util.Collections.singletonList;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
|
public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
|
||||||
|
|
||||||
public static final String ZHIPU = "bigmodel";
|
|
||||||
private final OpenAiClient client;
|
private final OpenAiClient client;
|
||||||
private final String baseUrl;
|
private final String baseUrl;
|
||||||
private final String modelName;
|
private final String modelName;
|
||||||
@@ -111,7 +110,7 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
|
|||||||
.connectTimeout(timeout).readTimeout(timeout).writeTimeout(timeout).proxy(proxy)
|
.connectTimeout(timeout).readTimeout(timeout).writeTimeout(timeout).proxy(proxy)
|
||||||
.logRequests(logRequests).logResponses(logResponses).userAgent(DEFAULT_USER_AGENT)
|
.logRequests(logRequests).logResponses(logResponses).userAgent(DEFAULT_USER_AGENT)
|
||||||
.customHeaders(customHeaders).build();
|
.customHeaders(customHeaders).build();
|
||||||
this.modelName = getOrDefault(modelName, GPT_3_5_TURBO);
|
this.modelName = getOrDefault(modelName, GPT_3_5_TURBO.name());
|
||||||
this.apiVersion = apiVersion;
|
this.apiVersion = apiVersion;
|
||||||
this.temperature = getOrDefault(temperature, 0.7);
|
this.temperature = getOrDefault(temperature, 0.7);
|
||||||
this.topP = topP;
|
this.topP = topP;
|
||||||
@@ -130,7 +129,7 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
|
|||||||
this.strictTools = getOrDefault(strictTools, false);
|
this.strictTools = getOrDefault(strictTools, false);
|
||||||
this.parallelToolCalls = parallelToolCalls;
|
this.parallelToolCalls = parallelToolCalls;
|
||||||
this.maxRetries = getOrDefault(maxRetries, 3);
|
this.maxRetries = getOrDefault(maxRetries, 3);
|
||||||
this.tokenizer = getOrDefault(tokenizer, OpenAiTokenizer::new);
|
this.tokenizer = getOrDefault(tokenizer, () -> new OpenAiTokenizer(this.modelName));
|
||||||
this.listeners = listeners == null ? emptyList() : new ArrayList<>(listeners);
|
this.listeners = listeners == null ? emptyList() : new ArrayList<>(listeners);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,9 +191,7 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
|
|||||||
.responseFormat(responseFormat).seed(seed).user(user)
|
.responseFormat(responseFormat).seed(seed).user(user)
|
||||||
.parallelToolCalls(parallelToolCalls);
|
.parallelToolCalls(parallelToolCalls);
|
||||||
|
|
||||||
if (!(baseUrl.contains(ZHIPU))) {
|
requestBuilder.temperature(temperature);
|
||||||
requestBuilder.temperature(temperature);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (toolSpecifications != null && !toolSpecifications.isEmpty()) {
|
if (toolSpecifications != null && !toolSpecifications.isEmpty()) {
|
||||||
requestBuilder.tools(toTools(toolSpecifications, strictTools));
|
requestBuilder.tools(toTools(toolSpecifications, strictTools));
|
||||||
|
|||||||
@@ -1,16 +0,0 @@
|
|||||||
package dev.langchain4j.model.zhipu;
|
|
||||||
|
|
||||||
public enum ChatCompletionModel {
|
|
||||||
GLM_4("glm-4"), GLM_3_TURBO("glm-3-turbo"), CHATGLM_TURBO("chatglm_turbo");
|
|
||||||
|
|
||||||
private final String value;
|
|
||||||
|
|
||||||
ChatCompletionModel(String value) {
|
|
||||||
this.value = value;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
return this.value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
package dev.langchain4j.model.zhipu;
|
|
||||||
|
|
||||||
import dev.langchain4j.agent.tool.ToolSpecification;
|
|
||||||
import dev.langchain4j.data.message.AiMessage;
|
|
||||||
import dev.langchain4j.data.message.ChatMessage;
|
|
||||||
import dev.langchain4j.model.chat.ChatLanguageModel;
|
|
||||||
import dev.langchain4j.model.output.Response;
|
|
||||||
import dev.langchain4j.model.zhipu.chat.ChatCompletionRequest;
|
|
||||||
import dev.langchain4j.model.zhipu.chat.ChatCompletionResponse;
|
|
||||||
import dev.langchain4j.model.zhipu.spi.ZhipuAiChatModelBuilderFactory;
|
|
||||||
import lombok.Builder;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static dev.langchain4j.internal.RetryUtils.withRetry;
|
|
||||||
import static dev.langchain4j.internal.Utils.getOrDefault;
|
|
||||||
import static dev.langchain4j.internal.Utils.isNullOrEmpty;
|
|
||||||
import static dev.langchain4j.internal.ValidationUtils.ensureNotEmpty;
|
|
||||||
import static dev.langchain4j.model.zhipu.DefaultZhipuAiHelper.aiMessageFrom;
|
|
||||||
import static dev.langchain4j.model.zhipu.DefaultZhipuAiHelper.finishReasonFrom;
|
|
||||||
import static dev.langchain4j.model.zhipu.DefaultZhipuAiHelper.toTools;
|
|
||||||
import static dev.langchain4j.model.zhipu.DefaultZhipuAiHelper.toZhipuAiMessages;
|
|
||||||
import static dev.langchain4j.model.zhipu.DefaultZhipuAiHelper.tokenUsageFrom;
|
|
||||||
import static dev.langchain4j.model.zhipu.chat.ToolChoiceMode.AUTO;
|
|
||||||
import static dev.langchain4j.spi.ServiceHelper.loadFactories;
|
|
||||||
import static java.util.Collections.singletonList;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents an ZhipuAi language model with a chat completion interface, such as glm-3-turbo and
|
|
||||||
* glm-4. You can find description of parameters
|
|
||||||
* <a href="https://open.bigmodel.cn/dev/api">here</a>.
|
|
||||||
*/
|
|
||||||
public class ZhipuAiChatModel implements ChatLanguageModel {
|
|
||||||
|
|
||||||
private final String baseUrl;
|
|
||||||
private final Double temperature;
|
|
||||||
private final Double topP;
|
|
||||||
private final String model;
|
|
||||||
private final Integer maxRetries;
|
|
||||||
private final Integer maxToken;
|
|
||||||
private final ZhipuAiClient client;
|
|
||||||
|
|
||||||
@Builder
|
|
||||||
public ZhipuAiChatModel(String baseUrl, String apiKey, Double temperature, Double topP,
|
|
||||||
String model, Integer maxRetries, Integer maxToken, Boolean logRequests,
|
|
||||||
Boolean logResponses) {
|
|
||||||
this.baseUrl = getOrDefault(baseUrl, "https://open.bigmodel.cn/");
|
|
||||||
this.temperature = getOrDefault(temperature, 0.7);
|
|
||||||
this.topP = topP;
|
|
||||||
this.model = getOrDefault(model, ChatCompletionModel.GLM_4.toString());
|
|
||||||
this.maxRetries = getOrDefault(maxRetries, 3);
|
|
||||||
this.maxToken = getOrDefault(maxToken, 512);
|
|
||||||
this.client = ZhipuAiClient.builder().baseUrl(this.baseUrl).apiKey(apiKey)
|
|
||||||
.logRequests(getOrDefault(logRequests, false))
|
|
||||||
.logResponses(getOrDefault(logResponses, false)).build();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ZhipuAiChatModelBuilder builder() {
|
|
||||||
for (ZhipuAiChatModelBuilderFactory factories : loadFactories(
|
|
||||||
ZhipuAiChatModelBuilderFactory.class)) {
|
|
||||||
return factories.get();
|
|
||||||
}
|
|
||||||
return new ZhipuAiChatModelBuilder();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Response<AiMessage> generate(List<ChatMessage> messages) {
|
|
||||||
return generate(messages, (ToolSpecification) null);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Response<AiMessage> generate(List<ChatMessage> messages,
|
|
||||||
List<ToolSpecification> toolSpecifications) {
|
|
||||||
ensureNotEmpty(messages, "messages");
|
|
||||||
|
|
||||||
ChatCompletionRequest.Builder requestBuilder =
|
|
||||||
ChatCompletionRequest.builder().model(this.model).maxTokens(maxToken).stream(false)
|
|
||||||
.topP(topP).toolChoice(AUTO).messages(toZhipuAiMessages(messages));
|
|
||||||
|
|
||||||
if (!isNullOrEmpty(toolSpecifications)) {
|
|
||||||
requestBuilder.tools(toTools(toolSpecifications));
|
|
||||||
}
|
|
||||||
|
|
||||||
ChatCompletionResponse response =
|
|
||||||
withRetry(() -> client.chatCompletion(requestBuilder.build()), maxRetries);
|
|
||||||
return Response.from(aiMessageFrom(response), tokenUsageFrom(response.getUsage()),
|
|
||||||
finishReasonFrom(response.getChoices().get(0).getFinishReason()));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Response<AiMessage> generate(List<ChatMessage> messages,
|
|
||||||
ToolSpecification toolSpecification) {
|
|
||||||
return generate(messages,
|
|
||||||
toolSpecification != null ? singletonList(toolSpecification) : null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class ZhipuAiChatModelBuilder {
|
|
||||||
public ZhipuAiChatModelBuilder() {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user