mirror of
https://github.com/tencentmusic/supersonic.git
synced 2026-04-20 13:44:19 +08:00
Compare commits
219 Commits
v0.9.8
...
be2e380b4c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be2e380b4c | ||
|
|
eeaebe06aa | ||
|
|
4ab9cd715d | ||
|
|
fd306db3fe | ||
|
|
30adaa3f20 | ||
|
|
cc66ebd684 | ||
|
|
ef161fe1f2 | ||
|
|
0417f12324 | ||
|
|
d294fec2a0 | ||
|
|
de92b357df | ||
|
|
be5eeae707 | ||
|
|
65f0096724 | ||
|
|
416488b919 | ||
|
|
8900fde4a3 | ||
|
|
2b28aeea6f | ||
|
|
329756056c | ||
|
|
715adb5260 | ||
|
|
da6d28c18c | ||
|
|
cd863705a4 | ||
|
|
f264b3160f | ||
|
|
61e22c2104 | ||
|
|
6e4260f9f1 | ||
|
|
ab74acc84a | ||
|
|
4e653c1fb1 | ||
|
|
6fcfdc15e1 | ||
|
|
4738b9d01b | ||
|
|
e0f7ec0f40 | ||
|
|
7b46ef64fd | ||
|
|
6af345044a | ||
|
|
4821f31e15 | ||
|
|
be59b051fc | ||
|
|
83cb6967e7 | ||
|
|
22f6190e7c | ||
|
|
43140e695b | ||
|
|
a46e89af56 | ||
|
|
a2f54d4c80 | ||
|
|
24eaffb2d5 | ||
|
|
739514ddfa | ||
|
|
6486257c9e | ||
|
|
6f5e477e3c | ||
|
|
683f01c33b | ||
|
|
3e1e5ae209 | ||
|
|
0612833618 | ||
|
|
7145f27671 | ||
|
|
920d8f280a | ||
|
|
ade03627ce | ||
|
|
a23d1071a3 | ||
|
|
ce9ae1c0c1 | ||
|
|
d834e98a66 | ||
|
|
94267f6028 | ||
|
|
a4d2df4063 | ||
|
|
8e03531424 | ||
|
|
16600ed6f0 | ||
|
|
d04a086c88 | ||
|
|
68963b9ec9 | ||
|
|
265e51c429 | ||
|
|
d40400d2a4 | ||
|
|
c483bb891a | ||
|
|
6738aba19e | ||
|
|
493a8035cd | ||
|
|
b425c49c5b | ||
|
|
4dca6eec5a | ||
|
|
642d6a02e1 | ||
|
|
5de5b0a5e2 | ||
|
|
214d90772d | ||
|
|
d8b8c4e6b9 | ||
|
|
10a510409f | ||
|
|
19c1d306df | ||
|
|
8c6ae62522 | ||
|
|
758d170bbf | ||
|
|
7dc013dfb3 | ||
|
|
72780f9acf | ||
|
|
7b49412bde | ||
|
|
9f63aca132 | ||
|
|
933b54085c | ||
|
|
f7fce0217f | ||
|
|
c2d155705f | ||
|
|
5faf5f3ac4 | ||
|
|
d88d8b3beb | ||
|
|
caf03588c6 | ||
|
|
4cb2256351 | ||
|
|
8b69d57c4b | ||
|
|
91856ddebd | ||
|
|
94e97c9a1d | ||
|
|
9faa858c22 | ||
|
|
6fcd105249 | ||
|
|
1d0f5612b7 | ||
|
|
b57eed47e2 | ||
|
|
aa3b8997bd | ||
|
|
84944fa341 | ||
|
|
b4c19533a4 | ||
|
|
831fbfe475 | ||
|
|
62b64c74a3 | ||
|
|
ed5c129a4a | ||
|
|
14087825df | ||
|
|
f9cc395e49 | ||
|
|
d014a27aff | ||
|
|
2e28a4c7a5 | ||
|
|
9f8793bfe2 | ||
|
|
a8868ba9ab | ||
|
|
8c44c9f42f | ||
|
|
f97ac1da83 | ||
|
|
07f6be51c7 | ||
|
|
4062a13126 | ||
|
|
f6622319a4 | ||
|
|
2567742115 | ||
|
|
215391bb2d | ||
|
|
4a6938956b | ||
|
|
e586983ff1 | ||
|
|
a12486591b | ||
|
|
848b4a1e44 | ||
|
|
a298c670ed | ||
|
|
f2f7caa32c | ||
|
|
3db9a0dcec | ||
|
|
09ea5db0ba | ||
|
|
8000ee5237 | ||
|
|
e55f43c737 | ||
|
|
43ba1a0ba6 | ||
|
|
4bd521020a | ||
|
|
aa400176a5 | ||
|
|
c98c5ade9e | ||
|
|
28d5f38ffb | ||
|
|
9e24fd04a5 | ||
|
|
02c6a7d404 | ||
|
|
fe57a4e5cf | ||
|
|
79ab1be306 | ||
|
|
3e903b862e | ||
|
|
0ce79cbfc0 | ||
|
|
cf79ac9ece | ||
|
|
02f0063846 | ||
|
|
b8924ed45e | ||
|
|
27c6b8ecb1 | ||
|
|
7de3662259 | ||
|
|
ba4d92e11c | ||
|
|
350b6089dc | ||
|
|
350567755a | ||
|
|
efac1efdb0 | ||
|
|
4592131b55 | ||
|
|
b3b1498af7 | ||
|
|
83cfae609e | ||
|
|
0fc29304a8 | ||
|
|
89d2ac802d | ||
|
|
dee9b119eb | ||
|
|
a232c2bc4f | ||
|
|
0f1c50167d | ||
|
|
1c73453c5f | ||
|
|
639d1a78da | ||
|
|
5837a5b3ae | ||
|
|
6ecc5a9362 | ||
|
|
8299084c95 | ||
|
|
bb60c93824 | ||
|
|
d22a7d1cc6 | ||
|
|
4358ffe4a1 | ||
|
|
82c63a7f22 | ||
|
|
50accb152a | ||
|
|
66ed711416 | ||
|
|
4076d37b9b | ||
|
|
67377bc7dc | ||
|
|
593597fe26 | ||
|
|
a058dc8b6e | ||
|
|
13d4fc3feb | ||
|
|
b4669cf110 | ||
|
|
c80794e8fc | ||
|
|
111304486b | ||
|
|
25559fdaa5 | ||
|
|
dad065d0ba | ||
|
|
7bf1ba09c5 | ||
|
|
97867ca015 | ||
|
|
9bccbae3bc | ||
|
|
224c114d20 | ||
|
|
1de999dc1d | ||
|
|
2eca2d1c14 | ||
|
|
40bfcdce2c | ||
|
|
b84dde3799 | ||
|
|
91b16f95ff | ||
|
|
860fd5d299 | ||
|
|
c22e3ef2e8 | ||
|
|
296ce5cc55 | ||
|
|
722f40cdf7 | ||
|
|
0edadd01eb | ||
|
|
a942132b83 | ||
|
|
cb183b7ac8 | ||
|
|
244052e806 | ||
|
|
46d64d78f3 | ||
|
|
62fc2dd18a | ||
|
|
be05f977d5 | ||
|
|
d7586a5d3b | ||
|
|
e990b37433 | ||
|
|
eb502e740c | ||
|
|
62a4d60a0b | ||
|
|
8d63ed170a | ||
|
|
f4f0e58bfb | ||
|
|
534da49309 | ||
|
|
8c6da5dc54 | ||
|
|
159d91fd0f | ||
|
|
5a8c20a00b | ||
|
|
cd889b479c | ||
|
|
a0f53359ef | ||
|
|
36d221ab74 | ||
|
|
e8c9855163 | ||
|
|
ba1938f04b | ||
|
|
5e22b412c6 | ||
|
|
87729956e8 | ||
|
|
ea6a9ebc5f | ||
|
|
14a19a901f | ||
|
|
ca4545bb15 | ||
|
|
e0e167fd40 | ||
|
|
d4a9d5a7e6 | ||
|
|
c9c6dc4e44 | ||
|
|
524ec38edc | ||
|
|
9edcb9f91c | ||
|
|
6f2af79756 | ||
|
|
7be885d9c8 | ||
|
|
9a05b5cce6 | ||
|
|
3b65b1c80b | ||
|
|
1e5bf7909e | ||
|
|
6a4458a572 | ||
|
|
1867447b6e | ||
|
|
ff7fb50030 |
69
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
69
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
@@ -11,59 +11,28 @@ body:
|
||||
If it is an idea or help wanted, please go to:
|
||||
[Github Discussion](https://github.com/tencentmusic/supersonic/discussions)
|
||||
|
||||
- type: checkboxes
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [issues](https://github.com/tencentmusic/supersonic/issues?q=is%3Aissue) first to see
|
||||
whether the same issue was reported already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [issues](https://github.com/tencentmusic/supersonic/issues?q=is%3Aissue) and found no similar
|
||||
issues.
|
||||
required: true
|
||||
label: SuperSonic version
|
||||
description: Please tell us which version you are using.
|
||||
placeholder: "0.9.8"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
- type: input
|
||||
id: organization
|
||||
attributes:
|
||||
label: Version
|
||||
description: What is the current version
|
||||
placeholder: >
|
||||
Please provide the version you are using.
|
||||
If it is the trunk version, please input commit id.
|
||||
label: Your organization
|
||||
description: Please tell us your organization so that we can provide you better support and advice.
|
||||
placeholder: "TME..."
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: What's Wrong?
|
||||
description: Describe the bug.
|
||||
placeholder: >
|
||||
Describe the specific problem, the more detailed the better.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: What You Expected?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: How to Reproduce?
|
||||
placeholder: >
|
||||
Please try to give reproducing steps to facilitate quick location of the problem.
|
||||
|
||||
- What actions were performed
|
||||
- Table building statement
|
||||
- Import statement
|
||||
- Cluster information: number of nodes, configuration, etc.
|
||||
|
||||
If it is hard to reproduce, please also explain the general scene.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Anything Else?
|
||||
label: Description
|
||||
description: Describe the bug you met.
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
@@ -74,16 +43,6 @@ body:
|
||||
options:
|
||||
- label: Yes I am willing to submit a PR!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: >
|
||||
I agree to follow this project's
|
||||
[Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
||||
34
.github/ISSUE_TEMPLATE/enhancement_request.yml
vendored
34
.github/ISSUE_TEMPLATE/enhancement_request.yml
vendored
@@ -8,30 +8,20 @@ body:
|
||||
attributes:
|
||||
value: |
|
||||
Thank you very much for your good enhancement for SuperSonic.
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [issues](https://github.com/tencentmusic/supersonic/issues?q=is%3Aissue) first to see
|
||||
whether the same issue was reported already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [issues](https://github.com/tencentmusic/supersonic/issues?q=is%3Aissue) and found no similar
|
||||
issues.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Description
|
||||
description: Describe the enhancement what you want, including motivation if it exists.
|
||||
|
||||
- type: textarea
|
||||
- type: input
|
||||
id: organization
|
||||
attributes:
|
||||
label: Solution
|
||||
placeholder: >
|
||||
Add overview of proposed solution.
|
||||
|
||||
Add related materials like links if they exist.
|
||||
label: Your organization
|
||||
description: Please tell us your organization so that we can provide you better support and advice.
|
||||
placeholder: "TME..."
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
@@ -42,16 +32,6 @@ body:
|
||||
options:
|
||||
- label: Yes I am willing to submit a PR!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: >
|
||||
I agree to follow this project's
|
||||
[Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
||||
40
.github/ISSUE_TEMPLATE/feature_request.yml
vendored
40
.github/ISSUE_TEMPLATE/feature_request.yml
vendored
@@ -8,33 +8,19 @@ body:
|
||||
value: |
|
||||
Thank you very much for your good ideas and suggestions for SuperSonic
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Search before asking
|
||||
description: >
|
||||
Please make sure to search in the [issues](https://github.com/tencentmusic/supersonic/issues?q=is%3Aissue) first to see
|
||||
whether the same issue was reported already.
|
||||
options:
|
||||
- label: >
|
||||
I had searched in the [issues](https://github.com/tencentmusic/supersonic/issues?q=is%3Aissue) and found no similar
|
||||
issues.
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Description
|
||||
description: Describe your ideas and needs.
|
||||
|
||||
- type: textarea
|
||||
- type: input
|
||||
id: organization
|
||||
attributes:
|
||||
label: Use case
|
||||
placeholder: >
|
||||
What problem does this feature mainly solve, or what scenarios it is suitable for.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Related issues
|
||||
description: Is there currently another issue associated with this?
|
||||
label: Your organization
|
||||
description: Please tell us your organization so that we can provide you better support and advice.
|
||||
placeholder: "TME..."
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
@@ -45,16 +31,4 @@ body:
|
||||
options:
|
||||
- label: Yes I am willing to submit a PR!
|
||||
|
||||
- type: checkboxes
|
||||
attributes:
|
||||
label: Code of Conduct
|
||||
description: The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it.
|
||||
options:
|
||||
- label: >
|
||||
I agree to follow this project's
|
||||
[Code of Conduct](https://www.apache.org/foundation/policies/conduct)
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: "Thanks for completing our form!"
|
||||
|
||||
46
.github/ISSUE_TEMPLATE/question_request.yml
vendored
46
.github/ISSUE_TEMPLATE/question_request.yml
vendored
@@ -8,6 +8,7 @@ body:
|
||||
value: |
|
||||
## Ask a Question about SuperSonic
|
||||
Please provide a detailed description of your question or the clarification you seek regarding the SuperSonic project.
|
||||
|
||||
- type: textarea
|
||||
id: describe-question
|
||||
attributes:
|
||||
@@ -16,43 +17,12 @@ body:
|
||||
placeholder: "Type your question here..."
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: additional-context
|
||||
|
||||
- type: input
|
||||
id: organization
|
||||
attributes:
|
||||
label: Provide any additional context or information
|
||||
description: If your question is related to a specific part of the SuperSonic project or if you have already looked through certain documentation, please provide that information here.
|
||||
placeholder: "Add context here..."
|
||||
label: Your organization
|
||||
description: Please tell us your organization so that we can provide you better support and advice.
|
||||
placeholder: "TME..."
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: tried-to-resolve
|
||||
attributes:
|
||||
label: What have you tried to resolve your question
|
||||
description: Let us know what you have done to try and understand or resolve your question. This can help us provide you with the most useful guidance.
|
||||
placeholder: "I've already tried..."
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: Your environment
|
||||
description: Share details about your environment to help us reproduce the issue. Include your operating system, version of SuperSonic, and any other relevant details.
|
||||
placeholder: "OS, SuperSonic version, etc..."
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: screenshots-logs
|
||||
attributes:
|
||||
label: Screenshots or Logs
|
||||
description: If applicable, add screenshots or logs to help explain your problem.
|
||||
placeholder: "Paste your logs or attach screenshots here..."
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: additional-information
|
||||
attributes:
|
||||
label: Additional information
|
||||
description: Add any other context or details you think might be helpful for understanding your question.
|
||||
placeholder: "Any other information..."
|
||||
validations:
|
||||
required: false
|
||||
required: true
|
||||
29
.github/workflows/centos-ci.yml
vendored
29
.github/workflows/centos-ci.yml
vendored
@@ -15,11 +15,17 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
java-version: [8, 11, 21] # 定义要测试的JDK版本
|
||||
java-version: [21] # 定义要测试的JDK版本
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up JDK ${{ matrix.java-version }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
java-version: ${{ matrix.java-version }}
|
||||
distribution: 'adopt'
|
||||
|
||||
- name: Reset DNF repositories
|
||||
run: |
|
||||
cd /etc/yum.repos.d/
|
||||
@@ -29,21 +35,11 @@ jobs:
|
||||
- name: Update DNF package index
|
||||
run: dnf makecache
|
||||
|
||||
- name: Install Java and Maven with retry
|
||||
- name: Install Maven with retry
|
||||
run: |
|
||||
if [ ${{ matrix.java-version }} -eq 8 ]; then
|
||||
for i in {1..5}; do
|
||||
dnf install -y java-1.8.0-openjdk-devel maven && break || sleep 15
|
||||
done
|
||||
elif [ ${{ matrix.java-version }} -eq 11 ]; then
|
||||
for i in {1..5}; do
|
||||
dnf install -y java-11-openjdk-devel maven && break || sleep 15
|
||||
done
|
||||
elif [ ${{ matrix.java-version }} -eq 21 ]; then
|
||||
for i in {1..5}; do
|
||||
dnf install -y java-21-openjdk-devel maven && break || sleep 15
|
||||
done
|
||||
fi
|
||||
for i in {1..5}; do
|
||||
dnf install -y maven && break || sleep 15
|
||||
done
|
||||
|
||||
- name: Verify Java and Maven installation
|
||||
run: |
|
||||
@@ -61,4 +57,5 @@ jobs:
|
||||
run: mvn -B package --file pom.xml
|
||||
|
||||
- name: Test with Maven
|
||||
run: mvn test
|
||||
run: mvn test
|
||||
|
||||
|
||||
32
.github/workflows/docker-publish.yml
vendored
Normal file
32
.github/workflows/docker-publish.yml
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
name: Docker Publish
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Version of the Docker image'
|
||||
required: true
|
||||
default: 'latest'
|
||||
|
||||
jobs:
|
||||
build-and-publish:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and publish Docker image
|
||||
run: |
|
||||
VERSION=${{ github.event.inputs.version }}
|
||||
chmod +x docker/docker-build-publish.sh
|
||||
sh docker/docker-build-publish.sh $VERSION
|
||||
2
.github/workflows/mac-ci.yml
vendored
2
.github/workflows/mac-ci.yml
vendored
@@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
java-version: [8, 11, 21] # Define the JDK versions to test
|
||||
java-version: [21] # Define the JDK versions to test
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
2
.github/workflows/ubuntu-ci.yml
vendored
2
.github/workflows/ubuntu-ci.yml
vendored
@@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
java-version: [8, 11, 21] # 定义要测试的JDK版本
|
||||
java-version: [21] # 定义要测试的JDK版本
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
2
.github/workflows/windows-ci.yml
vendored
2
.github/workflows/windows-ci.yml
vendored
@@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
java-version: [8, 11, 21] # Add JDK 21 to the matrix
|
||||
java-version: [21] # Add JDK 21 to the matrix
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -19,4 +19,5 @@ assembly/runtime/*
|
||||
chm_db/
|
||||
__pycache__/
|
||||
/dict
|
||||
assembly/build/*-SNAPSHOT
|
||||
assembly/build/*-SNAPSHOT
|
||||
**/node_modules/
|
||||
@@ -7,11 +7,11 @@
|
||||
## SuperSonic [0.9.8] - 2024-11-01
|
||||
- Add LLM management module to reuse connection across agents.
|
||||
- Add ChatAPP configuration sub-module in Agent Management.
|
||||
- Add dimension value management sub-module.
|
||||
- Enhance dimension value management sub-module.
|
||||
- Enhance memory management and term management sub-module.
|
||||
- Support semantic translation of complex S2SQL.
|
||||
- Enhance semantic translation of complex S2SQL.
|
||||
- Enhance user experience in Chat UI.
|
||||
- Introduce LLM-based semantic corrector and data interpreter.
|
||||
- Introduce new experience in Chat UI.
|
||||
|
||||
## SuperSonic [0.9.2] - 2024-06-01
|
||||
|
||||
|
||||
18
LICENSE
18
LICENSE
@@ -1,19 +1,11 @@
|
||||
SuperSonic is licensed under the MIT License, with the following additional conditions:
|
||||
|
||||
1. You may provide SuperSonic to third parties as a commercial software or service. However,
|
||||
when the following conditions are met, you must contact the producer to obtain a commercial license:
|
||||
|
||||
a. Multi-tenant SaaS service: Unless explicitly authorized by SuperSonic in writing, you may not use the
|
||||
SuperSonic source code to operate a multi-tenant SaaS service.
|
||||
b. LOGO and copyright information: In the process of using SuperSonic, you may not remove or modify
|
||||
the LOGO or copyright information on the SuperSonic UI. This restriction is inapplicable to uses of
|
||||
SuperSonic that do not involve its frontend components.
|
||||
|
||||
SuperSonic is licensed under the MIT License, you can freely use or integrate SuperSonic within
|
||||
your organization. However, if you want to provide or integrate SuperSonic to third parties
|
||||
as a commercial software or service, you must contact the producer to obtain a commercial license.
|
||||
Please contact jerryjzhang@tencent.com by email to inquire about licensing matters.
|
||||
|
||||
2. As a contributor, you should agree that:
|
||||
As a SuperSonic contributor, you should agree that:
|
||||
|
||||
a. The producer can adjust the open-source agreement to be more strict or relaxed as deemed necessary.
|
||||
a. The producer can adjust the open-source agreement to be stricter or relaxed as deemed necessary.
|
||||
b. Your contributed code may be used for commercial purposes, including but not limited to its business operations.
|
||||
|
||||
Terms of the MIT License:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
# SuperSonic
|
||||
|
||||
SuperSonic is the next-generation BI platform that unifies **Chat BI** (powered by LLM) and **Headless BI** (powered by semantic layer) paradigms. This unification ensures that Chat BI has access to the same curated and governed semantic data models as traditional BI. Furthermore, the implementation of both paradigms benefit from each other:
|
||||
SuperSonic is the next-generation AI+BI platform that unifies **Chat BI** (powered by LLM) and **Headless BI** (powered by semantic layer) paradigms. This unification ensures that Chat BI has access to the same curated and governed semantic data models as traditional BI. Furthermore, the implementation of both paradigms benefit from each other:
|
||||
|
||||
- Chat BI's Text2SQL gets augmented with context-retrieval from semantic models.
|
||||
- Headless BI's query interface gets extended with natural language API.
|
||||
|
||||
@@ -4,7 +4,13 @@ sbinDir=$(cd "$(dirname "$0")"; pwd)
|
||||
chmod +x $sbinDir/supersonic-common.sh
|
||||
source $sbinDir/supersonic-common.sh
|
||||
cd $projectDir
|
||||
MVN_VERSION=$(mvn help:evaluate -Dexpression=project.version | grep -e '^[^\[]')
|
||||
|
||||
MVN_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout | grep -v '^\[' | sed -n '/^[0-9]/p')
|
||||
if [ -z "$MVN_VERSION" ]; then
|
||||
echo "Failed to retrieve Maven project version."
|
||||
exit 1
|
||||
fi
|
||||
echo "Maven project version: $MVN_VERSION"
|
||||
|
||||
cd $baseDir
|
||||
service=$1
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package com.tencent.supersonic.auth.api.authentication.adaptor;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.UserToken;
|
||||
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@@ -18,6 +18,9 @@ public class AuthenticationConfig {
|
||||
@Value("${s2.authentication.include.path:/api}")
|
||||
private String includePath;
|
||||
|
||||
@Value("${s2.authentication.strategy:http}")
|
||||
private String strategy;
|
||||
|
||||
@Value("${s2.authentication.enable:false}")
|
||||
private boolean enabled;
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import java.util.Map;
|
||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_CREATE_TIME;
|
||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_IS_ADMIN;
|
||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_DISPLAY_NAME;
|
||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_EMAIL;
|
||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_ID;
|
||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_NAME;
|
||||
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_PASSWORD;
|
||||
@@ -38,6 +39,7 @@ public class UserWithPassword extends User {
|
||||
claims.put(TOKEN_USER_NAME, StringUtils.isEmpty(user.getName()) ? "" : user.getName());
|
||||
claims.put(TOKEN_USER_PASSWORD,
|
||||
StringUtils.isEmpty(user.getPassword()) ? "" : user.getPassword());
|
||||
claims.put(TOKEN_USER_EMAIL, StringUtils.isEmpty(user.getEmail()) ? "" : user.getEmail());
|
||||
claims.put(TOKEN_USER_DISPLAY_NAME, user.getDisplayName());
|
||||
claims.put(TOKEN_CREATE_TIME, System.currentTimeMillis());
|
||||
claims.put(TOKEN_IS_ADMIN, user.getIsAdmin());
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.tencent.supersonic.auth.api.authentication.request;
|
||||
|
||||
import javax.validation.constraints.NotBlank;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.tencent.supersonic.auth.api.authentication.request;
|
||||
|
||||
import javax.validation.constraints.NotBlank;
|
||||
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
package com.tencent.supersonic.auth.api.authentication.service;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.UserToken;
|
||||
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
package com.tencent.supersonic.auth.api.authentication.service;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
|
||||
public interface UserStrategy {
|
||||
|
||||
String getStrategyName();
|
||||
|
||||
boolean accept(boolean isEnableAuthentication);
|
||||
|
||||
User findUser(HttpServletRequest request, HttpServletResponse response);
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
package com.tencent.supersonic.auth.api.authentication.utils;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.service.UserStrategy;
|
||||
import com.tencent.supersonic.common.config.SystemConfig;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.service.SystemConfigService;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
public final class UserHolder {
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
package com.tencent.supersonic.auth.authentication.adaptor;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor;
|
||||
@@ -16,6 +14,7 @@ import com.tencent.supersonic.auth.authentication.utils.TokenService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.util.AESEncryptionUtil;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
|
||||
|
||||
@@ -1,21 +1,13 @@
|
||||
package com.tencent.supersonic.auth.authentication.interceptor;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
||||
import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
|
||||
import com.tencent.supersonic.auth.authentication.utils.TokenService;
|
||||
import com.tencent.supersonic.common.util.S2ThreadContext;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.catalina.connector.RequestFacade;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.tomcat.util.http.MimeHeaders;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest;
|
||||
import org.springframework.web.servlet.HandlerInterceptor;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
@@ -28,8 +20,6 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor {
|
||||
|
||||
protected TokenService tokenService;
|
||||
|
||||
protected S2ThreadContext s2ThreadContext;
|
||||
|
||||
protected boolean isExcludedUri(String uri) {
|
||||
String excludePathStr = authenticationConfig.getExcludePath();
|
||||
if (StringUtils.isEmpty(excludePathStr)) {
|
||||
@@ -54,52 +44,4 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor {
|
||||
return includePaths.stream().anyMatch(uri::startsWith);
|
||||
}
|
||||
|
||||
protected boolean isInternalRequest(HttpServletRequest request) {
|
||||
String internal = request.getHeader(UserConstants.INTERNAL);
|
||||
return "true".equalsIgnoreCase(internal);
|
||||
}
|
||||
|
||||
protected boolean isAppRequest(HttpServletRequest request) {
|
||||
String appId = request.getHeader(authenticationConfig.getAppId());
|
||||
return StringUtils.isNotBlank(appId);
|
||||
}
|
||||
|
||||
protected void reflectSetParam(HttpServletRequest request, String key, String value) {
|
||||
try {
|
||||
if (request instanceof StandardMultipartHttpServletRequest) {
|
||||
RequestFacade servletRequest =
|
||||
(RequestFacade) ((StandardMultipartHttpServletRequest) request)
|
||||
.getRequest();
|
||||
Class<? extends HttpServletRequest> servletRequestClazz = servletRequest.getClass();
|
||||
Field request1 = servletRequestClazz.getDeclaredField("request");
|
||||
request1.setAccessible(true);
|
||||
Object o = request1.get(servletRequest);
|
||||
Field coyoteRequest = o.getClass().getDeclaredField("coyoteRequest");
|
||||
coyoteRequest.setAccessible(true);
|
||||
Object o1 = coyoteRequest.get(o);
|
||||
Field headers = o1.getClass().getDeclaredField("headers");
|
||||
headers.setAccessible(true);
|
||||
MimeHeaders o2 = (MimeHeaders) headers.get(o1);
|
||||
if (o2.getValue(key) != null) {
|
||||
o2.setValue(key).setString(value);
|
||||
} else {
|
||||
o2.addValue(key).setString(value);
|
||||
}
|
||||
} else {
|
||||
Class<? extends HttpServletRequest> requestClass = request.getClass();
|
||||
Field request1 = requestClass.getDeclaredField("request");
|
||||
request1.setAccessible(true);
|
||||
Object o = request1.get(request);
|
||||
Field coyoteRequest = o.getClass().getDeclaredField("coyoteRequest");
|
||||
coyoteRequest.setAccessible(true);
|
||||
Object o1 = coyoteRequest.get(o);
|
||||
Field headers = o1.getClass().getDeclaredField("headers");
|
||||
headers.setAccessible(true);
|
||||
MimeHeaders o2 = (MimeHeaders) headers.get(o1);
|
||||
o2.addValue(key).setString(value);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("reflectSetParam error:", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,15 @@
|
||||
package com.tencent.supersonic.auth.authentication.interceptor;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.annotation.AuthenticationIgnore;
|
||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
|
||||
import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
|
||||
import com.tencent.supersonic.auth.authentication.utils.TokenService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.exception.AccessException;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.S2ThreadContext;
|
||||
import com.tencent.supersonic.common.util.ThreadContext;
|
||||
import io.jsonwebtoken.Claims;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.method.HandlerMethod;
|
||||
|
||||
@@ -36,19 +32,10 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor
|
||||
authenticationConfig = ContextUtils.getBean(AuthenticationConfig.class);
|
||||
userServiceImpl = ContextUtils.getBean(UserServiceImpl.class);
|
||||
tokenService = ContextUtils.getBean(TokenService.class);
|
||||
s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class);
|
||||
if (!authenticationConfig.isEnabled()) {
|
||||
setFakerUser(request);
|
||||
return true;
|
||||
}
|
||||
if (isInternalRequest(request)) {
|
||||
setFakerUser(request);
|
||||
return true;
|
||||
}
|
||||
if (isAppRequest(request)) {
|
||||
setFakerUser(request);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (handler instanceof HandlerMethod) {
|
||||
HandlerMethod handlerMethod = (HandlerMethod) handler;
|
||||
Method method = handlerMethod.getMethod();
|
||||
@@ -69,35 +56,11 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor
|
||||
|
||||
UserWithPassword user = getUserWithPassword(request);
|
||||
if (user != null) {
|
||||
setContext(user.getName(), request);
|
||||
return true;
|
||||
}
|
||||
throw new AccessException("authentication failed, please login");
|
||||
}
|
||||
|
||||
private void setFakerUser(HttpServletRequest request) {
|
||||
String token = generateAdminToken(request);
|
||||
reflectSetParam(request, authenticationConfig.getTokenHttpHeaderKey(), token);
|
||||
setContext(User.getDefaultUser().getName(), request);
|
||||
}
|
||||
|
||||
private void setContext(String userName, HttpServletRequest request) {
|
||||
ThreadContext threadContext = ThreadContext.builder()
|
||||
.token(request.getHeader(authenticationConfig.getTokenHttpHeaderKey()))
|
||||
.userName(userName).build();
|
||||
s2ThreadContext.set(threadContext);
|
||||
}
|
||||
|
||||
public String generateAdminToken(HttpServletRequest request) {
|
||||
UserWithPassword admin = new UserWithPassword("admin");
|
||||
admin.setId(1L);
|
||||
admin.setName("admin");
|
||||
admin.setPassword("c3VwZXJzb25pY0BiaWNvbdktJJYWw6A3rEmBUPzbn/6DNeYnD+y3mAwDKEMS3KVT");
|
||||
admin.setDisplayName("admin");
|
||||
admin.setIsAdmin(1);
|
||||
return tokenService.generateToken(UserWithPassword.convert(admin), request);
|
||||
}
|
||||
|
||||
public UserWithPassword getUserWithPassword(HttpServletRequest request) {
|
||||
final Optional<Claims> claimsOptional = tokenService.getClaims(request);
|
||||
if (!claimsOptional.isPresent()) {
|
||||
|
||||
@@ -51,7 +51,7 @@ public class UserRepositoryImpl implements UserRepository {
|
||||
@Override
|
||||
public List<UserTokenDO> getUserTokenListByName(String userName) {
|
||||
QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
|
||||
queryWrapper.eq("user_name", userName);
|
||||
queryWrapper.lambda().eq(UserTokenDO::getUserName, userName);
|
||||
return userTokenDOMapper.selectList(queryWrapper);
|
||||
}
|
||||
|
||||
@@ -68,7 +68,7 @@ public class UserRepositoryImpl implements UserRepository {
|
||||
@Override
|
||||
public void deleteUserTokenByName(String userName) {
|
||||
QueryWrapper<UserTokenDO> queryWrapper = new QueryWrapper<>();
|
||||
queryWrapper.eq("user_name", userName);
|
||||
queryWrapper.lambda().eq(UserTokenDO::getUserName, userName);
|
||||
userTokenDOMapper.delete(queryWrapper);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
package com.tencent.supersonic.auth.authentication.rest;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.UserToken;
|
||||
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
||||
import com.tencent.supersonic.auth.api.authentication.request.UserTokenReq;
|
||||
import com.tencent.supersonic.auth.api.authentication.service.UserService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
package com.tencent.supersonic.auth.authentication.service;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.UserToken;
|
||||
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
||||
@@ -12,6 +9,8 @@ import com.tencent.supersonic.auth.authentication.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.common.config.SystemConfig;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.service.SystemConfigService;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
package com.tencent.supersonic.auth.authentication.strategy;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.service.UserStrategy;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class FakeUserStrategy implements UserStrategy {
|
||||
|
||||
public static final String STRATEGY_NAME = "fake";
|
||||
|
||||
@Override
|
||||
public String getStrategyName() {
|
||||
return STRATEGY_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accept(boolean isEnableAuthentication) {
|
||||
return !isEnableAuthentication;
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
package com.tencent.supersonic.auth.authentication.strategy;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
||||
import com.tencent.supersonic.auth.api.authentication.service.UserStrategy;
|
||||
import com.tencent.supersonic.auth.authentication.utils.TokenService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import io.jsonwebtoken.Claims;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Optional;
|
||||
@@ -15,12 +14,18 @@ import java.util.Optional;
|
||||
@Service
|
||||
public class HttpHeaderUserStrategy implements UserStrategy {
|
||||
|
||||
public static final String STRATEGY_NAME = "http";
|
||||
private final TokenService tokenService;
|
||||
|
||||
public HttpHeaderUserStrategy(TokenService tokenService) {
|
||||
this.tokenService = tokenService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getStrategyName() {
|
||||
return STRATEGY_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accept(boolean isEnableAuthentication) {
|
||||
return isEnableAuthentication;
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
package com.tencent.supersonic.auth.authentication.strategy;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||
import com.tencent.supersonic.auth.api.authentication.service.UserStrategy;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.Data;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Configuration
|
||||
@Data
|
||||
@@ -26,10 +27,26 @@ public class UserStrategyFactory {
|
||||
|
||||
@PostConstruct
|
||||
public void setUserStrategy() {
|
||||
for (UserStrategy userStrategy : userStrategyList) {
|
||||
if (userStrategy.accept(authenticationConfig.isEnabled())) {
|
||||
UserHolder.setStrategy(userStrategy);
|
||||
|
||||
boolean enabled = authenticationConfig.isEnabled();
|
||||
if (!enabled) {
|
||||
for (UserStrategy userStrategy : userStrategyList) {
|
||||
if (userStrategy.accept(authenticationConfig.isEnabled())) {
|
||||
UserHolder.setStrategy(userStrategy);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
String strategy = authenticationConfig.getStrategy();
|
||||
Optional<UserStrategy> strategyOptional = userStrategyList.stream()
|
||||
.filter(t -> t.accept(true) && strategy.equalsIgnoreCase(t.getStrategyName()))
|
||||
.findAny();
|
||||
|
||||
if (strategyOptional.isPresent()) {
|
||||
UserHolder.setStrategy(strategyOptional.get());
|
||||
} else {
|
||||
throw new IllegalStateException("strategy is not found: " + strategy);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
package com.tencent.supersonic.auth.authentication.utils;
|
||||
|
||||
|
||||
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
|
||||
@@ -9,6 +10,7 @@ import com.tencent.supersonic.common.pojo.exception.AccessException;
|
||||
import io.jsonwebtoken.Claims;
|
||||
import io.jsonwebtoken.Jwts;
|
||||
import io.jsonwebtoken.SignatureAlgorithm;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
<artifactId>auth-authorization</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
package com.tencent.supersonic.auth.authorization.rest;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
|
||||
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
||||
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
||||
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
|
||||
@@ -8,7 +8,6 @@ import java.util.List;
|
||||
|
||||
/** extended information command about model */
|
||||
@Data
|
||||
@ToString
|
||||
public class ChatConfigBaseReq {
|
||||
|
||||
private Long modelId;
|
||||
|
||||
@@ -17,6 +17,8 @@ public class ChatMemoryFilter {
|
||||
|
||||
private Integer agentId;
|
||||
|
||||
private Long queryId;
|
||||
|
||||
private String question;
|
||||
|
||||
private List<String> questions;
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
package com.tencent.supersonic.chat.api.pojo.request;
|
||||
|
||||
import javax.validation.constraints.NotNull;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ChatMemoryUpdateReq {
|
||||
|
||||
@NotNull(message = "id不可为空")
|
||||
@@ -21,4 +26,8 @@ public class ChatMemoryUpdateReq {
|
||||
private MemoryReviewResult humanReviewRet;
|
||||
|
||||
private String humanReviewCmt;
|
||||
|
||||
private MemoryReviewResult llmReviewRet;
|
||||
|
||||
private String llmReviewCmt;
|
||||
}
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
package com.tencent.supersonic.chat.api.pojo.request;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** the entity info about the model */
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@ToString
|
||||
@NoArgsConstructor
|
||||
public class Entity {
|
||||
|
||||
/** uniquely identifies an entity */
|
||||
private Long entityId;
|
||||
|
||||
/** entity name list */
|
||||
private List<String> names;
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
public class ItemNameVisibility {
|
||||
|
||||
private ItemNameVisibilityInfo aggVisibilityInfo;
|
||||
|
||||
@@ -7,7 +7,6 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
public class ItemNameVisibilityInfo {
|
||||
|
||||
/** invisible dimensions */
|
||||
|
||||
@@ -7,7 +7,6 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
public class ItemVisibility {
|
||||
|
||||
/** invisible dimensions */
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
package com.tencent.supersonic.chat.api.pojo.request;
|
||||
|
||||
import javax.validation.constraints.NotNull;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.Data;
|
||||
|
||||
/** information about dictionary about the model */
|
||||
|
||||
@@ -6,7 +6,6 @@ import lombok.NoArgsConstructor;
|
||||
import lombok.ToString;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RecommendedQuestionReq {
|
||||
|
||||
@@ -6,7 +6,6 @@ import lombok.ToString;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
@ToString
|
||||
@Data
|
||||
public class DictLatestTaskResp {
|
||||
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
package com.tencent.supersonic.chat.api.pojo.response;
|
||||
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class EntityRichInfoResp {
|
||||
/** entity alias */
|
||||
private List<String> names;
|
||||
|
||||
private SchemaElement dimItem;
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.api.pojo.response;
|
||||
import com.tencent.supersonic.common.pojo.QueryAuthorization;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.headless.api.pojo.AggregateInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.QueryState;
|
||||
@@ -26,7 +25,6 @@ public class QueryResult {
|
||||
private String textResult;
|
||||
private String textSummary;
|
||||
private Long queryTimeCost;
|
||||
private EntityInfo entityInfo;
|
||||
private List<SchemaElement> recommendedDimensions;
|
||||
private AggregateInfo aggregateInfo;
|
||||
private String errorMsg;
|
||||
|
||||
@@ -3,10 +3,12 @@ package com.tencent.supersonic.chat.api.pojo.response;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RecommendQuestionResp {
|
||||
private Long modelId;
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
package com.tencent.supersonic.chat.api.pojo.response;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SimilarQueryRecallResp {
|
||||
|
||||
private Long queryId;
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package com.tencent.supersonic.chat.server.agent;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.chat.server.memory.MemoryReviewTask;
|
||||
import com.tencent.supersonic.common.pojo.ChatApp;
|
||||
import com.tencent.supersonic.common.pojo.RecordInfo;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import lombok.Data;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@@ -12,6 +14,7 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Data
|
||||
@@ -33,6 +36,8 @@ public class Agent extends RecordInfo {
|
||||
private String toolConfig;
|
||||
private Map<String, ChatApp> chatAppConfig = Collections.emptyMap();
|
||||
private VisualConfig visualConfig;
|
||||
private List<String> admins = Lists.newArrayList();
|
||||
private List<String> viewers = Lists.newArrayList();
|
||||
|
||||
public List<String> getTools(AgentToolType type) {
|
||||
Map<String, Object> map = JSONObject.parseObject(toolConfig, Map.class);
|
||||
@@ -105,4 +110,9 @@ public class Agent extends RecordInfo {
|
||||
.filter(dataSetIds -> !CollectionUtils.isEmpty(dataSetIds))
|
||||
.flatMap(Collection::stream).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
public boolean contains(User user, Function<Agent, List<String>> list) {
|
||||
return list.apply(this).contains(user.getName());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import java.util.stream.Collectors;
|
||||
public class PlainTextExecutor implements ChatQueryExecutor {
|
||||
|
||||
public static final String APP_KEY = "SMALL_TALK";
|
||||
private static final String INSTRUCTION = "" + "#Role: You are a nice person to talk to."
|
||||
private static final String INSTRUCTION = "#Role: You are a nice person to talk to."
|
||||
+ "\n#Task: Respond quickly and nicely to the user."
|
||||
+ "\n#Rules: 1.ALWAYS use the same language as the `#Current Input`."
|
||||
+ "\n#History Inputs: %s" + "\n#Current Input: %s" + "\n#Response: ";
|
||||
|
||||
@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.executor;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
|
||||
import com.tencent.supersonic.chat.server.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
||||
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
||||
import com.tencent.supersonic.chat.server.service.ChatContextService;
|
||||
import com.tencent.supersonic.chat.server.service.MemoryService;
|
||||
@@ -44,7 +44,7 @@ public class SqlExecutor implements ChatQueryExecutor {
|
||||
Text2SQLExemplar.class);
|
||||
|
||||
MemoryService memoryService = ContextUtils.getBean(MemoryService.class);
|
||||
memoryService.createMemory(ChatMemoryDO.builder()
|
||||
memoryService.createMemory(ChatMemory.builder().queryId(queryResult.getQueryId())
|
||||
.agentId(executeContext.getAgent().getId()).status(MemoryStatus.PENDING)
|
||||
.question(exemplar.getQuestion()).sideInfo(exemplar.getSideInfo())
|
||||
.dbSchema(exemplar.getDbSchema()).s2sql(exemplar.getSql())
|
||||
@@ -77,6 +77,7 @@ public class SqlExecutor implements ChatQueryExecutor {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
QueryResult queryResult = new QueryResult();
|
||||
queryResult.setQueryId(executeContext.getRequest().getQueryId());
|
||||
queryResult.setChatContext(parseInfo);
|
||||
queryResult.setQueryMode(parseInfo.getQueryMode());
|
||||
queryResult.setQueryTimeCost(System.currentTimeMillis() - startTime);
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
package com.tencent.supersonic.chat.server.memory;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
|
||||
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
||||
import com.tencent.supersonic.chat.server.service.AgentService;
|
||||
import com.tencent.supersonic.chat.server.service.MemoryService;
|
||||
import com.tencent.supersonic.common.pojo.ChatApp;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AppModule;
|
||||
import com.tencent.supersonic.common.util.ChatAppManager;
|
||||
import com.tencent.supersonic.headless.server.utils.ModelConfigHelper;
|
||||
@@ -21,6 +25,7 @@ import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -57,24 +62,36 @@ public class MemoryReviewTask {
|
||||
|
||||
@Scheduled(fixedDelay = 60 * 1000)
|
||||
public void review() {
|
||||
memoryService.getMemoriesForLlmReview().stream().forEach(memory -> {
|
||||
try {
|
||||
processMemory(memory);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception occurred while processing memory with id {}: {}",
|
||||
memory.getId(), e.getMessage(), e);
|
||||
List<Agent> agentList = agentService.getAgents();
|
||||
for (Agent agent : agentList) {
|
||||
if (!agent.enableMemoryReview()) {
|
||||
continue;
|
||||
}
|
||||
});
|
||||
ChatMemoryFilter chatMemoryFilter =
|
||||
ChatMemoryFilter.builder().agentId(agent.getId()).build();
|
||||
memoryService.getMemories(chatMemoryFilter).forEach(memory -> {
|
||||
try {
|
||||
processMemory(memory, agent);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception occurred while processing memory with id {}: {}",
|
||||
memory.getId(), e.getMessage(), e);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void processMemory(ChatMemoryDO m) {
|
||||
Agent chatAgent = agentService.getAgent(m.getAgentId());
|
||||
if (Objects.isNull(chatAgent)) {
|
||||
private void processMemory(ChatMemory m, Agent agent) {
|
||||
if (Objects.isNull(agent)) {
|
||||
log.warn("Agent id {} not found or memory review disabled", m.getAgentId());
|
||||
return;
|
||||
}
|
||||
|
||||
ChatApp chatApp = chatAgent.getChatAppConfig().get(APP_KEY);
|
||||
// if either LLM or human has reviewed, just return
|
||||
if (Objects.nonNull(m.getLlmReviewRet()) || Objects.nonNull(m.getHumanReviewRet())) {
|
||||
return;
|
||||
}
|
||||
|
||||
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
|
||||
if (Objects.isNull(chatApp) || !chatApp.isEnable()) {
|
||||
return;
|
||||
}
|
||||
@@ -90,25 +107,28 @@ public class MemoryReviewTask {
|
||||
response);
|
||||
processResponse(response, m);
|
||||
} else {
|
||||
log.debug("ChatLanguageModel not found for agent:{}", chatAgent.getId());
|
||||
log.debug("ChatLanguageModel not found for agent:{}", agent.getId());
|
||||
}
|
||||
}
|
||||
|
||||
private String createPromptString(ChatMemoryDO m, String promptTemplate) {
|
||||
private String createPromptString(ChatMemory m, String promptTemplate) {
|
||||
return String.format(promptTemplate, m.getQuestion(), m.getDbSchema(), m.getSideInfo(),
|
||||
m.getS2sql());
|
||||
}
|
||||
|
||||
private void processResponse(String response, ChatMemoryDO m) {
|
||||
private void processResponse(String response, ChatMemory m) {
|
||||
Matcher matcher = OUTPUT_PATTERN.matcher(response);
|
||||
if (matcher.find()) {
|
||||
m.setLlmReviewRet(MemoryReviewResult.getMemoryReviewResult(matcher.group(1)));
|
||||
m.setLlmReviewCmt(matcher.group(2));
|
||||
// directly enable memory if the LLM determines it positive
|
||||
if (MemoryReviewResult.POSITIVE.equals(m.getLlmReviewRet())) {
|
||||
memoryService.enableMemory(m);
|
||||
m.setStatus(MemoryStatus.ENABLED);
|
||||
}
|
||||
memoryService.updateMemory(m);
|
||||
ChatMemoryUpdateReq memoryUpdateReq = ChatMemoryUpdateReq.builder().id(m.getId())
|
||||
.status(m.getStatus()).llmReviewRet(m.getLlmReviewRet())
|
||||
.llmReviewCmt(m.getLlmReviewCmt()).build();
|
||||
memoryService.updateMemory(memoryUpdateReq, User.getDefaultUser());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,11 +83,15 @@ public class NL2SQLParser implements ChatQueryParser {
|
||||
if (Objects.isNull(parseContext.getRequest().getSelectedParse())) {
|
||||
QueryNLReq queryNLReq = QueryReqConverter.buildQueryNLReq(parseContext);
|
||||
queryNLReq.setText2SQLType(Text2SQLType.ONLY_RULE);
|
||||
if (parseContext.enableLLM()) {
|
||||
queryNLReq.setText2SQLType(Text2SQLType.NONE);
|
||||
}
|
||||
|
||||
// for every requested dataSet, recursively invoke rule-based parser with different
|
||||
// mapModes
|
||||
Set<Long> requestedDatasets = queryNLReq.getDataSetIds();
|
||||
List<SemanticParseInfo> candidateParses = Lists.newArrayList();
|
||||
StringBuilder errMsg = new StringBuilder();
|
||||
for (Long datasetId : requestedDatasets) {
|
||||
queryNLReq.setDataSetIds(Collections.singleton(datasetId));
|
||||
ChatParseResp parseResp = new ChatParseResp(parseContext.getRequest().getQueryId());
|
||||
@@ -96,10 +100,16 @@ public class NL2SQLParser implements ChatQueryParser {
|
||||
queryNLReq.setMapModeEnum(mode);
|
||||
doParse(queryNLReq, parseResp);
|
||||
}
|
||||
if (parseResp.getSelectedParses().isEmpty()) {
|
||||
|
||||
if (parseResp.getSelectedParses().isEmpty() && candidateParses.isEmpty()) {
|
||||
queryNLReq.setMapModeEnum(MapModeEnum.LOOSE);
|
||||
doParse(queryNLReq, parseResp);
|
||||
}
|
||||
|
||||
if (parseResp.getSelectedParses().isEmpty()) {
|
||||
errMsg.append(parseResp.getErrorMsg());
|
||||
continue;
|
||||
}
|
||||
// for one dataset select the top 1 parse after sorting
|
||||
SemanticParseInfo.sort(parseResp.getSelectedParses());
|
||||
candidateParses.add(parseResp.getSelectedParses().get(0));
|
||||
@@ -110,6 +120,10 @@ public class NL2SQLParser implements ChatQueryParser {
|
||||
SemanticParseInfo.sort(candidateParses);
|
||||
parseContext.getResponse().setSelectedParses(
|
||||
candidateParses.subList(0, Math.min(parserShowCount, candidateParses.size())));
|
||||
if (parseContext.getResponse().getSelectedParses().isEmpty()) {
|
||||
parseContext.getResponse().setState(ParseResp.ParseState.FAILED);
|
||||
parseContext.getResponse().setErrorMsg(errMsg.toString());
|
||||
}
|
||||
}
|
||||
|
||||
// next go with llm-based parsers unless LLM is disabled or use feedback is needed.
|
||||
@@ -125,11 +139,18 @@ public class NL2SQLParser implements ChatQueryParser {
|
||||
SemanticParseInfo userSelectParse = parseContext.getRequest().getSelectedParse();
|
||||
queryNLReq.setSelectedParseInfo(Objects.nonNull(userSelectParse) ? userSelectParse
|
||||
: parseContext.getResponse().getSelectedParses().get(0));
|
||||
|
||||
parseContext.setResponse(new ChatParseResp(parseContext.getResponse().getQueryId()));
|
||||
|
||||
rewriteMultiTurn(parseContext, queryNLReq);
|
||||
addDynamicExemplars(parseContext, queryNLReq);
|
||||
doParse(queryNLReq, parseContext.getResponse());
|
||||
|
||||
// try again with all semantic fields passed to LLM
|
||||
if (parseContext.getResponse().getState().equals(ParseResp.ParseState.FAILED)) {
|
||||
queryNLReq.setSelectedParseInfo(null);
|
||||
queryNLReq.setMapModeEnum(MapModeEnum.ALL);
|
||||
doParse(queryNLReq, parseContext.getResponse());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -40,4 +40,8 @@ public class AgentDO {
|
||||
private String chatModelConfig;
|
||||
|
||||
private String visualConfig;
|
||||
|
||||
private String admin;
|
||||
|
||||
private String viewer;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
package com.tencent.supersonic.chat.server.persistence.dataobject;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
|
||||
@@ -7,9 +10,10 @@ import java.util.Date;
|
||||
|
||||
@Data
|
||||
@ToString
|
||||
@TableName("s2_chat_config")
|
||||
public class ChatConfigDO {
|
||||
|
||||
/** database auto-increment primary key */
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
|
||||
private Long modelId;
|
||||
|
||||
@@ -1,16 +1,22 @@
|
||||
package com.tencent.supersonic.chat.server.persistence.dataobject;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
|
||||
@Data
|
||||
@TableName("s2_chat_context")
|
||||
public class ChatContextDO implements Serializable {
|
||||
|
||||
@TableId
|
||||
private Integer chatId;
|
||||
private Instant modifiedAt;
|
||||
private String user;
|
||||
@TableField("query_user")
|
||||
private String queryUser;
|
||||
private String queryText;
|
||||
private String semanticParse;
|
||||
}
|
||||
|
||||
@@ -4,17 +4,17 @@ import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.ToString;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@ToString
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@TableName("s2_chat_memory")
|
||||
public class ChatMemoryDO {
|
||||
@TableId(type = IdType.AUTO)
|
||||
@@ -23,6 +23,9 @@ public class ChatMemoryDO {
|
||||
@TableField("agent_id")
|
||||
private Integer agentId;
|
||||
|
||||
@TableField("query_id")
|
||||
private Long queryId;
|
||||
|
||||
@TableField("question")
|
||||
private String question;
|
||||
|
||||
@@ -36,16 +39,16 @@ public class ChatMemoryDO {
|
||||
private String s2sql;
|
||||
|
||||
@TableField("status")
|
||||
private MemoryStatus status;
|
||||
private String status;
|
||||
|
||||
@TableField("llm_review")
|
||||
private MemoryReviewResult llmReviewRet;
|
||||
private String llmReviewRet;
|
||||
|
||||
@TableField("llm_comment")
|
||||
private String llmReviewCmt;
|
||||
|
||||
@TableField("human_review")
|
||||
private MemoryReviewResult humanReviewRet;
|
||||
private String humanReviewRet;
|
||||
|
||||
@TableField("human_comment")
|
||||
private String humanReviewCmt;
|
||||
|
||||
@@ -5,9 +5,6 @@ import lombok.Data;
|
||||
@Data
|
||||
public class QueryDO {
|
||||
|
||||
public String aggregator = "trend";
|
||||
public String startTime;
|
||||
public String endTime;
|
||||
private long id;
|
||||
private long questionId;
|
||||
private String createTime;
|
||||
@@ -25,7 +22,6 @@ public class QueryDO {
|
||||
private int topNum;
|
||||
private String querySql;
|
||||
private Object queryColumn;
|
||||
private Object entityInfo;
|
||||
private int score;
|
||||
private String feedback;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.tencent.supersonic.chat.server.persistence.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.tencent.supersonic.chat.server.config.ChatConfigFilterInternal;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatConfigDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
@@ -7,11 +8,7 @@ import org.apache.ibatis.annotations.Mapper;
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface ChatConfigMapper {
|
||||
|
||||
Long addConfig(ChatConfigDO chaConfigPO);
|
||||
|
||||
Long editConfig(ChatConfigDO chaConfigPO);
|
||||
public interface ChatConfigMapper extends BaseMapper<ChatConfigDO> {
|
||||
|
||||
List<ChatConfigDO> search(ChatConfigFilterInternal filterInternal);
|
||||
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
package com.tencent.supersonic.chat.server.persistence.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface ChatContextMapper {
|
||||
public interface ChatContextMapper extends BaseMapper<ChatContextDO> {
|
||||
|
||||
ChatContextDO getContextByChatId(Integer chatId);
|
||||
|
||||
int updateContext(ChatContextDO contextDO);
|
||||
|
||||
int addContext(ChatContextDO contextDO);
|
||||
}
|
||||
|
||||
@@ -32,15 +32,15 @@ public class ChatConfigRepositoryImpl implements ChatConfigRepository {
|
||||
@Override
|
||||
public Long createConfig(ChatConfig chaConfig) {
|
||||
ChatConfigDO chaConfigDO = chatConfigHelper.chatConfig2DO(chaConfig);
|
||||
chatConfigMapper.addConfig(chaConfigDO);
|
||||
chatConfigMapper.insert(chaConfigDO);
|
||||
return chaConfigDO.getId();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long updateConfig(ChatConfig chaConfig) {
|
||||
ChatConfigDO chaConfigDO = chatConfigHelper.chatConfig2DO(chaConfig);
|
||||
|
||||
return chatConfigMapper.editConfig(chaConfigDO);
|
||||
chatConfigMapper.updateById(chaConfigDO);
|
||||
return chaConfigDO.getId();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -35,18 +35,13 @@ public class ChatContextRepositoryImpl implements ChatContextRepository {
|
||||
|
||||
@Override
|
||||
public void updateContext(ChatContext chatCtx) {
|
||||
ChatContextDO context = cast(chatCtx);
|
||||
if (chatContextMapper.getContextByChatId(chatCtx.getChatId()) == null) {
|
||||
chatContextMapper.addContext(context);
|
||||
} else {
|
||||
chatContextMapper.updateContext(context);
|
||||
}
|
||||
chatContextMapper.insertOrUpdate(cast(chatCtx));
|
||||
}
|
||||
|
||||
private ChatContext cast(ChatContextDO contextDO) {
|
||||
ChatContext chatContext = new ChatContext();
|
||||
chatContext.setChatId(contextDO.getChatId());
|
||||
chatContext.setUser(contextDO.getUser());
|
||||
chatContext.setUser(contextDO.getQueryUser());
|
||||
chatContext.setQueryText(contextDO.getQueryText());
|
||||
if (contextDO.getSemanticParse() != null && !contextDO.getSemanticParse().isEmpty()) {
|
||||
SemanticParseInfo semanticParseInfo =
|
||||
@@ -60,7 +55,7 @@ public class ChatContextRepositoryImpl implements ChatContextRepository {
|
||||
ChatContextDO chatContextDO = new ChatContextDO();
|
||||
chatContextDO.setChatId(chatContext.getChatId());
|
||||
chatContextDO.setQueryText(chatContext.getQueryText());
|
||||
chatContextDO.setUser(chatContext.getUser());
|
||||
chatContextDO.setQueryUser(chatContext.getUser());
|
||||
if (chatContext.getParseInfo() != null) {
|
||||
Gson g = new Gson();
|
||||
chatContextDO.setSemanticParse(g.toJson(chatContext.getParseInfo()));
|
||||
|
||||
@@ -17,10 +17,10 @@ import com.tencent.supersonic.chat.server.persistence.mapper.ChatParseMapper;
|
||||
import com.tencent.supersonic.chat.server.persistence.mapper.ChatQueryDOMapper;
|
||||
import com.tencent.supersonic.chat.server.persistence.mapper.custom.ShowCaseCustomMapper;
|
||||
import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryRepository;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.common.util.PageUtils;
|
||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ParseTimeCostResp;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
@@ -117,6 +117,20 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
|
||||
JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class);
|
||||
if (queryResult != null) {
|
||||
queryResult.setQueryId(chatQueryDO.getQuestionId());
|
||||
// fix bugs, compatible with bugs caused by history field changes
|
||||
if (!CollectionUtils.isEmpty(queryResult.getQueryColumns())) {
|
||||
List<QueryColumn> queryColumns = queryResult.getQueryColumns().stream().peek(x -> {
|
||||
if (StringUtils.isEmpty(x.getBizName())
|
||||
&& StringUtils.isNotEmpty(x.getNameEn())) {
|
||||
x.setBizName(x.getNameEn());
|
||||
}
|
||||
if (StringUtils.isNotEmpty(x.getBizName())
|
||||
&& StringUtils.isEmpty(x.getNameEn())) {
|
||||
x.setNameEn(x.getBizName());
|
||||
}
|
||||
}).collect(Collectors.toList());
|
||||
queryResult.setQueryColumns(queryColumns);
|
||||
}
|
||||
queryResp.setQueryResult(queryResult);
|
||||
}
|
||||
queryResp.setSimilarQueries(JSONObject.parseArray(chatQueryDO.getSimilarQueries(),
|
||||
|
||||
@@ -7,7 +7,7 @@ import java.util.Map;
|
||||
|
||||
public class PluginQueryManager {
|
||||
|
||||
private static Map<String, PluginSemanticQuery> pluginQueries = new HashMap<>();
|
||||
private static final Map<String, PluginSemanticQuery> pluginQueries = new HashMap<>();
|
||||
|
||||
public static void register(String queryMode, PluginSemanticQuery pluginSemanticQuery) {
|
||||
pluginQueries.put(queryMode, pluginSemanticQuery);
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.tencent.supersonic.chat.server.pojo;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
||||
import lombok.*;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@ToString
|
||||
public class ChatMemory {
|
||||
private Long id;
|
||||
|
||||
private Integer agentId;
|
||||
|
||||
private Long queryId;
|
||||
|
||||
private String question;
|
||||
|
||||
private String sideInfo;
|
||||
|
||||
private String dbSchema;
|
||||
|
||||
private String s2sql;
|
||||
|
||||
private MemoryStatus status;
|
||||
|
||||
private MemoryReviewResult llmReviewRet;
|
||||
|
||||
private String llmReviewCmt;
|
||||
|
||||
private MemoryReviewResult humanReviewRet;
|
||||
|
||||
private String humanReviewCmt;
|
||||
|
||||
private String createdBy;
|
||||
|
||||
private Date createdAt;
|
||||
|
||||
private String updatedBy;
|
||||
|
||||
private Date updatedAt;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.tencent.supersonic.chat.server.pojo;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||
import lombok.Data;
|
||||
@@ -8,6 +9,7 @@ import lombok.Data;
|
||||
@Data
|
||||
public class ExecuteContext {
|
||||
private ChatExecuteReq request;
|
||||
private QueryResult response;
|
||||
private Agent agent;
|
||||
private SemanticParseInfo parseInfo;
|
||||
|
||||
|
||||
@@ -43,12 +43,17 @@ public class DataInterpretProcessor implements ExecuteResultProcessor {
|
||||
|
||||
|
||||
@Override
|
||||
public void process(ExecuteContext executeContext, QueryResult queryResult) {
|
||||
public boolean accept(ExecuteContext executeContext) {
|
||||
Agent agent = executeContext.getAgent();
|
||||
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
|
||||
return Objects.nonNull(chatApp) && chatApp.isEnable();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ExecuteContext executeContext) {
|
||||
QueryResult queryResult = executeContext.getResponse();
|
||||
Agent agent = executeContext.getAgent();
|
||||
ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY);
|
||||
if (Objects.isNull(chatApp) || !chatApp.isEnable()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, Object> variable = new HashMap<>();
|
||||
variable.put("question", executeContext.getRequest().getQueryText());
|
||||
|
||||
@@ -27,17 +27,18 @@ public class DimensionRecommendProcessor implements ExecuteResultProcessor {
|
||||
private static final int recommend_dimension_size = 5;
|
||||
|
||||
@Override
|
||||
public void process(ExecuteContext executeContext, QueryResult queryResult) {
|
||||
public boolean accept(ExecuteContext executeContext) {
|
||||
SemanticParseInfo semanticParseInfo = executeContext.getParseInfo();
|
||||
return QueryType.AGGREGATE.equals(semanticParseInfo.getQueryType())
|
||||
&& !CollectionUtils.isEmpty(semanticParseInfo.getMetrics());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ExecuteContext executeContext) {
|
||||
QueryResult queryResult = executeContext.getResponse();
|
||||
SemanticParseInfo semanticParseInfo = executeContext.getParseInfo();
|
||||
if (!QueryType.AGGREGATE.equals(semanticParseInfo.getQueryType())
|
||||
|| CollectionUtils.isEmpty(semanticParseInfo.getMetrics())) {
|
||||
return;
|
||||
}
|
||||
Long dataSetId = semanticParseInfo.getDataSetId();
|
||||
Optional<SchemaElement> firstMetric = semanticParseInfo.getMetrics().stream().findFirst();
|
||||
if (!firstMetric.isPresent()) {
|
||||
return;
|
||||
}
|
||||
List<SchemaElement> dimensionRecommended =
|
||||
getDimensions(firstMetric.get().getId(), dataSetId);
|
||||
queryResult.setRecommendedDimensions(dimensionRecommended);
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package com.tencent.supersonic.chat.server.processor.execute;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
||||
import com.tencent.supersonic.chat.server.processor.ResultProcessor;
|
||||
|
||||
/** A ExecuteResultProcessor wraps things up before returning execution results to the users. */
|
||||
public interface ExecuteResultProcessor extends ResultProcessor {
|
||||
|
||||
void process(ExecuteContext executeContext, QueryResult queryResult);
|
||||
boolean accept(ExecuteContext executeContext);
|
||||
|
||||
void process(ExecuteContext executeContext);
|
||||
}
|
||||
|
||||
@@ -59,14 +59,18 @@ import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT;
|
||||
public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
|
||||
|
||||
@Override
|
||||
public void process(ExecuteContext executeContext, QueryResult queryResult) {
|
||||
public boolean accept(ExecuteContext executeContext) {
|
||||
SemanticParseInfo semanticParseInfo = executeContext.getParseInfo();
|
||||
AggregatorConfig aggregatorConfig = ContextUtils.getBean(AggregatorConfig.class);
|
||||
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics())
|
||||
|| !aggregatorConfig.getEnableRatio()
|
||||
|| !QueryType.AGGREGATE.equals(semanticParseInfo.getQueryType())) {
|
||||
return;
|
||||
}
|
||||
return !CollectionUtils.isEmpty(semanticParseInfo.getMetrics())
|
||||
&& aggregatorConfig.getEnableRatio()
|
||||
&& QueryType.AGGREGATE.equals(semanticParseInfo.getQueryType());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ExecuteContext executeContext) {
|
||||
QueryResult queryResult = executeContext.getResponse();
|
||||
SemanticParseInfo semanticParseInfo = executeContext.getParseInfo();
|
||||
AggregateInfo aggregateInfo = getAggregateInfo(executeContext.getRequest().getUser(),
|
||||
semanticParseInfo, queryResult);
|
||||
queryResult.setAggregateInfo(aggregateInfo);
|
||||
@@ -117,8 +121,12 @@ public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
|
||||
|
||||
CompletableFuture.allOf(metricInfoRoll, metricInfoOver).join();
|
||||
|
||||
metricInfo.setName(metricInfoRoll.get().getName());
|
||||
metricInfo.setValue(metricInfoRoll.get().getValue());
|
||||
if (metricInfoRoll.get().getName() != null) {
|
||||
metricInfo.setName(metricInfoRoll.get().getName());
|
||||
}
|
||||
if (metricInfoOver.get().getValue() != null) {
|
||||
metricInfo.setValue(metricInfoRoll.get().getValue());
|
||||
}
|
||||
metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics());
|
||||
metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics());
|
||||
|
||||
@@ -134,7 +142,7 @@ public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
|
||||
return new HashSet<>();
|
||||
}
|
||||
return queryResult.getQueryColumns().stream()
|
||||
.flatMap(c -> SqlSelectHelper.getColumnFromExpr(c.getNameEn()).stream())
|
||||
.flatMap(c -> SqlSelectHelper.getFieldsFromExpr(c.getBizName()).stream())
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
@@ -159,16 +167,16 @@ public class MetricRatioCalcProcessor implements ExecuteResultProcessor {
|
||||
|
||||
Map<String, Object> result = queryResp.getResultList().get(0);
|
||||
Optional<QueryColumn> valueColumn = queryResp.getColumns().stream()
|
||||
.filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst();
|
||||
.filter(c -> c.getBizName().equals(metric.getBizName())).findFirst();
|
||||
|
||||
if (!valueColumn.isPresent()) {
|
||||
return metricInfo;
|
||||
}
|
||||
String valueField = String.format("%s_%s", valueColumn.get().getNameEn(),
|
||||
String valueField = String.format("%s_%s", valueColumn.get().getBizName(),
|
||||
aggOperatorEnum.getOperator());
|
||||
if (result.containsKey(valueColumn.get().getNameEn())) {
|
||||
if (result.containsKey(valueColumn.get().getBizName())) {
|
||||
DecimalFormat df = new DecimalFormat("#.####");
|
||||
metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn())));
|
||||
metricInfo.setValue(df.format(result.get(valueColumn.get().getBizName())));
|
||||
}
|
||||
String ratio = "";
|
||||
if (Objects.nonNull(result.get(valueField))) {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.tencent.supersonic.chat.server.processor.execute;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.server.pojo.ExecuteContext;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||
@@ -16,14 +15,7 @@ import dev.langchain4j.store.embedding.RetrieveQuery;
|
||||
import dev.langchain4j.store.embedding.RetrieveQueryResult;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
@@ -34,16 +26,20 @@ public class MetricRecommendProcessor implements ExecuteResultProcessor {
|
||||
private static final int METRIC_RECOMMEND_SIZE = 5;
|
||||
|
||||
@Override
|
||||
public void process(ExecuteContext executeContext, QueryResult queryResult) {
|
||||
public boolean accept(ExecuteContext executeContext) {
|
||||
SemanticParseInfo parseInfo = executeContext.getParseInfo();
|
||||
return Objects.nonNull(parseInfo.getQueryType())
|
||||
&& parseInfo.getQueryType().equals(QueryType.AGGREGATE)
|
||||
&& !CollectionUtils.isEmpty(parseInfo.getMetrics())
|
||||
&& parseInfo.getMetrics().size() <= METRIC_RECOMMEND_SIZE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ExecuteContext executeContext) {
|
||||
fillSimilarMetric(executeContext.getParseInfo());
|
||||
}
|
||||
|
||||
private void fillSimilarMetric(SemanticParseInfo parseInfo) {
|
||||
if (!parseInfo.getQueryType().equals(QueryType.AGGREGATE)
|
||||
|| parseInfo.getMetrics().size() > METRIC_RECOMMEND_SIZE
|
||||
|| CollectionUtils.isEmpty(parseInfo.getMetrics())) {
|
||||
return;
|
||||
}
|
||||
List<String> metricNames =
|
||||
Collections.singletonList(parseInfo.getMetrics().iterator().next().getName());
|
||||
Map<String, Object> filterCondition = new HashMap<>();
|
||||
|
||||
@@ -40,17 +40,20 @@ public class ErrorMsgRewriteProcessor implements ParseResultProcessor {
|
||||
ChatAppManager.register(APP_KEY_ERROR_MESSAGE,
|
||||
ChatApp.builder().prompt(REWRITE_ERROR_MESSAGE_INSTRUCTION).name("异常提示改写")
|
||||
.appModule(AppModule.CHAT).description("通过大模型将异常信息改写为更友好和引导性的提示用语")
|
||||
.enable(false).build());
|
||||
.enable(true).build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accept(ParseContext parseContext) {
|
||||
ChatApp chatApp = parseContext.getAgent().getChatAppConfig().get(APP_KEY_ERROR_MESSAGE);
|
||||
return StringUtils.isNotBlank(parseContext.getResponse().getErrorMsg())
|
||||
&& Objects.nonNull(chatApp) && chatApp.isEnable();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ParseContext parseContext) {
|
||||
String errMsg = parseContext.getResponse().getErrorMsg();
|
||||
ChatApp chatApp = parseContext.getAgent().getChatAppConfig().get(APP_KEY_ERROR_MESSAGE);
|
||||
if (StringUtils.isBlank(errMsg) || Objects.isNull(chatApp) || !chatApp.isEnable()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, Object> variables = new HashMap<>();
|
||||
variables.put("user_question", parseContext.getRequest().getQueryText());
|
||||
variables.put("system_message", errMsg);
|
||||
|
||||
@@ -1,34 +1,52 @@
|
||||
package com.tencent.supersonic.chat.server.processor.parse;
|
||||
|
||||
import com.tencent.supersonic.chat.server.plugin.PluginQueryManager;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.chat.server.pojo.ParseContext;
|
||||
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
|
||||
import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
|
||||
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
|
||||
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* ParseInfoFormatProcessor formats parse info to make it more readable to the users.
|
||||
**/
|
||||
@Slf4j
|
||||
public class ParseInfoFormatProcessor implements ParseResultProcessor {
|
||||
|
||||
@Override
|
||||
public boolean accept(ParseContext parseContext) {
|
||||
return !parseContext.getResponse().getSelectedParses().isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ParseContext parseContext) {
|
||||
parseContext.getResponse().getSelectedParses().forEach(p -> {
|
||||
if (PluginQueryManager.isPluginQuery(p.getQueryMode())
|
||||
|| "PLAIN_TEXT".equals(p.getQueryMode())) {
|
||||
if (Objects.isNull(p.getDataSet()) || Objects.isNull(p.getSqlInfo().getParsedS2SQL())) {
|
||||
return;
|
||||
}
|
||||
|
||||
formatNL2SQLParseInfo(p);
|
||||
buildParseInfoFromSQL(p);
|
||||
buildTextInfo(p);
|
||||
});
|
||||
}
|
||||
|
||||
private static void formatNL2SQLParseInfo(SemanticParseInfo parseInfo) {
|
||||
private void buildTextInfo(SemanticParseInfo parseInfo) {
|
||||
StringBuilder textBuilder = new StringBuilder();
|
||||
textBuilder.append("**数据集:** ").append(parseInfo.getDataSet().getName()).append(" ");
|
||||
List<String> metricNames = parseInfo.getMetrics().stream().map(SchemaElement::getName)
|
||||
@@ -60,4 +78,195 @@ public class ParseInfoFormatProcessor implements ParseResultProcessor {
|
||||
}
|
||||
parseInfo.setTextInfo(textBuilder.toString());
|
||||
}
|
||||
|
||||
|
||||
private void buildParseInfoFromSQL(SemanticParseInfo parseInfo) {
|
||||
SqlInfo sqlInfo = parseInfo.getSqlInfo();
|
||||
String s2SQL = sqlInfo.getCorrectedS2SQL();
|
||||
if (StringUtils.isBlank(s2SQL)) {
|
||||
return;
|
||||
}
|
||||
|
||||
parseQueryType(parseInfo);
|
||||
List<FieldExpression> expressions = SqlSelectHelper.getFilterExpression(s2SQL);
|
||||
Long dataSetId = parseInfo.getDataSetId();
|
||||
SemanticLayerService semanticLayerService =
|
||||
ContextUtils.getBean(SemanticLayerService.class);
|
||||
DataSetSchema dsSchema = semanticLayerService.getDataSetSchema(dataSetId);
|
||||
|
||||
// extract date filter from S2SQL
|
||||
try {
|
||||
if (!CollectionUtils.isEmpty(expressions)) {
|
||||
parseInfo.setDateInfo(extractDateFilter(expressions, dsSchema));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("failed to extract date range:", e);
|
||||
}
|
||||
|
||||
// extract dimension filters from S2SQL
|
||||
try {
|
||||
List<QueryFilter> queryFilters = extractDimensionFilter(dsSchema, expressions);
|
||||
parseInfo.getDimensionFilters().addAll(queryFilters);
|
||||
} catch (Exception e) {
|
||||
log.error("failed to extract dimension filters:", e);
|
||||
}
|
||||
|
||||
// extract metrics from S2SQL
|
||||
List<String> allFields =
|
||||
filterDateField(dsSchema, SqlSelectHelper.getAllSelectFields(s2SQL));
|
||||
Set<SchemaElement> metrics = matchSchemaElements(allFields, dsSchema.getMetrics());
|
||||
parseInfo.setMetrics(metrics);
|
||||
|
||||
// extract dimensions from S2SQL
|
||||
if (QueryType.AGGREGATE.equals(parseInfo.getQueryType())) {
|
||||
List<String> groupByFields = SqlSelectHelper.getGroupByFields(s2SQL);
|
||||
List<String> groupByDimensions = filterDateField(dsSchema, groupByFields);
|
||||
parseInfo.setDimensions(
|
||||
matchSchemaElements(groupByDimensions, dsSchema.getDimensions()));
|
||||
} else if (QueryType.DETAIL.equals(parseInfo.getQueryType())) {
|
||||
List<String> selectFields = SqlSelectHelper.getSelectFields(s2SQL);
|
||||
List<String> selectDimensions = filterDateField(dsSchema, selectFields);
|
||||
parseInfo
|
||||
.setDimensions(matchSchemaElements(selectDimensions, dsSchema.getDimensions()));
|
||||
}
|
||||
}
|
||||
|
||||
private Set<SchemaElement> matchSchemaElements(List<String> allFields,
|
||||
Set<SchemaElement> elements) {
|
||||
return elements.stream().filter(schemaElement -> {
|
||||
if (CollectionUtils.isEmpty(schemaElement.getAlias())) {
|
||||
return allFields.contains(schemaElement.getName());
|
||||
}
|
||||
Set<String> allFieldsSet = new HashSet<>(allFields);
|
||||
Set<String> aliasSet = new HashSet<>(schemaElement.getAlias());
|
||||
List<String> intersection =
|
||||
allFieldsSet.stream().filter(aliasSet::contains).collect(Collectors.toList());
|
||||
return allFields.contains(schemaElement.getName())
|
||||
|| !CollectionUtils.isEmpty(intersection);
|
||||
}).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
private List<String> filterDateField(DataSetSchema dataSetSchema, List<String> allFields) {
|
||||
return allFields.stream().filter(entry -> !isPartitionDimension(dataSetSchema, entry))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private List<QueryFilter> extractDimensionFilter(DataSetSchema dsSchema,
|
||||
List<FieldExpression> fieldExpressions) {
|
||||
|
||||
Map<String, SchemaElement> fieldNameToElement = getNameToElement(dsSchema);
|
||||
List<QueryFilter> result = Lists.newArrayList();
|
||||
for (FieldExpression expression : fieldExpressions) {
|
||||
QueryFilter dimensionFilter = new QueryFilter();
|
||||
dimensionFilter.setValue(expression.getFieldValue());
|
||||
SchemaElement schemaElement = fieldNameToElement.get(expression.getFieldName());
|
||||
if (Objects.isNull(schemaElement)
|
||||
|| isPartitionDimension(dsSchema, schemaElement.getName())) {
|
||||
continue;
|
||||
}
|
||||
dimensionFilter.setName(schemaElement.getName());
|
||||
dimensionFilter.setBizName(schemaElement.getBizName());
|
||||
dimensionFilter.setElementID(schemaElement.getId());
|
||||
|
||||
FilterOperatorEnum operatorEnum =
|
||||
FilterOperatorEnum.getSqlOperator(expression.getOperator());
|
||||
dimensionFilter.setOperator(operatorEnum);
|
||||
dimensionFilter.setFunction(expression.getFunction());
|
||||
result.add(dimensionFilter);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private DateConf extractDateFilter(List<FieldExpression> fieldExpressions,
|
||||
DataSetSchema dataSetSchema) {
|
||||
List<FieldExpression> dateExpressions = fieldExpressions.stream().filter(
|
||||
expression -> isPartitionDimension(dataSetSchema, expression.getFieldName()))
|
||||
.collect(Collectors.toList());
|
||||
if (CollectionUtils.isEmpty(dateExpressions)) {
|
||||
return null;
|
||||
}
|
||||
DateConf dateInfo = new DateConf();
|
||||
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
|
||||
FieldExpression firstExpression = dateExpressions.get(0);
|
||||
|
||||
FilterOperatorEnum firstOperator =
|
||||
FilterOperatorEnum.getSqlOperator(firstExpression.getOperator());
|
||||
if (FilterOperatorEnum.EQUALS.equals(firstOperator)
|
||||
&& Objects.nonNull(firstExpression.getFieldValue())) {
|
||||
dateInfo.setStartDate(firstExpression.getFieldValue().toString());
|
||||
dateInfo.setEndDate(firstExpression.getFieldValue().toString());
|
||||
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
|
||||
return dateInfo;
|
||||
}
|
||||
if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.GREATER_THAN,
|
||||
FilterOperatorEnum.GREATER_THAN_EQUALS)) {
|
||||
dateInfo.setStartDate(firstExpression.getFieldValue().toString());
|
||||
if (hasSecondDate(dateExpressions)) {
|
||||
dateInfo.setEndDate(dateExpressions.get(1).getFieldValue().toString());
|
||||
}
|
||||
}
|
||||
if (containOperators(firstExpression, firstOperator, FilterOperatorEnum.MINOR_THAN,
|
||||
FilterOperatorEnum.MINOR_THAN_EQUALS)) {
|
||||
dateInfo.setEndDate(firstExpression.getFieldValue().toString());
|
||||
if (hasSecondDate(dateExpressions)) {
|
||||
dateInfo.setStartDate(dateExpressions.get(1).getFieldValue().toString());
|
||||
}
|
||||
}
|
||||
return dateInfo;
|
||||
}
|
||||
|
||||
private static boolean isPartitionDimension(DataSetSchema dataSetSchema, String sqlFieldName) {
|
||||
if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension())
|
||||
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
|
||||
return false;
|
||||
}
|
||||
return sqlFieldName.equalsIgnoreCase(dataSetSchema.getPartitionDimension().getName());
|
||||
}
|
||||
|
||||
private boolean containOperators(FieldExpression expression, FilterOperatorEnum firstOperator,
|
||||
FilterOperatorEnum... operatorEnums) {
|
||||
return (Arrays.asList(operatorEnums).contains(firstOperator)
|
||||
&& Objects.nonNull(expression.getFieldValue()));
|
||||
}
|
||||
|
||||
private boolean hasSecondDate(List<FieldExpression> dateExpressions) {
|
||||
return dateExpressions.size() > 1
|
||||
&& Objects.nonNull(dateExpressions.get(1).getFieldValue());
|
||||
}
|
||||
|
||||
private Map<String, SchemaElement> getNameToElement(DataSetSchema dsSchema) {
|
||||
Set<SchemaElement> dimensions = dsSchema.getDimensions();
|
||||
Set<SchemaElement> metrics = dsSchema.getMetrics();
|
||||
|
||||
List<SchemaElement> allElements = Lists.newArrayList();
|
||||
allElements.addAll(dimensions);
|
||||
allElements.addAll(metrics);
|
||||
// support alias
|
||||
return allElements.stream().flatMap(schemaElement -> {
|
||||
Set<Pair<String, SchemaElement>> result = new HashSet<>();
|
||||
result.add(Pair.of(schemaElement.getName(), schemaElement));
|
||||
List<String> aliasList = schemaElement.getAlias();
|
||||
if (!org.springframework.util.CollectionUtils.isEmpty(aliasList)) {
|
||||
for (String alias : aliasList) {
|
||||
result.add(Pair.of(alias, schemaElement));
|
||||
}
|
||||
}
|
||||
return result.stream();
|
||||
}).collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (value1, value2) -> value2));
|
||||
}
|
||||
|
||||
private void parseQueryType(SemanticParseInfo parseInfo) {
|
||||
parseInfo.setQueryType(QueryType.DETAIL);
|
||||
SqlInfo sqlInfo = parseInfo.getSqlInfo();
|
||||
if (Objects.isNull(sqlInfo) || StringUtils.isBlank(sqlInfo.getCorrectedS2SQL())) {
|
||||
parseInfo.setQueryType(QueryType.DETAIL);
|
||||
}
|
||||
|
||||
// 2. AGG queryType
|
||||
if (Objects.nonNull(sqlInfo) && StringUtils.isNotBlank(sqlInfo.getParsedS2SQL())
|
||||
&& SqlSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectedS2SQL())) {
|
||||
parseInfo.setQueryType(QueryType.AGGREGATE);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -6,5 +6,7 @@ import com.tencent.supersonic.chat.server.processor.ResultProcessor;
|
||||
/** A ParseResultProcessor wraps things up before returning parsing results to the users. */
|
||||
public interface ParseResultProcessor extends ResultProcessor {
|
||||
|
||||
boolean accept(ParseContext parseContext);
|
||||
|
||||
void process(ParseContext parseContext);
|
||||
}
|
||||
|
||||
@@ -23,6 +23,11 @@ import java.util.stream.Collectors;
|
||||
@Slf4j
|
||||
public class QueryRecommendProcessor implements ParseResultProcessor {
|
||||
|
||||
@Override
|
||||
public boolean accept(ParseContext parseContext) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ParseContext parseContext) {
|
||||
CompletableFuture.runAsync(() -> doProcess(parseContext));
|
||||
@@ -56,8 +61,7 @@ public class QueryRecommendProcessor implements ParseResultProcessor {
|
||||
private void updateChatQuery(ChatQueryDO chatQueryDO) {
|
||||
ChatQueryRepository chatQueryRepository = ContextUtils.getBean(ChatQueryRepository.class);
|
||||
UpdateWrapper<ChatQueryDO> updateWrapper = new UpdateWrapper<>();
|
||||
updateWrapper.eq("question_id", chatQueryDO.getQuestionId());
|
||||
updateWrapper.set("similar_queries", chatQueryDO.getSimilarQueries());
|
||||
updateWrapper.lambda().eq(ChatQueryDO::getQuestionId, chatQueryDO.getQuestionId());
|
||||
chatQueryRepository.updateChatQuery(chatQueryDO, updateWrapper);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,11 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@Slf4j
|
||||
public class TimeCostCalcProcessor implements ParseResultProcessor {
|
||||
|
||||
@Override
|
||||
public boolean accept(ParseContext parseContext) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(ParseContext parseContext) {
|
||||
ChatParseResp parseResp = parseContext.getResponse();
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
package com.tencent.supersonic.chat.server.rest;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||
import com.tencent.supersonic.chat.server.agent.AgentToolType;
|
||||
import com.tencent.supersonic.chat.server.service.AgentService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
@@ -15,6 +15,7 @@ import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.List;
|
||||
@@ -48,8 +49,11 @@ public class AgentController {
|
||||
}
|
||||
|
||||
@RequestMapping("/getAgentList")
|
||||
public List<Agent> getAgentList() {
|
||||
return agentService.getAgents();
|
||||
public List<Agent> getAgentList(
|
||||
@RequestParam(value = "authType", required = false) AuthType authType,
|
||||
HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
|
||||
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
|
||||
return agentService.getAgents(user, authType);
|
||||
}
|
||||
|
||||
@RequestMapping("/getToolTypes")
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
package com.tencent.supersonic.chat.server.rest;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
|
||||
@@ -14,6 +11,8 @@ import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ItemResp;
|
||||
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
package com.tencent.supersonic.chat.server.rest;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
|
||||
@@ -10,6 +7,8 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatDO;
|
||||
import com.tencent.supersonic.chat.server.service.ChatManageService;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
@@ -54,7 +53,7 @@ public class ChatController {
|
||||
}
|
||||
|
||||
@PostMapping("/updateQAFeedback")
|
||||
public Boolean updateQAFeedback(@RequestParam(value = "id") Integer id,
|
||||
public Boolean updateQAFeedback(@RequestParam(value = "id") Long id,
|
||||
@RequestParam(value = "score") Integer score,
|
||||
@RequestParam(value = "feedback", required = false) String feedback) {
|
||||
return chatService.updateFeedback(id, score, feedback);
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
package com.tencent.supersonic.chat.server.rest;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.validation.Valid;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
||||
@@ -14,6 +10,9 @@ import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
|
||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import jakarta.validation.Valid;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
package com.tencent.supersonic.chat.server.rest;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryCreateReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
|
||||
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
||||
import com.tencent.supersonic.chat.server.service.MemoryService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
@@ -32,7 +31,7 @@ public class MemoryController {
|
||||
public Boolean createMemory(@RequestBody ChatMemoryCreateReq chatMemoryCreateReq,
|
||||
HttpServletRequest request, HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
memoryService.createMemory(ChatMemoryDO.builder().agentId(chatMemoryCreateReq.getAgentId())
|
||||
memoryService.createMemory(ChatMemory.builder().agentId(chatMemoryCreateReq.getAgentId())
|
||||
.s2sql(chatMemoryCreateReq.getS2sql()).question(chatMemoryCreateReq.getQuestion())
|
||||
.dbSchema(chatMemoryCreateReq.getDbSchema()).status(chatMemoryCreateReq.getStatus())
|
||||
.humanReviewRet(MemoryReviewResult.POSITIVE).createdBy(user.getName())
|
||||
@@ -49,7 +48,7 @@ public class MemoryController {
|
||||
}
|
||||
|
||||
@RequestMapping("/pageMemories")
|
||||
public PageInfo<ChatMemoryDO> pageMemories(@RequestBody PageMemoryReq pageMemoryReq) {
|
||||
public PageInfo<ChatMemory> pageMemories(@RequestBody PageMemoryReq pageMemoryReq) {
|
||||
return memoryService.pageMemories(pageMemoryReq);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
package com.tencent.supersonic.chat.server.rest;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.annotation.AuthenticationIgnore;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq;
|
||||
import com.tencent.supersonic.chat.server.plugin.ChatPlugin;
|
||||
import com.tencent.supersonic.chat.server.service.PluginService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
@@ -2,10 +2,12 @@ package com.tencent.supersonic.chat.server.service;
|
||||
|
||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface AgentService {
|
||||
List<Agent> getAgents(User user, AuthType authType);
|
||||
|
||||
List<Agent> getAgents();
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@@ -24,7 +23,7 @@ public interface ChatManageService {
|
||||
|
||||
boolean updateChatName(Long chatId, String chatName, String userName);
|
||||
|
||||
boolean updateFeedback(Integer id, Integer score, String feedback);
|
||||
boolean updateFeedback(Long id, Integer score, String feedback);
|
||||
|
||||
boolean updateChatIsTop(Long chatId, int isTop);
|
||||
|
||||
|
||||
@@ -4,27 +4,20 @@ import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
|
||||
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface MemoryService {
|
||||
void createMemory(ChatMemoryDO memory);
|
||||
void createMemory(ChatMemory memory);
|
||||
|
||||
void updateMemory(ChatMemoryUpdateReq chatMemoryUpdateReq, User user);
|
||||
|
||||
void updateMemory(ChatMemoryDO memory);
|
||||
|
||||
void enableMemory(ChatMemoryDO memory);
|
||||
|
||||
void disableMemory(ChatMemoryDO memory);
|
||||
|
||||
void batchDelete(List<Long> ids);
|
||||
|
||||
PageInfo<ChatMemoryDO> pageMemories(PageMemoryReq pageMemoryReq);
|
||||
PageInfo<ChatMemory> pageMemories(PageMemoryReq pageMemoryReq);
|
||||
|
||||
List<ChatMemoryDO> getMemories(ChatMemoryFilter chatMemoryFilter);
|
||||
List<ChatMemory> getMemories(ChatMemoryFilter chatMemoryFilter);
|
||||
|
||||
List<ChatMemoryDO> getMemoriesForLlmReview();
|
||||
}
|
||||
|
||||
@@ -6,26 +6,27 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
||||
import com.tencent.supersonic.chat.server.agent.Agent;
|
||||
import com.tencent.supersonic.chat.server.agent.VisualConfig;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.AgentDO;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
|
||||
import com.tencent.supersonic.chat.server.persistence.mapper.AgentDOMapper;
|
||||
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
||||
import com.tencent.supersonic.chat.server.service.AgentService;
|
||||
import com.tencent.supersonic.chat.server.service.ChatQueryService;
|
||||
import com.tencent.supersonic.chat.server.service.MemoryService;
|
||||
import com.tencent.supersonic.common.config.ChatModel;
|
||||
import com.tencent.supersonic.common.pojo.ChatApp;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||
import com.tencent.supersonic.common.service.ChatModelService;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@@ -41,7 +42,30 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
|
||||
@Autowired
|
||||
private ChatModelService chatModelService;
|
||||
|
||||
private ExecutorService executorService = Executors.newFixedThreadPool(1);
|
||||
@Autowired
|
||||
@Qualifier("chatExecutor")
|
||||
private ThreadPoolExecutor executor;
|
||||
|
||||
@Override
|
||||
public List<Agent> getAgents(User user, AuthType authType) {
|
||||
return getAgentDOList().stream().map(this::convert)
|
||||
.filter(agent -> filterByAuth(agent, user, authType)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private boolean filterByAuth(Agent agent, User user, AuthType authType) {
|
||||
if (user.isSuperAdmin() || user.getName().equals(agent.getCreatedBy())) {
|
||||
return true;
|
||||
}
|
||||
authType = authType == null ? AuthType.VIEWER : authType;
|
||||
switch (authType) {
|
||||
case ADMIN:
|
||||
return agent.contains(user, Agent::getAdmins);
|
||||
case VIEWER:
|
||||
default:
|
||||
return agent.contains(user, Agent::getAdmins)
|
||||
|| agent.contains(user, Agent::getViewers);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Agent> getAgents() {
|
||||
@@ -86,7 +110,7 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
|
||||
* @param agent
|
||||
*/
|
||||
private void executeAgentExamplesAsync(Agent agent) {
|
||||
executorService.execute(() -> doExecuteAgentExamples(agent));
|
||||
executor.execute(() -> doExecuteAgentExamples(agent));
|
||||
}
|
||||
|
||||
private synchronized void doExecuteAgentExamples(Agent agent) {
|
||||
@@ -99,7 +123,7 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
|
||||
ChatMemoryFilter chatMemoryFilter =
|
||||
ChatMemoryFilter.builder().agentId(agent.getId()).questions(examples).build();
|
||||
List<String> memoriesExisted = memoryService.getMemories(chatMemoryFilter).stream()
|
||||
.map(ChatMemoryDO::getQuestion).collect(Collectors.toList());
|
||||
.map(ChatMemory::getQuestion).collect(Collectors.toList());
|
||||
for (String example : examples) {
|
||||
if (memoriesExisted.contains(example)) {
|
||||
continue;
|
||||
@@ -135,6 +159,8 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
|
||||
c.setChatModelConfig(chatModelService.getChatModel(c.getChatModelId()).getConfig());
|
||||
}
|
||||
});
|
||||
agent.setAdmins(JsonUtil.toList(agentDO.getAdmin(), String.class));
|
||||
agent.setViewers(JsonUtil.toList(agentDO.getViewer(), String.class));
|
||||
return agent;
|
||||
}
|
||||
|
||||
@@ -145,6 +171,8 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO> implem
|
||||
agentDO.setExamples(JsonUtil.toString(agent.getExamples()));
|
||||
agentDO.setChatModelConfig(JsonUtil.toString(agent.getChatAppConfig()));
|
||||
agentDO.setVisualConfig(JsonUtil.toString(agent.getVisualConfig()));
|
||||
agentDO.setAdmin(JsonUtil.toString(agent.getAdmins()));
|
||||
agentDO.setViewer(JsonUtil.toString(agent.getViewers()));
|
||||
if (agentDO.getStatus() == null) {
|
||||
agentDO.setStatus(1);
|
||||
}
|
||||
|
||||
@@ -2,9 +2,9 @@ package com.tencent.supersonic.chat.server.service.impl;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatExecuteReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatParseReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.*;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ChatParseResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
@@ -15,11 +15,12 @@ import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.QueryDO;
|
||||
import com.tencent.supersonic.chat.server.persistence.repository.ChatQueryRepository;
|
||||
import com.tencent.supersonic.chat.server.persistence.repository.ChatRepository;
|
||||
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
||||
import com.tencent.supersonic.chat.server.service.ChatManageService;
|
||||
import com.tencent.supersonic.chat.server.service.MemoryService;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -38,6 +39,8 @@ public class ChatManageServiceImpl implements ChatManageService {
|
||||
private ChatRepository chatRepository;
|
||||
@Autowired
|
||||
private ChatQueryRepository chatQueryRepository;
|
||||
@Autowired
|
||||
private MemoryService memoryService;
|
||||
|
||||
@Override
|
||||
public Long addChat(User user, String chatName, Integer agentId) {
|
||||
@@ -64,11 +67,28 @@ public class ChatManageServiceImpl implements ChatManageService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean updateFeedback(Integer id, Integer score, String feedback) {
|
||||
public boolean updateFeedback(Long id, Integer score, String feedback) {
|
||||
QueryDO intelligentQueryDO = new QueryDO();
|
||||
intelligentQueryDO.setId(id);
|
||||
intelligentQueryDO.setQuestionId(id);
|
||||
intelligentQueryDO.setScore(score);
|
||||
intelligentQueryDO.setFeedback(feedback);
|
||||
|
||||
// enable or disable memory based on user feedback
|
||||
if (score >= 5 || score <= 1) {
|
||||
ChatMemoryFilter memoryFilter = ChatMemoryFilter.builder().queryId(id).build();
|
||||
List<ChatMemory> memories = memoryService.getMemories(memoryFilter);
|
||||
memories.forEach(m -> {
|
||||
MemoryStatus status = score >= 5 ? MemoryStatus.ENABLED : MemoryStatus.DISABLED;
|
||||
MemoryReviewResult reviewResult =
|
||||
score >= 5 ? MemoryReviewResult.POSITIVE : MemoryReviewResult.NEGATIVE;
|
||||
ChatMemoryUpdateReq memoryUpdateReq = ChatMemoryUpdateReq.builder().id(m.getId())
|
||||
.status(status).humanReviewRet(reviewResult)
|
||||
.humanReviewCmt("Reviewed as per user feedback").build();
|
||||
memoryService.updateMemory(memoryUpdateReq, User.getDefaultUser());
|
||||
});
|
||||
}
|
||||
|
||||
return chatRepository.updateFeedback(intelligentQueryDO);
|
||||
}
|
||||
|
||||
|
||||
@@ -18,18 +18,12 @@ import com.tencent.supersonic.chat.server.service.ChatManageService;
|
||||
import com.tencent.supersonic.chat.server.service.ChatQueryService;
|
||||
import com.tencent.supersonic.chat.server.util.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.server.util.QueryReqConverter;
|
||||
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
|
||||
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
|
||||
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
|
||||
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
|
||||
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
|
||||
import com.tencent.supersonic.common.jsqlparser.*;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateUtils;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
|
||||
import com.tencent.supersonic.headless.api.pojo.EntityInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
|
||||
@@ -50,11 +44,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import net.sf.jsqlparser.expression.Expression;
|
||||
import net.sf.jsqlparser.expression.LongValue;
|
||||
import net.sf.jsqlparser.expression.StringValue;
|
||||
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
|
||||
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
|
||||
import net.sf.jsqlparser.expression.operators.relational.InExpression;
|
||||
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
|
||||
import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
|
||||
import net.sf.jsqlparser.expression.operators.relational.*;
|
||||
import net.sf.jsqlparser.schema.Column;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
@@ -62,14 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@@ -108,11 +91,17 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
Long queryId = chatParseReq.getQueryId();
|
||||
if (Objects.isNull(queryId)) {
|
||||
queryId = chatManageService.createChatQuery(chatParseReq);
|
||||
chatParseReq.setQueryId(queryId);
|
||||
}
|
||||
|
||||
ParseContext parseContext = buildParseContext(chatParseReq, new ChatParseResp(queryId));
|
||||
chatQueryParsers.forEach(p -> p.parse(parseContext));
|
||||
parseResultProcessors.forEach(p -> p.process(parseContext));
|
||||
|
||||
for (ParseResultProcessor processor : parseResultProcessors) {
|
||||
if (processor.accept(parseContext)) {
|
||||
processor.process(parseContext);
|
||||
}
|
||||
}
|
||||
|
||||
if (!parseContext.needFeedback()) {
|
||||
chatManageService.batchAddParse(chatParseReq, parseContext.getResponse());
|
||||
@@ -133,9 +122,12 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
}
|
||||
}
|
||||
|
||||
executeContext.setResponse(queryResult);
|
||||
if (queryResult != null) {
|
||||
for (ExecuteResultProcessor processor : executeResultProcessors) {
|
||||
processor.process(executeContext, queryResult);
|
||||
if (processor.accept(executeContext)) {
|
||||
processor.process(executeContext);
|
||||
}
|
||||
}
|
||||
saveQueryResult(chatExecuteReq, queryResult);
|
||||
}
|
||||
@@ -198,7 +190,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
handleRuleQueryMode(semanticQuery, dataSetSchema, user);
|
||||
}
|
||||
|
||||
return executeQuery(semanticQuery, user, dataSetSchema);
|
||||
return executeQuery(semanticQuery, user);
|
||||
}
|
||||
|
||||
private List<String> getFieldsFromSql(SemanticParseInfo parseInfo) {
|
||||
@@ -212,20 +204,22 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
private void handleLLMQueryMode(ChatQueryDataReq chatQueryDataReq, SemanticQuery semanticQuery,
|
||||
DataSetSchema dataSetSchema, User user) throws Exception {
|
||||
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
|
||||
List<String> fields = getFieldsFromSql(parseInfo);
|
||||
if (checkMetricReplace(fields, chatQueryDataReq.getMetrics())) {
|
||||
log.info("llm begin replace metrics!");
|
||||
String rebuiltS2SQL;
|
||||
if (checkMetricReplace(chatQueryDataReq, parseInfo)) {
|
||||
log.info("rebuild S2SQL with adjusted metrics!");
|
||||
SchemaElement metricToReplace = chatQueryDataReq.getMetrics().iterator().next();
|
||||
replaceMetrics(parseInfo, metricToReplace);
|
||||
rebuiltS2SQL = replaceMetrics(parseInfo, metricToReplace);
|
||||
} else {
|
||||
log.info("llm begin revise filters!");
|
||||
String correctorSql = reviseCorrectS2SQL(chatQueryDataReq, parseInfo, dataSetSchema);
|
||||
parseInfo.getSqlInfo().setCorrectedS2SQL(correctorSql);
|
||||
semanticQuery.setParseInfo(parseInfo);
|
||||
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
|
||||
SemanticTranslateResp explain = semanticLayerService.translate(semanticQueryReq, user);
|
||||
parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL());
|
||||
log.info("rebuild S2SQL with adjusted filters!");
|
||||
rebuiltS2SQL = replaceFilters(chatQueryDataReq, parseInfo, dataSetSchema);
|
||||
}
|
||||
// reset SqlInfo and request re-translation
|
||||
parseInfo.getSqlInfo().setCorrectedS2SQL(rebuiltS2SQL);
|
||||
parseInfo.getSqlInfo().setParsedS2SQL(rebuiltS2SQL);
|
||||
parseInfo.getSqlInfo().setQuerySQL(null);
|
||||
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
|
||||
SemanticTranslateResp explain = semanticLayerService.translate(semanticQueryReq, user);
|
||||
parseInfo.getSqlInfo().setQuerySQL(explain.getQuerySQL());
|
||||
}
|
||||
|
||||
private void handleRuleQueryMode(SemanticQuery semanticQuery, DataSetSchema dataSetSchema,
|
||||
@@ -233,23 +227,22 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
log.info("rule begin replace metrics and revise filters!");
|
||||
validFilter(semanticQuery.getParseInfo().getDimensionFilters());
|
||||
validFilter(semanticQuery.getParseInfo().getMetricFilters());
|
||||
semanticQuery.initS2Sql(dataSetSchema, user);
|
||||
semanticQuery.buildS2Sql(dataSetSchema);
|
||||
}
|
||||
|
||||
private QueryResult executeQuery(SemanticQuery semanticQuery, User user,
|
||||
DataSetSchema dataSetSchema) throws Exception {
|
||||
private QueryResult executeQuery(SemanticQuery semanticQuery, User user) throws Exception {
|
||||
SemanticQueryReq semanticQueryReq = semanticQuery.buildSemanticQueryReq();
|
||||
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
|
||||
QueryResult queryResult = doExecution(semanticQueryReq, parseInfo.getQueryMode(), user);
|
||||
queryResult.setChatContext(semanticQuery.getParseInfo());
|
||||
SemanticLayerService semanticService = ContextUtils.getBean(SemanticLayerService.class);
|
||||
EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, dataSetSchema, user);
|
||||
queryResult.setEntityInfo(entityInfo);
|
||||
parseInfo.getSqlInfo().setQuerySQL(queryResult.getQuerySql());
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
private boolean checkMetricReplace(List<String> oriFields, Set<SchemaElement> metrics) {
|
||||
private boolean checkMetricReplace(ChatQueryDataReq chatQueryDataReq,
|
||||
SemanticParseInfo parseInfo) {
|
||||
List<String> oriFields = getFieldsFromSql(parseInfo);
|
||||
Set<SchemaElement> metrics = chatQueryDataReq.getMetrics();
|
||||
if (CollectionUtils.isEmpty(oriFields) || CollectionUtils.isEmpty(metrics)) {
|
||||
return false;
|
||||
}
|
||||
@@ -258,7 +251,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
return !oriFields.containsAll(metricNames);
|
||||
}
|
||||
|
||||
private String reviseCorrectS2SQL(ChatQueryDataReq queryData, SemanticParseInfo parseInfo,
|
||||
private String replaceFilters(ChatQueryDataReq queryData, SemanticParseInfo parseInfo,
|
||||
DataSetSchema dataSetSchema) {
|
||||
String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL();
|
||||
log.info("correctorSql before replacing:{}", correctorSql);
|
||||
@@ -296,7 +289,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
return correctorSql;
|
||||
}
|
||||
|
||||
private void replaceMetrics(SemanticParseInfo parseInfo, SchemaElement metric) {
|
||||
private String replaceMetrics(SemanticParseInfo parseInfo, SchemaElement metric) {
|
||||
List<String> oriMetrics = parseInfo.getMetrics().stream().map(SchemaElement::getName)
|
||||
.collect(Collectors.toList());
|
||||
String correctorSql = parseInfo.getSqlInfo().getCorrectedS2SQL();
|
||||
@@ -308,7 +301,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
correctorSql = SqlReplaceHelper.replaceAggFields(correctorSql, fieldMap);
|
||||
}
|
||||
log.info("after replaceMetrics:{}", correctorSql);
|
||||
parseInfo.getSqlInfo().setCorrectedS2SQL(correctorSql);
|
||||
return correctorSql;
|
||||
}
|
||||
|
||||
private QueryResult doExecution(SemanticQueryReq semanticQueryReq, String queryMode, User user)
|
||||
@@ -483,6 +476,9 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
}
|
||||
|
||||
private void mergeParseInfo(SemanticParseInfo parseInfo, ChatQueryDataReq queryData) {
|
||||
if (Objects.nonNull(queryData.getDateInfo())) {
|
||||
parseInfo.setDateInfo(queryData.getDateInfo());
|
||||
}
|
||||
if (LLMSqlQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) {
|
||||
return;
|
||||
}
|
||||
@@ -498,9 +494,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
|
||||
if (!CollectionUtils.isEmpty(queryData.getMetricFilters())) {
|
||||
parseInfo.setMetricFilters(queryData.getMetricFilters());
|
||||
}
|
||||
if (Objects.nonNull(queryData.getDateInfo())) {
|
||||
parseInfo.setDateInfo(queryData.getDateInfo());
|
||||
}
|
||||
|
||||
parseInfo.setSqlInfo(new SqlInfo());
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.Entity;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
|
||||
@@ -16,7 +15,6 @@ import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ChatDetailRichConfigResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ItemVisibilityInfo;
|
||||
import com.tencent.supersonic.chat.server.config.ChatConfig;
|
||||
import com.tencent.supersonic.chat.server.persistence.repository.ChatConfigRepository;
|
||||
@@ -88,16 +86,6 @@ public class ConfigServiceImpl implements ConfigService {
|
||||
return configEditCmd.getId();
|
||||
}
|
||||
|
||||
public ItemNameVisibilityInfo getVisibilityByModelId(Long modelId) {
|
||||
ChatConfigResp chatConfigResp = fetchConfigByModelId(modelId);
|
||||
ChatConfig chatConfig = new ChatConfig();
|
||||
chatConfig.setModelId(modelId);
|
||||
chatConfig.setChatAggConfig(chatConfigResp.getChatAggConfig());
|
||||
chatConfig.setChatDetailConfig(chatConfigResp.getChatDetailConfig());
|
||||
ItemNameVisibilityInfo itemNameVisibility = getItemNameVisibility(chatConfig);
|
||||
return itemNameVisibility;
|
||||
}
|
||||
|
||||
public ItemNameVisibilityInfo getItemNameVisibility(ChatConfig chatConfig) {
|
||||
Long modelId = chatConfig.getModelId();
|
||||
|
||||
@@ -240,19 +228,6 @@ public class ConfigServiceImpl implements ConfigService {
|
||||
return detailRichConfig;
|
||||
}
|
||||
|
||||
private EntityRichInfoResp generateRichEntity(Entity entity, DataSetSchema modelSchema) {
|
||||
EntityRichInfoResp entityRichInfo = new EntityRichInfoResp();
|
||||
if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) {
|
||||
return entityRichInfo;
|
||||
}
|
||||
BeanUtils.copyProperties(entity, entityRichInfo);
|
||||
Map<Long, SchemaElement> dimIdAndRespPair = modelSchema.getDimensions().stream().collect(
|
||||
Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
|
||||
|
||||
entityRichInfo.setDimItem(dimIdAndRespPair.get(entity.getEntityId()));
|
||||
return entityRichInfo;
|
||||
}
|
||||
|
||||
private ChatAggRichConfigResp fillChatAggRichConfig(DataSetSchema modelSchema,
|
||||
ChatConfigResp chatConfigResp) {
|
||||
if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) {
|
||||
@@ -327,7 +302,7 @@ public class ConfigServiceImpl implements ConfigService {
|
||||
}
|
||||
Map<Long, SchemaElement> dimIdAndRespPair = modelSchema.getDimensions().stream().collect(
|
||||
Collectors.toMap(SchemaElement::getId, Function.identity(), (k1, k2) -> k1));
|
||||
knowledgeInfos.stream().forEach(knowledgeInfo -> {
|
||||
knowledgeInfos.forEach(knowledgeInfo -> {
|
||||
if (Objects.nonNull(knowledgeInfo)) {
|
||||
SchemaElement dimSchemaResp = dimIdAndRespPair.get(knowledgeInfo.getItemId());
|
||||
if (Objects.nonNull(dimSchemaResp)) {
|
||||
|
||||
@@ -1,27 +1,33 @@
|
||||
package com.tencent.supersonic.chat.server.service.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.github.pagehelper.PageHelper;
|
||||
import com.github.pagehelper.PageInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryReviewResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.enums.MemoryStatus;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatMemoryUpdateReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.PageMemoryReq;
|
||||
import com.tencent.supersonic.chat.server.persistence.dataobject.ChatMemoryDO;
|
||||
import com.tencent.supersonic.chat.server.persistence.mapper.ChatMemoryMapper;
|
||||
import com.tencent.supersonic.chat.server.persistence.repository.ChatMemoryRepository;
|
||||
import com.tencent.supersonic.chat.server.pojo.ChatMemory;
|
||||
import com.tencent.supersonic.chat.server.service.MemoryService;
|
||||
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||
import com.tencent.supersonic.common.pojo.Text2SQLExemplar;
|
||||
import com.tencent.supersonic.common.pojo.User;
|
||||
import com.tencent.supersonic.common.service.ExemplarService;
|
||||
import com.tencent.supersonic.common.util.BeanMapper;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class MemoryServiceImpl implements MemoryService {
|
||||
@@ -29,6 +35,9 @@ public class MemoryServiceImpl implements MemoryService {
|
||||
@Autowired
|
||||
private ChatMemoryRepository chatMemoryRepository;
|
||||
|
||||
@Autowired
|
||||
private ChatMemoryMapper chatMemoryMapper;
|
||||
|
||||
@Autowired
|
||||
private ExemplarService exemplarService;
|
||||
|
||||
@@ -36,34 +45,52 @@ public class MemoryServiceImpl implements MemoryService {
|
||||
private EmbeddingConfig embeddingConfig;
|
||||
|
||||
@Override
|
||||
public void createMemory(ChatMemoryDO memory) {
|
||||
public void createMemory(ChatMemory memory) {
|
||||
// if an existing enabled memory has the same question, just skip
|
||||
List<ChatMemoryDO> memories =
|
||||
List<ChatMemory> memories =
|
||||
getMemories(ChatMemoryFilter.builder().agentId(memory.getAgentId())
|
||||
.question(memory.getQuestion()).status(MemoryStatus.ENABLED).build());
|
||||
if (memories.size() == 0) {
|
||||
chatMemoryRepository.createMemory(memory);
|
||||
if (memories.isEmpty()) {
|
||||
ChatMemoryDO memoryDO = getMemoryDO(memory);
|
||||
chatMemoryRepository.createMemory(memoryDO);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateMemory(ChatMemoryUpdateReq chatMemoryUpdateReq, User user) {
|
||||
ChatMemoryDO chatMemoryDO = chatMemoryRepository.getMemory(chatMemoryUpdateReq.getId());
|
||||
chatMemoryDO.setUpdatedBy(user.getName());
|
||||
chatMemoryDO.setUpdatedAt(new Date());
|
||||
BeanMapper.mapper(chatMemoryUpdateReq, chatMemoryDO);
|
||||
boolean hadEnabled = MemoryStatus.ENABLED.equals(chatMemoryDO.getStatus());
|
||||
boolean hadEnabled =
|
||||
MemoryStatus.ENABLED.toString().equals(chatMemoryDO.getStatus().trim());
|
||||
if (MemoryStatus.ENABLED.equals(chatMemoryUpdateReq.getStatus()) && !hadEnabled) {
|
||||
enableMemory(chatMemoryDO);
|
||||
} else if (MemoryStatus.DISABLED.equals(chatMemoryUpdateReq.getStatus()) && hadEnabled) {
|
||||
disableMemory(chatMemoryDO);
|
||||
}
|
||||
updateMemory(chatMemoryDO);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateMemory(ChatMemoryDO memory) {
|
||||
chatMemoryRepository.updateMemory(memory);
|
||||
LambdaUpdateWrapper<ChatMemoryDO> updateWrapper = new LambdaUpdateWrapper<>();
|
||||
updateWrapper.eq(ChatMemoryDO::getId, chatMemoryDO.getId());
|
||||
if (Objects.nonNull(chatMemoryUpdateReq.getStatus())) {
|
||||
updateWrapper.set(ChatMemoryDO::getStatus, chatMemoryUpdateReq.getStatus());
|
||||
}
|
||||
if (Objects.nonNull(chatMemoryUpdateReq.getLlmReviewRet())) {
|
||||
updateWrapper.set(ChatMemoryDO::getLlmReviewRet,
|
||||
chatMemoryUpdateReq.getLlmReviewRet().toString());
|
||||
}
|
||||
if (Objects.nonNull(chatMemoryUpdateReq.getLlmReviewCmt())) {
|
||||
updateWrapper.set(ChatMemoryDO::getLlmReviewCmt, chatMemoryUpdateReq.getLlmReviewCmt());
|
||||
}
|
||||
if (Objects.nonNull(chatMemoryUpdateReq.getHumanReviewRet())) {
|
||||
updateWrapper.set(ChatMemoryDO::getHumanReviewRet,
|
||||
chatMemoryUpdateReq.getHumanReviewRet().toString());
|
||||
}
|
||||
if (Objects.nonNull(chatMemoryUpdateReq.getHumanReviewCmt())) {
|
||||
updateWrapper.set(ChatMemoryDO::getHumanReviewCmt,
|
||||
chatMemoryUpdateReq.getHumanReviewCmt());
|
||||
}
|
||||
updateWrapper.set(ChatMemoryDO::getUpdatedAt, new Date());
|
||||
updateWrapper.set(ChatMemoryDO::getUpdatedBy, user.getName());
|
||||
|
||||
chatMemoryMapper.update(updateWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -72,7 +99,7 @@ public class MemoryServiceImpl implements MemoryService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public PageInfo<ChatMemoryDO> pageMemories(PageMemoryReq pageMemoryReq) {
|
||||
public PageInfo<ChatMemory> pageMemories(PageMemoryReq pageMemoryReq) {
|
||||
ChatMemoryFilter chatMemoryFilter = pageMemoryReq.getChatMemoryFilter();
|
||||
chatMemoryFilter.setSort(pageMemoryReq.getSort());
|
||||
chatMemoryFilter.setOrderCondition(pageMemoryReq.getOrderCondition());
|
||||
@@ -81,11 +108,14 @@ public class MemoryServiceImpl implements MemoryService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ChatMemoryDO> getMemories(ChatMemoryFilter chatMemoryFilter) {
|
||||
public List<ChatMemory> getMemories(ChatMemoryFilter chatMemoryFilter) {
|
||||
QueryWrapper<ChatMemoryDO> queryWrapper = new QueryWrapper<>();
|
||||
if (chatMemoryFilter.getAgentId() != null) {
|
||||
queryWrapper.lambda().eq(ChatMemoryDO::getAgentId, chatMemoryFilter.getAgentId());
|
||||
}
|
||||
if (chatMemoryFilter.getQueryId() != null) {
|
||||
queryWrapper.lambda().eq(ChatMemoryDO::getQueryId, chatMemoryFilter.getQueryId());
|
||||
}
|
||||
if (StringUtils.isNotBlank(chatMemoryFilter.getQuestion())) {
|
||||
queryWrapper.lambda().like(ChatMemoryDO::getQuestion, chatMemoryFilter.getQuestion());
|
||||
}
|
||||
@@ -109,32 +139,52 @@ public class MemoryServiceImpl implements MemoryService {
|
||||
queryWrapper.orderBy(true, chatMemoryFilter.isAsc(),
|
||||
chatMemoryFilter.getOrderCondition());
|
||||
}
|
||||
return chatMemoryRepository.getMemories(queryWrapper);
|
||||
List<ChatMemoryDO> chatMemoryDOS = chatMemoryRepository.getMemories(queryWrapper);
|
||||
return chatMemoryDOS.stream().map(this::getMemory).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ChatMemoryDO> getMemoriesForLlmReview() {
|
||||
QueryWrapper<ChatMemoryDO> queryWrapper = new QueryWrapper<>();
|
||||
queryWrapper.lambda().eq(ChatMemoryDO::getStatus, MemoryStatus.PENDING)
|
||||
.isNull(ChatMemoryDO::getLlmReviewRet);
|
||||
return chatMemoryRepository.getMemories(queryWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enableMemory(ChatMemoryDO memory) {
|
||||
memory.setStatus(MemoryStatus.ENABLED);
|
||||
memory.setStatus(MemoryStatus.ENABLED.toString());
|
||||
exemplarService.storeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
|
||||
Text2SQLExemplar.builder().question(memory.getQuestion())
|
||||
.sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema())
|
||||
.sql(memory.getS2sql()).build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void disableMemory(ChatMemoryDO memory) {
|
||||
memory.setStatus(MemoryStatus.DISABLED);
|
||||
memory.setStatus(MemoryStatus.DISABLED.toString());
|
||||
exemplarService.removeExemplar(embeddingConfig.getMemoryCollectionName(memory.getAgentId()),
|
||||
Text2SQLExemplar.builder().question(memory.getQuestion())
|
||||
.sideInfo(memory.getSideInfo()).dbSchema(memory.getDbSchema())
|
||||
.sql(memory.getS2sql()).build());
|
||||
}
|
||||
|
||||
private ChatMemoryDO getMemoryDO(ChatMemory memory) {
|
||||
ChatMemoryDO memoryDO = new ChatMemoryDO();
|
||||
BeanUtils.copyProperties(memory, memoryDO);
|
||||
memoryDO.setStatus(memory.getStatus().toString().trim());
|
||||
if (Objects.nonNull(memory.getHumanReviewRet())) {
|
||||
memoryDO.setHumanReviewRet(memory.getHumanReviewRet().toString().trim());
|
||||
}
|
||||
if (Objects.nonNull(memory.getLlmReviewRet())) {
|
||||
memoryDO.setLlmReviewRet(memory.getLlmReviewRet().toString().trim());
|
||||
}
|
||||
|
||||
return memoryDO;
|
||||
}
|
||||
|
||||
private ChatMemory getMemory(ChatMemoryDO memoryDO) {
|
||||
ChatMemory memory = new ChatMemory();
|
||||
BeanUtils.copyProperties(memoryDO, memory);
|
||||
memory.setStatus(MemoryStatus.valueOf(memoryDO.getStatus().trim()));
|
||||
if (Objects.nonNull(memoryDO.getHumanReviewRet())) {
|
||||
memory.setHumanReviewRet(
|
||||
MemoryReviewResult.valueOf(memoryDO.getHumanReviewRet().trim()));
|
||||
}
|
||||
if (Objects.nonNull(memoryDO.getLlmReviewRet())) {
|
||||
memory.setLlmReviewRet(MemoryReviewResult.valueOf(memoryDO.getLlmReviewRet().trim()));
|
||||
}
|
||||
return memory;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ public class ResultFormatter {
|
||||
}
|
||||
for (Map<String, Object> row : queryResults) {
|
||||
for (QueryColumn column : queryColumns) {
|
||||
String columnKey = column.getNameEn();
|
||||
String columnKey = column.getBizName();
|
||||
Object value = row.get(columnKey);
|
||||
table.append("| ").append(value != null ? value.toString() : "").append(" ");
|
||||
}
|
||||
|
||||
@@ -20,54 +20,6 @@
|
||||
<result column="updated_at" property="updatedAt"/>
|
||||
</resultMap>
|
||||
|
||||
<insert id="addConfig"
|
||||
parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.ChatConfigDO"
|
||||
useGeneratedKeys="true" keyProperty="id">
|
||||
insert into s2_chat_config
|
||||
(
|
||||
model_id, `chat_detail_config`, chat_agg_config, recommended_questions, status, llm_examples, created_by, updated_by, created_at, updated_at
|
||||
)
|
||||
values
|
||||
(
|
||||
#{modelId}, #{chatDetailConfig}, #{chatAggConfig}, #{recommendedQuestions}, #{status}, #{llmExamples}, #{createdBy}, #{updatedBy}, #{createdAt}, #{updatedAt}
|
||||
)
|
||||
</insert>
|
||||
|
||||
|
||||
<update id="editConfig">
|
||||
update s2_chat_config
|
||||
<set>
|
||||
`updated_at` = #{updatedAt} ,
|
||||
<if test="chatDetailConfig != null and chatDetailConfig != ''">
|
||||
`chat_detail_config` = #{chatDetailConfig} ,
|
||||
</if>
|
||||
<if test="chatAggConfig != null and chatAggConfig != ''">
|
||||
chat_agg_config = #{chatAggConfig} ,
|
||||
</if>
|
||||
<if test="recommendedQuestions != null and recommendedQuestions != ''">
|
||||
recommended_questions = #{recommendedQuestions} ,
|
||||
</if>
|
||||
<if test="status != null and status != ''">
|
||||
status = #{status} ,
|
||||
</if>
|
||||
<if test="updatedBy != null and updatedBy != ''">
|
||||
updated_by = #{updatedBy} ,
|
||||
</if>
|
||||
<if test="llmExamples != null and llmExamples != ''">
|
||||
llm_examples = #{llmExamples} ,
|
||||
</if>
|
||||
</set>
|
||||
|
||||
<where>
|
||||
<if test="id != null and id != ''">
|
||||
id = #{id}
|
||||
</if>
|
||||
<if test="modelId != null and modelId != ''">
|
||||
and model_id = #{modelId}
|
||||
</if>
|
||||
</where>
|
||||
</update>
|
||||
|
||||
<select id="search" resultMap="chaConfigDO">
|
||||
select *
|
||||
from s2_chat_config
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
type="com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO">
|
||||
<id column="chat_id" property="chatId"/>
|
||||
<result column="modified_at" property="modifiedAt"/>
|
||||
<result column="user" property="user"/>
|
||||
<result column="query_user" property="queryUser"/>
|
||||
<result column="query_text" property="queryText"/>
|
||||
<result column="semantic_parse" property="semanticParse"/>
|
||||
<!--<result column="ext_data" property="extData"/>-->
|
||||
@@ -20,11 +20,4 @@
|
||||
from s2_chat_context where chat_id=#{chatId} limit 1
|
||||
</select>
|
||||
|
||||
<insert id="addContext" parameterType="com.tencent.supersonic.chat.server.persistence.dataobject.ChatContextDO" >
|
||||
insert into s2_chat_context (chat_id,user,query_text,semantic_parse) values (#{chatId}, #{user},#{queryText}, #{semanticParse})
|
||||
</insert>
|
||||
<update id="updateContext">
|
||||
update s2_chat_context set query_text=#{queryText},semantic_parse=#{semanticParse} where chat_id=#{chatId}
|
||||
</update>
|
||||
|
||||
</mapper>
|
||||
</mapper>
|
||||
|
||||
@@ -31,21 +31,21 @@
|
||||
|
||||
<!--http-->
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.5.6</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpmime</artifactId>
|
||||
<version>${httpmime.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<groupId>org.apache.httpcomponents.client5</groupId>
|
||||
<artifactId>httpclient5</artifactId>
|
||||
<version>${httpclient5.version}</version> <!-- 请确认使用最新稳定版本 -->
|
||||
</dependency>
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>org.apache.httpcomponents</groupId>-->
|
||||
<!-- <artifactId>httpmime</artifactId>-->
|
||||
<!-- <version>${httpmime.version}</version>-->
|
||||
<!-- <exclusions>-->
|
||||
<!-- <exclusion>-->
|
||||
<!-- <artifactId>httpclient</artifactId>-->
|
||||
<!-- <groupId>org.apache.httpcomponents</groupId>-->
|
||||
<!-- </exclusion>-->
|
||||
<!-- </exclusions>-->
|
||||
<!-- </dependency>-->
|
||||
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
@@ -130,7 +130,7 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>com.baomidou</groupId>
|
||||
<artifactId>mybatis-plus-boot-starter</artifactId>
|
||||
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
|
||||
<version>${mybatis.plus.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
@@ -19,19 +19,6 @@ public class Term {
|
||||
this.nature = nature;
|
||||
}
|
||||
|
||||
public Term(String word, Nature nature, int offset) {
|
||||
this.word = word;
|
||||
this.nature = nature;
|
||||
this.offset = offset;
|
||||
}
|
||||
|
||||
public Term(String word, Nature nature, int offset, int frequency) {
|
||||
this.word = word;
|
||||
this.nature = nature;
|
||||
this.offset = offset;
|
||||
this.frequency = frequency;
|
||||
}
|
||||
|
||||
public int length() {
|
||||
return this.word.length();
|
||||
}
|
||||
|
||||
@@ -80,6 +80,9 @@ public class Configuration {
|
||||
.setQuoting(Quoting.SINGLE_QUOTE).setQuotedCasing(Casing.TO_UPPER)
|
||||
.setUnquotedCasing(Casing.TO_UPPER).setConformance(sqlDialect.getConformance())
|
||||
.setLex(Lex.BIG_QUERY);
|
||||
if (EngineType.HANADB.equals(engineType)) {
|
||||
parserConfig = parserConfig.setQuoting(Quoting.DOUBLE_QUOTE);
|
||||
}
|
||||
parserConfig = parserConfig.setQuotedCasing(Casing.UNCHANGED);
|
||||
parserConfig = parserConfig.setUnquotedCasing(Casing.UNCHANGED);
|
||||
return parserConfig.build();
|
||||
|
||||
@@ -21,6 +21,11 @@ public class SqlDialectFactory {
|
||||
.withDatabaseProduct(DatabaseProduct.BIG_QUERY).withLiteralQuoteString("'")
|
||||
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(false);
|
||||
public static final Context HANADB_CONTEXT =
|
||||
SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
|
||||
.withLiteralQuoteString("'").withIdentifierQuoteString("\"")
|
||||
.withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
|
||||
.withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
|
||||
private static Map<EngineType, SemanticSqlDialect> sqlDialectMap;
|
||||
|
||||
static {
|
||||
@@ -29,6 +34,7 @@ public class SqlDialectFactory {
|
||||
sqlDialectMap.put(EngineType.MYSQL, new SemanticSqlDialect(DEFAULT_CONTEXT));
|
||||
sqlDialectMap.put(EngineType.H2, new SemanticSqlDialect(DEFAULT_CONTEXT));
|
||||
sqlDialectMap.put(EngineType.POSTGRESQL, new SemanticSqlDialect(POSTGRESQL_CONTEXT));
|
||||
sqlDialectMap.put(EngineType.HANADB, new SemanticSqlDialect(HANADB_CONTEXT));
|
||||
}
|
||||
|
||||
public static SemanticSqlDialect getSqlDialect(EngineType engineType) {
|
||||
|
||||
@@ -38,14 +38,14 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
|
||||
new Parameter("s2.embedding.store.timeout", "60", "超时时间(秒)", "", "number", MODULE_NAME);
|
||||
|
||||
public static final Parameter EMBEDDING_STORE_DIMENSION =
|
||||
new Parameter("s2.embedding.store.dimension", "", "纬度", "", "number", MODULE_NAME, null,
|
||||
getDimensionDependency());
|
||||
new Parameter("s2.embedding.store.dimension", "", "向量维度", "", "number", MODULE_NAME,
|
||||
null, getDimensionDependency());
|
||||
public static final Parameter EMBEDDING_STORE_DATABASE_NAME =
|
||||
new Parameter("s2.embedding.store.databaseName", "", "DatabaseName", "", "string",
|
||||
MODULE_NAME, null, getDatabaseNameDependency());
|
||||
|
||||
public static final Parameter EMBEDDING_STORE_POST = new Parameter("s2.embedding.store.post",
|
||||
"", "端口", "", "number", MODULE_NAME, null, getPostDependency());
|
||||
public static final Parameter EMBEDDING_STORE_POST = new Parameter("s2.embedding.store.port",
|
||||
"", "端口", "", "number", MODULE_NAME, null, getPortDependency());
|
||||
|
||||
public static final Parameter EMBEDDING_STORE_USER = new Parameter("s2.embedding.store.user",
|
||||
"", "用户名", "", "string", MODULE_NAME, null, getUserDependency());
|
||||
@@ -101,10 +101,8 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
|
||||
|
||||
private static List<Parameter.Dependency> getApiKeyDependency() {
|
||||
return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
|
||||
Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
|
||||
EmbeddingStoreType.PGVECTOR.name()),
|
||||
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO,
|
||||
EmbeddingStoreType.PGVECTOR.name(), DEMO));
|
||||
Lists.newArrayList(EmbeddingStoreType.MILVUS.name()),
|
||||
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), DEMO));
|
||||
}
|
||||
|
||||
private static List<Parameter.Dependency> getPathDependency() {
|
||||
@@ -118,7 +116,7 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
|
||||
Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
|
||||
EmbeddingStoreType.PGVECTOR.name()),
|
||||
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "384",
|
||||
EmbeddingStoreType.PGVECTOR.name(), "768"));
|
||||
EmbeddingStoreType.PGVECTOR.name(), "512"));
|
||||
}
|
||||
|
||||
private static List<Parameter.Dependency> getDatabaseNameDependency() {
|
||||
@@ -129,7 +127,7 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
|
||||
EmbeddingStoreType.PGVECTOR.name(), "postgres"));
|
||||
}
|
||||
|
||||
private static List<Parameter.Dependency> getPostDependency() {
|
||||
private static List<Parameter.Dependency> getPortDependency() {
|
||||
return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
|
||||
Lists.newArrayList(EmbeddingStoreType.PGVECTOR.name()),
|
||||
ImmutableMap.of(EmbeddingStoreType.PGVECTOR.name(), "54333"));
|
||||
@@ -140,12 +138,14 @@ public class EmbeddingStoreParameterConfig extends ParameterConfig {
|
||||
Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
|
||||
EmbeddingStoreType.PGVECTOR.name()),
|
||||
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus",
|
||||
EmbeddingStoreType.PGVECTOR.name(), "pgvector"));
|
||||
EmbeddingStoreType.PGVECTOR.name(), "postgres"));
|
||||
}
|
||||
|
||||
private static List<Parameter.Dependency> getPasswordDependency() {
|
||||
return getDependency(EMBEDDING_STORE_PROVIDER.getName(),
|
||||
Lists.newArrayList(EmbeddingStoreType.MILVUS.name()),
|
||||
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus"));
|
||||
Lists.newArrayList(EmbeddingStoreType.MILVUS.name(),
|
||||
EmbeddingStoreType.PGVECTOR.name()),
|
||||
ImmutableMap.of(EmbeddingStoreType.MILVUS.name(), "milvus",
|
||||
EmbeddingStoreType.PGVECTOR.name(), "postgres"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
package com.tencent.supersonic.common.config;
|
||||
|
||||
import com.tencent.supersonic.common.util.S2ThreadContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class ThreadContextConfig {
|
||||
|
||||
@Bean
|
||||
public S2ThreadContext s2ThreadContext() {
|
||||
return new S2ThreadContext();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.tencent.supersonic.common.config;
|
||||
|
||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Component
|
||||
public class ThreadPoolConfig {
|
||||
|
||||
@Bean("eventExecutor")
|
||||
public ThreadPoolExecutor getTaskEventExecutor() {
|
||||
return new ThreadPoolExecutor(4, 8, 60 * 3, TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(1024),
|
||||
new ThreadFactoryBuilder().setNameFormat("supersonic-event-pool-").build(),
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
|
||||
@Bean("commonExecutor")
|
||||
public ThreadPoolExecutor getCommonExecutor() {
|
||||
return new ThreadPoolExecutor(8, 16, 60 * 3, TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(1024),
|
||||
new ThreadFactoryBuilder().setNameFormat("supersonic-common-pool-").build(),
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
|
||||
@Bean("mapExecutor")
|
||||
public ThreadPoolExecutor getMapExecutor() {
|
||||
return new ThreadPoolExecutor(8, 16, 60 * 3, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
|
||||
new ThreadFactoryBuilder().setNameFormat("supersonic-map-pool-").build(),
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
|
||||
@Bean("chatExecutor")
|
||||
public ThreadPoolExecutor getChatExecutor() {
|
||||
return new ThreadPoolExecutor(8, 16, 60 * 3, TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(1024),
|
||||
new ThreadFactoryBuilder().setNameFormat("supersonic-chat-pool-").build(),
|
||||
new ThreadPoolExecutor.CallerRunsPolicy());
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user