mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-11 20:25:12 +00:00
Compare commits
102 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be158a1776 | ||
|
|
c12f5d23f0 | ||
|
|
7ec77c7d23 | ||
|
|
f04cc28f25 | ||
|
|
b28eb637c8 | ||
|
|
97c767a45b | ||
|
|
7afa42b4bc | ||
|
|
a375a922c2 | ||
|
|
add74b9589 | ||
|
|
566321895e | ||
|
|
f154c2a2b4 | ||
|
|
9c6bd7cf19 | ||
|
|
20c8456705 | ||
|
|
c154f476cb | ||
|
|
1ef1aa53a3 | ||
|
|
08c184c7b0 | ||
|
|
36edc0c1b4 | ||
|
|
026cf2056d | ||
|
|
fc82350af5 | ||
|
|
6b5d84a13f | ||
|
|
3ba9073a1b | ||
|
|
cbf38ed785 | ||
|
|
f017f41201 | ||
|
|
b40670b0e3 | ||
|
|
7af5afc3eb | ||
|
|
0abbd83f51 | ||
|
|
3e77fc3069 | ||
|
|
b019f4d9bb | ||
|
|
90f9da162e | ||
|
|
a06a1fa898 | ||
|
|
dfb8e3a427 | ||
|
|
d4eecc1bf8 | ||
|
|
71c491a80d | ||
|
|
93c3ce1631 | ||
|
|
c181ce6945 | ||
|
|
07e6924cfd | ||
|
|
b2beecb5b8 | ||
|
|
eb08667d90 | ||
|
|
74b89a9430 | ||
|
|
7707179faa | ||
|
|
af103f3aa3 | ||
|
|
7f65057a0f | ||
|
|
a6818fb6ff | ||
|
|
e7d654f150 | ||
|
|
e29ecec0c9 | ||
|
|
dcc1f26542 | ||
|
|
3436b36552 | ||
|
|
4322ae42ac | ||
|
|
e9c7237794 | ||
|
|
3a5349c916 | ||
|
|
1e93282c9f | ||
|
|
9c8039c499 | ||
|
|
61da52650c | ||
|
|
87a60eeba2 | ||
|
|
052e217c8c | ||
|
|
bbad302efd | ||
|
|
ed54d7bae3 | ||
|
|
0a6160272b | ||
|
|
ad8079e058 | ||
|
|
8eef11f342 | ||
|
|
b55b4c130e | ||
|
|
0408f0fe9a | ||
|
|
062f7340e5 | ||
|
|
72bd79fe73 | ||
|
|
602b9547b8 | ||
|
|
ade96c3adc | ||
|
|
023e84c420 | ||
|
|
0858c13365 | ||
|
|
7acb48da0e | ||
|
|
52fea5311d | ||
|
|
d72166944c | ||
|
|
e7f13572d7 | ||
|
|
af1c560cc4 | ||
|
|
49f0a4dc1d | ||
|
|
31f1fc315d | ||
|
|
afa8fd74a6 | ||
|
|
56b0f35250 | ||
|
|
af6c8cdbda | ||
|
|
9e69002d70 | ||
|
|
cd727663a5 | ||
|
|
d0289a3243 | ||
|
|
327bab015e | ||
|
|
f788249b1a | ||
|
|
6c8ebdfe1a | ||
|
|
b706c4efb4 | ||
|
|
cf2b4bfb5c | ||
|
|
40c86810bb | ||
|
|
5ab1cade0a | ||
|
|
e0955c0618 | ||
|
|
125598bd6e | ||
|
|
0cbfe473dd | ||
|
|
090abbceed | ||
|
|
0a1f446fb8 | ||
|
|
fa38e37be3 | ||
|
|
7b580b7c94 | ||
|
|
2631352c30 | ||
|
|
f7914ff6f4 | ||
|
|
ab077df36d | ||
|
|
150d67f903 | ||
|
|
187dcacbe7 | ||
|
|
a194822cdd | ||
|
|
6bbc0a2cb4 |
31
.github/workflows/ci.yml
vendored
Normal file
31
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
name: supersonic CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- name: Set up JDK 8
|
||||||
|
uses: actions/setup-java@v2
|
||||||
|
with:
|
||||||
|
java-version: '8'
|
||||||
|
distribution: 'adopt'
|
||||||
|
- name: Cache Maven packages
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: ~/.m2
|
||||||
|
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||||
|
restore-keys: ${{ runner.os }}-m2
|
||||||
|
- name: Build with Maven
|
||||||
|
run: mvn -B package --file pom.xml
|
||||||
|
- name: Test with Maven
|
||||||
|
run: mvn test
|
||||||
@@ -4,6 +4,14 @@
|
|||||||
- "Breaking Changes" describes any changes that may break existing functionality or cause
|
- "Breaking Changes" describes any changes that may break existing functionality or cause
|
||||||
compatibility issues with previous versions.
|
compatibility issues with previous versions.
|
||||||
|
|
||||||
|
## SuperSonic [0.8.4] - 2024-01-19
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- support creating derived metrics.
|
||||||
|
- Support creating metrics using three methods: by measure, metric, and field expressions.
|
||||||
|
- added support for postgresql data source.
|
||||||
|
- code adjustment and abstract optimization for chat and headless.
|
||||||
|
|
||||||
## SuperSonic [0.8.2] - 2023-12-18
|
## SuperSonic [0.8.2] - 2023-12-18
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
[中文介绍](README_CN.md) | [文档中心](https://github.com/tencentmusic/supersonic/wiki)
|
[中文介绍](README_CN.md) | [文档中心](https://github.com/tencentmusic/supersonic/wiki)
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
# SuperSonic (超音数)
|
# SuperSonic (超音数)
|
||||||
|
|
||||||
**SuperSonic is the next-generation LLM-powered data analytics platform that integrates ChatBI and HeadlessBI**. SuperSonic provides a chat interface that empowers users to query data using natural language and visualize the results with suitable charts. To enable such experience, the only thing necessary is to build logical semantic models (definition of entities/metrics/dimensions/tags, along with their meaning, context and relationships) on top of physical data models, and **no data modification or copying** is required. Meanwhile, SuperSonic is designed to be **highly extensible**, allowing custom functionalities to be added and configured with Java SPI.
|
**SuperSonic is the next-generation LLM-powered data analytics platform that integrates ChatBI and HeadlessBI**. SuperSonic provides a chat interface that empowers users to query data using natural language and visualize the results with suitable charts. To enable such experience, the only thing necessary is to build logical semantic models (definition of entities/metrics/dimensions/tags, along with their meaning, context and relationships) on top of physical data models, and **no data modification or copying** is required. Meanwhile, SuperSonic is designed to be **highly extensible**, allowing custom functionalities to be added and configured with Java SPI.
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ if [ $? -ne 0 ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
#2. move package to build
|
#2. move package to build
|
||||||
cp $baseDir/../launchers/semantic/target/*.tar.gz ${buildDir}/supersonic-semantic.tar.gz
|
cp $baseDir/../launchers/headless/target/*.tar.gz ${buildDir}/supersonic-headless.tar.gz
|
||||||
cp $baseDir/../launchers/chat/target/*.tar.gz ${buildDir}/supersonic-chat.tar.gz
|
cp $baseDir/../launchers/chat/target/*.tar.gz ${buildDir}/supersonic-chat.tar.gz
|
||||||
cp $baseDir/../launchers/standalone/target/*.tar.gz ${buildDir}/supersonic-standalone.tar.gz
|
cp $baseDir/../launchers/standalone/target/*.tar.gz ${buildDir}/supersonic-standalone.tar.gz
|
||||||
|
|
||||||
@@ -38,7 +38,7 @@ fi
|
|||||||
cd $buildDir
|
cd $buildDir
|
||||||
tar xvf supersonic-webapp.tar.gz
|
tar xvf supersonic-webapp.tar.gz
|
||||||
mv supersonic-webapp webapp
|
mv supersonic-webapp webapp
|
||||||
cp -fr webapp ../../launchers/semantic/target/classes
|
cp -fr webapp ../../launchers/headless/target/classes
|
||||||
cp -fr webapp ../../launchers/chat/target/classes
|
cp -fr webapp ../../launchers/chat/target/classes
|
||||||
cp -fr webapp ../../launchers/standalone/target/classes
|
cp -fr webapp ../../launchers/standalone/target/classes
|
||||||
rm -fr ${buildDir}/webapp
|
rm -fr ${buildDir}/webapp
|
||||||
@@ -55,4 +55,4 @@ fi
|
|||||||
rm -fr $runtimeDir/supersonic*
|
rm -fr $runtimeDir/supersonic*
|
||||||
moveAllToRuntime
|
moveAllToRuntime
|
||||||
setEnvToWeb chat
|
setEnvToWeb chat
|
||||||
setEnvToWeb semantic
|
setEnvToWeb headless
|
||||||
|
|||||||
@@ -10,11 +10,11 @@ runtimeDir=$baseDir/../runtime
|
|||||||
buildDir=$baseDir/build
|
buildDir=$baseDir/build
|
||||||
|
|
||||||
readonly CHAT_APP_NAME="supersonic_chat"
|
readonly CHAT_APP_NAME="supersonic_chat"
|
||||||
readonly SEMANTIC_APP_NAME="supersonic_semantic"
|
readonly HEADLESS_APP_NAME="supersonic_headless"
|
||||||
readonly PYLLM_APP_NAME="supersonic_pyllm"
|
readonly PYLLM_APP_NAME="supersonic_pyllm"
|
||||||
readonly STANDALONE_APP_NAME="supersonic_standalone"
|
readonly STANDALONE_APP_NAME="supersonic_standalone"
|
||||||
readonly CHAT_SERVICE="chat"
|
readonly CHAT_SERVICE="chat"
|
||||||
readonly SEMANTIC_SERVICE="semantic"
|
readonly HEADLESS_SERVICE="headless"
|
||||||
readonly PYLLM_SERVICE="pyllm"
|
readonly PYLLM_SERVICE="pyllm"
|
||||||
readonly STANDALONE_SERVICE="standalone"
|
readonly STANDALONE_SERVICE="standalone"
|
||||||
readonly PYLLM_HOST="127.0.0.1"
|
readonly PYLLM_HOST="127.0.0.1"
|
||||||
@@ -46,7 +46,7 @@ function moveAllToRuntime {
|
|||||||
mv ${buildDir}/supersonic-webapp ${buildDir}/webapp
|
mv ${buildDir}/supersonic-webapp ${buildDir}/webapp
|
||||||
|
|
||||||
moveToRuntime chat
|
moveToRuntime chat
|
||||||
moveToRuntime semantic
|
moveToRuntime headless
|
||||||
moveToRuntime standalone
|
moveToRuntime standalone
|
||||||
rm -fr ${buildDir}/webapp
|
rm -fr ${buildDir}/webapp
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,8 +33,8 @@ cd $baseDir
|
|||||||
function setMainClass {
|
function setMainClass {
|
||||||
if [ "$service" == $CHAT_SERVICE ]; then
|
if [ "$service" == $CHAT_SERVICE ]; then
|
||||||
main_class="com.tencent.supersonic.ChatLauncher"
|
main_class="com.tencent.supersonic.ChatLauncher"
|
||||||
elif [ "$service" == $SEMANTIC_SERVICE ]; then
|
elif [ "$service" == $HEADLESS_SERVICE ]; then
|
||||||
main_class="com.tencent.supersonic.SemanticLauncher"
|
main_class="com.tencent.supersonic.HeadlessLauncher"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
setMainClass
|
setMainClass
|
||||||
@@ -42,8 +42,8 @@ setMainClass
|
|||||||
function setAppName {
|
function setAppName {
|
||||||
if [ "$service" == $CHAT_SERVICE ]; then
|
if [ "$service" == $CHAT_SERVICE ]; then
|
||||||
app_name=$CHAT_APP_NAME
|
app_name=$CHAT_APP_NAME
|
||||||
elif [ "$service" == $SEMANTIC_SERVICE ]; then
|
elif [ "$service" == $HEADLESS_SERVICE ]; then
|
||||||
app_name=$SEMANTIC_APP_NAME
|
app_name=$HEADLESS_APP_NAME
|
||||||
elif [ "$service" == $PYLLM_SERVICE ]; then
|
elif [ "$service" == $PYLLM_SERVICE ]; then
|
||||||
app_name=$PYLLM_APP_NAME
|
app_name=$PYLLM_APP_NAME
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -24,4 +24,13 @@ public class AuthenticationConfig {
|
|||||||
@Value("${authentication.token.http.header.key:Authorization}")
|
@Value("${authentication.token.http.header.key:Authorization}")
|
||||||
private String tokenHttpHeaderKey;
|
private String tokenHttpHeaderKey;
|
||||||
|
|
||||||
|
@Value("${authentication.app.appId:appId}")
|
||||||
|
private String appId;
|
||||||
|
|
||||||
|
@Value("${authentication.app.timestamp:timestamp}")
|
||||||
|
private String timestamp;
|
||||||
|
|
||||||
|
@Value("${authentication.app.signature:signature}")
|
||||||
|
private String signature;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,15 @@
|
|||||||
package com.tencent.supersonic.auth.api.authentication.pojo;
|
package com.tencent.supersonic.auth.api.authentication.pojo;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
|
@AllArgsConstructor
|
||||||
|
@NoArgsConstructor
|
||||||
public class Organization {
|
public class Organization {
|
||||||
|
|
||||||
private String id;
|
private String id;
|
||||||
|
|||||||
@@ -28,6 +28,11 @@ public class User {
|
|||||||
return new User(1L, "admin", "admin", "admin@email", 1);
|
return new User(1L, "admin", "admin", "admin@email", 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static User getAppUser(int appId) {
|
||||||
|
String name = String.format("app_%s", appId);
|
||||||
|
return new User(1L, name, name, "", 1);
|
||||||
|
}
|
||||||
|
|
||||||
public String getDisplayName() {
|
public String getDisplayName() {
|
||||||
return StringUtils.isBlank(displayName) ? name : displayName;
|
return StringUtils.isBlank(displayName) ? name : displayName;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,11 +4,15 @@ import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
|
|||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
||||||
|
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
public interface UserService {
|
public interface UserService {
|
||||||
|
|
||||||
|
User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse);
|
||||||
|
|
||||||
List<String> getUserNames();
|
List<String> getUserNames();
|
||||||
|
|
||||||
List<User> getUserList();
|
List<User> getUserList();
|
||||||
|
|||||||
@@ -2,6 +2,11 @@ package com.tencent.supersonic.auth.api.authentication.utils;
|
|||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.auth.api.authentication.service.UserStrategy;
|
import com.tencent.supersonic.auth.api.authentication.service.UserStrategy;
|
||||||
|
import com.tencent.supersonic.common.pojo.SysParameter;
|
||||||
|
import com.tencent.supersonic.common.service.SysParameterService;
|
||||||
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import javax.servlet.http.HttpServletResponse;
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
|
||||||
@@ -14,7 +19,14 @@ public final class UserHolder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static User findUser(HttpServletRequest request, HttpServletResponse response) {
|
public static User findUser(HttpServletRequest request, HttpServletResponse response) {
|
||||||
return REPO.findUser(request, response);
|
User user = REPO.findUser(request, response);
|
||||||
|
SysParameterService sysParameterService = ContextUtils.getBean(SysParameterService.class);
|
||||||
|
SysParameter sysParameter = sysParameterService.getSysParameter();
|
||||||
|
if (!CollectionUtils.isEmpty(sysParameter.getAdmins())
|
||||||
|
&& sysParameter.getAdmins().contains(user.getName())) {
|
||||||
|
user.setIsAdmin(1);
|
||||||
|
}
|
||||||
|
return user;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,7 +44,17 @@ public class DefaultUserAdaptor implements UserAdaptor {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<Organization> getOrganizationTree() {
|
public List<Organization> getOrganizationTree() {
|
||||||
return Lists.newArrayList();
|
Organization superSonic = new Organization("1", "0",
|
||||||
|
"SuperSonic", "SuperSonic", Lists.newArrayList(), true);
|
||||||
|
Organization hr = new Organization("2", "1",
|
||||||
|
"Hr", "SuperSonic/Hr", Lists.newArrayList(), false);
|
||||||
|
Organization sales = new Organization("3", "1",
|
||||||
|
"Sales", "SuperSonic/Sales", Lists.newArrayList(), false);
|
||||||
|
Organization marketing = new Organization("4", "1",
|
||||||
|
"Marketing", "SuperSonic/Marketing", Lists.newArrayList(), false);
|
||||||
|
List<Organization> subOrganization = Lists.newArrayList(hr, sales, marketing);
|
||||||
|
superSonic.setSubOrganizations(subOrganization);
|
||||||
|
return Lists.newArrayList(superSonic);
|
||||||
}
|
}
|
||||||
|
|
||||||
private User convert(UserDO userDO) {
|
private User convert(UserDO userDO) {
|
||||||
|
|||||||
@@ -5,16 +5,17 @@ import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
|||||||
import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
|
import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
|
||||||
import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils;
|
import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils;
|
||||||
import com.tencent.supersonic.common.util.S2ThreadContext;
|
import com.tencent.supersonic.common.util.S2ThreadContext;
|
||||||
import java.lang.reflect.Field;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import org.apache.catalina.connector.RequestFacade;
|
import org.apache.catalina.connector.RequestFacade;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.apache.tomcat.util.http.MimeHeaders;
|
import org.apache.tomcat.util.http.MimeHeaders;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest;
|
import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest;
|
||||||
import org.springframework.web.servlet.HandlerInterceptor;
|
import org.springframework.web.servlet.HandlerInterceptor;
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import java.lang.reflect.Field;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
|
||||||
public abstract class AuthenticationInterceptor implements HandlerInterceptor {
|
public abstract class AuthenticationInterceptor implements HandlerInterceptor {
|
||||||
@@ -58,6 +59,11 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor {
|
|||||||
return "true".equalsIgnoreCase(internal);
|
return "true".equalsIgnoreCase(internal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected boolean isAppRequest(HttpServletRequest request) {
|
||||||
|
String appId = request.getHeader(authenticationConfig.getAppId());
|
||||||
|
return StringUtils.isNotBlank(appId);
|
||||||
|
}
|
||||||
|
|
||||||
protected void reflectSetparam(HttpServletRequest request, String key, String value) {
|
protected void reflectSetparam(HttpServletRequest request, String key, String value) {
|
||||||
try {
|
try {
|
||||||
if (request instanceof StandardMultipartHttpServletRequest) {
|
if (request instanceof StandardMultipartHttpServletRequest) {
|
||||||
|
|||||||
@@ -10,12 +10,12 @@ import com.tencent.supersonic.common.pojo.exception.AccessException;
|
|||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.S2ThreadContext;
|
import com.tencent.supersonic.common.util.S2ThreadContext;
|
||||||
import com.tencent.supersonic.common.util.ThreadContext;
|
import com.tencent.supersonic.common.util.ThreadContext;
|
||||||
import java.lang.reflect.Method;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.web.method.HandlerMethod;
|
import org.springframework.web.method.HandlerMethod;
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor {
|
public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor {
|
||||||
@@ -35,7 +35,10 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor
|
|||||||
setFakerUser(request);
|
setFakerUser(request);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
if (isAppRequest(request)) {
|
||||||
|
setFakerUser(request);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
if (handler instanceof HandlerMethod) {
|
if (handler instanceof HandlerMethod) {
|
||||||
HandlerMethod handlerMethod = (HandlerMethod) handler;
|
HandlerMethod handlerMethod = (HandlerMethod) handler;
|
||||||
Method method = handlerMethod.getMethod();
|
Method method = handlerMethod.getMethod();
|
||||||
|
|||||||
@@ -5,18 +5,18 @@ import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
|
|||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
||||||
import com.tencent.supersonic.auth.api.authentication.service.UserService;
|
import com.tencent.supersonic.auth.api.authentication.service.UserService;
|
||||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestBody;
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
import org.springframework.web.bind.annotation.PathVariable;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/api/auth/user")
|
@RequestMapping("/api/auth/user")
|
||||||
@@ -31,7 +31,7 @@ public class UserController {
|
|||||||
|
|
||||||
@GetMapping("/getCurrentUser")
|
@GetMapping("/getCurrentUser")
|
||||||
public User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
|
public User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
|
||||||
return UserHolder.findUser(httpServletRequest, httpServletResponse);
|
return userService.getCurrentUser(httpServletRequest, httpServletResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/getUserNames")
|
@GetMapping("/getUserNames")
|
||||||
|
|||||||
@@ -4,15 +4,39 @@ import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
|
|||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
|
||||||
import com.tencent.supersonic.auth.api.authentication.service.UserService;
|
import com.tencent.supersonic.auth.api.authentication.service.UserService;
|
||||||
|
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||||
import com.tencent.supersonic.auth.authentication.utils.ComponentFactory;
|
import com.tencent.supersonic.auth.authentication.utils.ComponentFactory;
|
||||||
|
import com.tencent.supersonic.common.pojo.SysParameter;
|
||||||
|
import com.tencent.supersonic.common.service.SysParameterService;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class UserServiceImpl implements UserService {
|
public class UserServiceImpl implements UserService {
|
||||||
|
|
||||||
|
private SysParameterService sysParameterService;
|
||||||
|
|
||||||
|
public UserServiceImpl(SysParameterService sysParameterService) {
|
||||||
|
this.sysParameterService = sysParameterService;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public User getCurrentUser(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) {
|
||||||
|
User user = UserHolder.findUser(httpServletRequest, httpServletResponse);
|
||||||
|
if (user != null) {
|
||||||
|
SysParameter sysParameter = sysParameterService.getSysParameter();
|
||||||
|
if (!CollectionUtils.isEmpty(sysParameter.getAdmins())
|
||||||
|
&& sysParameter.getAdmins().contains(user.getName())) {
|
||||||
|
user.setIsAdmin(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return user;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getUserNames() {
|
public List<String> getUserNames() {
|
||||||
return ComponentFactory.getUserAdaptor().getUserNames();
|
return ComponentFactory.getUserAdaptor().getUserNames();
|
||||||
|
|||||||
@@ -23,7 +23,7 @@
|
|||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.tencent.supersonic</groupId>
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
<artifactId>semantic-api</artifactId>
|
<artifactId>headless-api</artifactId>
|
||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
<scope>compile</scope>
|
<scope>compile</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.api.pojo;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.NoArgsConstructor;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
@NoArgsConstructor
|
|
||||||
public class QueryContext {
|
|
||||||
|
|
||||||
private QueryReq request;
|
|
||||||
private List<SemanticQuery> candidateQueries = new ArrayList<>();
|
|
||||||
private SchemaMapInfo mapInfo = new SchemaMapInfo();
|
|
||||||
private SchemaModelClusterMapInfo modelClusterMapInfo = new SchemaModelClusterMapInfo();
|
|
||||||
|
|
||||||
public QueryContext(QueryReq request) {
|
|
||||||
this.request = request;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
package com.tencent.supersonic.chat.api.pojo;
|
package com.tencent.supersonic.chat.api.pojo;
|
||||||
|
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
import com.tencent.supersonic.common.pojo.ModelCluster;
|
import com.tencent.supersonic.common.pojo.ModelCluster;
|
||||||
import com.tencent.supersonic.common.pojo.Order;
|
import com.tencent.supersonic.common.pojo.Order;
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
package com.tencent.supersonic.chat.api.pojo;
|
package com.tencent.supersonic.chat.api.pojo;
|
||||||
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
public class SemanticSchema implements Serializable {
|
public class SemanticSchema implements Serializable {
|
||||||
|
|
||||||
@@ -117,7 +117,8 @@ public class SemanticSchema implements Serializable {
|
|||||||
|
|
||||||
public List<SchemaElement> getTags(Set<Long> modelIds) {
|
public List<SchemaElement> getTags(Set<Long> modelIds) {
|
||||||
List<SchemaElement> tags = new ArrayList<>();
|
List<SchemaElement> tags = new ArrayList<>();
|
||||||
modelSchemaList.stream().filter(schemaElement -> modelIds.contains(schemaElement.getModel()))
|
modelSchemaList.stream().filter(schemaElement ->
|
||||||
|
modelIds.contains(schemaElement.getModel().getModel()))
|
||||||
.forEach(d -> tags.addAll(d.getTags()));
|
.forEach(d -> tags.addAll(d.getTags()));
|
||||||
return tags;
|
return tags;
|
||||||
}
|
}
|
||||||
@@ -139,6 +140,11 @@ public class SemanticSchema implements Serializable {
|
|||||||
return entities;
|
return entities;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<SchemaElement> getEntities(Set<Long> modelIds) {
|
||||||
|
List<SchemaElement> entities = getEntities();
|
||||||
|
return getElementsByModelId(modelIds, entities);
|
||||||
|
}
|
||||||
|
|
||||||
private List<SchemaElement> getElementsByModelId(Set<Long> modelIds, List<SchemaElement> elements) {
|
private List<SchemaElement> getElementsByModelId(Set<Long> modelIds, List<SchemaElement> elements) {
|
||||||
return elements.stream()
|
return elements.stream()
|
||||||
.filter(schemaElement -> modelIds.contains(schemaElement.getModel()))
|
.filter(schemaElement -> modelIds.contains(schemaElement.getModel()))
|
||||||
@@ -154,7 +160,8 @@ public class SemanticSchema implements Serializable {
|
|||||||
private Optional<SchemaElement> getElementsByNameOrAlias(String name, List<SchemaElement> elements) {
|
private Optional<SchemaElement> getElementsByNameOrAlias(String name, List<SchemaElement> elements) {
|
||||||
return elements.stream()
|
return elements.stream()
|
||||||
.filter(schemaElement ->
|
.filter(schemaElement ->
|
||||||
name.equals(schemaElement.getName()) || schemaElement.getAlias().contains(name)
|
name.equals(schemaElement.getName()) || (Objects.nonNull(schemaElement.getAlias())
|
||||||
|
&& schemaElement.getAlias().contains(name))
|
||||||
).findFirst();
|
).findFirst();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.api.pojo.response;
|
|||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
|
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq.TimeMode;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -26,6 +26,6 @@ public class ChatDefaultRichConfigResp {
|
|||||||
*/
|
*/
|
||||||
private String period = Constants.DAY;
|
private String period = Constants.DAY;
|
||||||
|
|
||||||
private ChatDefaultConfigReq.TimeMode timeMode;
|
private TimeMode timeMode;
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -17,4 +17,6 @@ public class QueryResp {
|
|||||||
private QueryResult queryResult;
|
private QueryResult queryResult;
|
||||||
private List<SemanticParseInfo> parseInfos;
|
private List<SemanticParseInfo> parseInfos;
|
||||||
private List<SimilarQueryRecallResp> similarQueries;
|
private List<SimilarQueryRecallResp> similarQueries;
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -22,6 +22,69 @@
|
|||||||
<artifactId>spring-context</artifactId>
|
<artifactId>spring-context</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.hankcs</groupId>
|
||||||
|
<artifactId>hanlp</artifactId>
|
||||||
|
<version>${hanlp.version}</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-client</artifactId>
|
||||||
|
<version>${hadoop.version}</version>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
|
<artifactId>zookeeper</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.servlet</groupId>
|
||||||
|
<artifactId>servlet-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-hdfs</artifactId>
|
||||||
|
<version>${hadoop.version}</version>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
|
<artifactId>zookeeper</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.servlet</groupId>
|
||||||
|
<artifactId>servlet-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-common</artifactId>
|
||||||
|
<version>${hadoop.version}</version>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-log4j12</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>log4j</groupId>
|
||||||
|
<artifactId>log4j</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
|
<artifactId>zookeeper</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>org.apache.curator</groupId>
|
||||||
|
<artifactId>*</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>javax.servlet</groupId>
|
||||||
|
<artifactId>servlet-api</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.testng</groupId>
|
<groupId>org.testng</groupId>
|
||||||
<artifactId>testng</artifactId>
|
<artifactId>testng</artifactId>
|
||||||
@@ -72,17 +135,12 @@
|
|||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.tencent.supersonic</groupId>
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
<artifactId>chat-knowledge</artifactId>
|
<artifactId>headless-api</artifactId>
|
||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.tencent.supersonic</groupId>
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
<artifactId>semantic-api</artifactId>
|
<artifactId>headless-core</artifactId>
|
||||||
<version>${project.version}</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.tencent.supersonic</groupId>
|
|
||||||
<artifactId>semantic-query</artifactId>
|
|
||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
@@ -104,6 +162,12 @@
|
|||||||
<version>${mockito-inline.version}</version>
|
<version>${mockito-inline.version}</version>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
|
<artifactId>headless-server</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.hankcs.hanlp.collection.trie.bintrie;
|
package com.hankcs.hanlp.collection.trie.bintrie;
|
||||||
|
|
||||||
import com.hankcs.hanlp.corpus.io.ByteArray;
|
import com.hankcs.hanlp.corpus.io.ByteArray;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.LoadRemoveService;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.knowledge.service.LoadRemoveService;
|
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.ObjectInput;
|
import java.io.ObjectInput;
|
||||||
@@ -3,7 +3,7 @@ package com.hankcs.hanlp.seg.common;
|
|||||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||||
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
||||||
import com.hankcs.hanlp.dictionary.CustomDictionary;
|
import com.hankcs.hanlp.dictionary.CustomDictionary;
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.ToString;
|
import lombok.ToString;
|
||||||
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
|
|
||||||
import java.util.List;
|
|
||||||
import lombok.Data;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
public class EntityInternalDetail {
|
|
||||||
|
|
||||||
List<DimSchemaResp> dimensionList;
|
|
||||||
List<MetricSchemaResp> metricList;
|
|
||||||
}
|
|
||||||
@@ -1,15 +1,18 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
import com.alibaba.fastjson.JSONObject;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.common.pojo.RecordInfo;
|
import com.tencent.supersonic.common.pojo.RecordInfo;
|
||||||
import java.util.Objects;
|
import java.util.Collection;
|
||||||
import lombok.Data;
|
import java.util.HashSet;
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public class Agent extends RecordInfo {
|
public class Agent extends RecordInfo {
|
||||||
@@ -19,10 +22,13 @@ public class Agent extends RecordInfo {
|
|||||||
private String name;
|
private String name;
|
||||||
private String description;
|
private String description;
|
||||||
|
|
||||||
//0 offline, 1 online
|
/**
|
||||||
|
* 0 offline, 1 online
|
||||||
|
*/
|
||||||
private Integer status;
|
private Integer status;
|
||||||
private List<String> examples;
|
private List<String> examples;
|
||||||
private String agentConfig;
|
private String agentConfig;
|
||||||
|
|
||||||
public List<String> getTools(AgentToolType type) {
|
public List<String> getTools(AgentToolType type) {
|
||||||
Map map = JSONObject.parseObject(agentConfig, Map.class);
|
Map map = JSONObject.parseObject(agentConfig, Map.class);
|
||||||
if (CollectionUtils.isEmpty(map) || map.get("tools") == null) {
|
if (CollectionUtils.isEmpty(map) || map.get("tools") == null) {
|
||||||
@@ -45,4 +51,27 @@ public class Agent extends RecordInfo {
|
|||||||
return enableSearch != null && enableSearch == 1;
|
return enableSearch != null && enableSearch == 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static boolean containsAllModel(Set<Long> detectModelIds) {
|
||||||
|
return !CollectionUtils.isEmpty(detectModelIds) && detectModelIds.contains(-1L);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<NL2SQLTool> getParserTools(AgentToolType agentToolType) {
|
||||||
|
List<String> tools = this.getTools(agentToolType);
|
||||||
|
if (CollectionUtils.isEmpty(tools)) {
|
||||||
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
|
return tools.stream().map(tool -> JSONObject.parseObject(tool, NL2SQLTool.class))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
public Set<Long> getModelIds(AgentToolType agentToolType) {
|
||||||
|
List<NL2SQLTool> commonAgentTools = getParserTools(agentToolType);
|
||||||
|
if (CollectionUtils.isEmpty(commonAgentTools)) {
|
||||||
|
return new HashSet<>();
|
||||||
|
}
|
||||||
|
return commonAgentTools.stream().map(NL2SQLTool::getModelIds)
|
||||||
|
.filter(modelIds -> !CollectionUtils.isEmpty(modelIds))
|
||||||
|
.flatMap(Collection::stream)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
public enum AgentToolType {
|
public enum AgentToolType {
|
||||||
NL2SQL_RULE,
|
NL2SQL_RULE,
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.agent;
|
package com.tencent.supersonic.chat.core.agent;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
|
|
||||||
|
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
|
||||||
|
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
|
||||||
|
import java.util.List;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class EntityInternalDetail {
|
||||||
|
|
||||||
|
List<DimSchemaResp> dimensionList;
|
||||||
|
List<MetricSchemaResp> metricList;
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -16,10 +16,12 @@ public class LLMParserConfig {
|
|||||||
@Value("${query2sql.path:/query2sql}")
|
@Value("${query2sql.path:/query2sql}")
|
||||||
private String queryToSqlPath;
|
private String queryToSqlPath;
|
||||||
|
|
||||||
@Value("${dimension.topn:5}")
|
@Value("${dimension.topn:10}")
|
||||||
private Integer dimensionTopN;
|
private Integer dimensionTopN;
|
||||||
|
|
||||||
@Value("${metric.topn:5}")
|
@Value("${metric.topn:10}")
|
||||||
private Integer metricTopN;
|
private Integer metricTopN;
|
||||||
|
|
||||||
|
@Value("${all.model:false}")
|
||||||
|
private Boolean allModel;
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.tencent.supersonic.chat.config;
|
package com.tencent.supersonic.chat.core.config;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.SqlGenerationMode;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.SqlGenerationMode;
|
||||||
import com.tencent.supersonic.common.service.SysParameterService;
|
import com.tencent.supersonic.common.service.SysParameterService;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -64,18 +64,21 @@ public class OptimizationConfig {
|
|||||||
@Value("${s2SQL.use.switch:true}")
|
@Value("${s2SQL.use.switch:true}")
|
||||||
private boolean useS2SqlSwitch;
|
private boolean useS2SqlSwitch;
|
||||||
|
|
||||||
@Value("${text2sql.example.num:10}")
|
@Value("${text2sql.example.num:15}")
|
||||||
private int text2sqlExampleNum;
|
private int text2sqlExampleNum;
|
||||||
|
|
||||||
@Value("${text2sql.fewShots.num:5}")
|
@Value("${text2sql.fewShots.num:10}")
|
||||||
private int text2sqlFewShotsNum;
|
private int text2sqlFewShotsNum;
|
||||||
|
|
||||||
@Value("${text2sql.self.consistency.num:2}")
|
@Value("${text2sql.self.consistency.num:5}")
|
||||||
private int text2sqlSelfConsistencyNum;
|
private int text2sqlSelfConsistencyNum;
|
||||||
|
|
||||||
@Value("${text2sql.collection.name:text2dsl_agent_collection}")
|
@Value("${text2sql.collection.name:text2dsl_agent_collection}")
|
||||||
private String text2sqlCollectionName;
|
private String text2sqlCollectionName;
|
||||||
|
|
||||||
|
@Value("${parse.show.count:3}")
|
||||||
|
private Integer parseShowCount;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private SysParameterService sysParameterService;
|
private SysParameterService sysParameterService;
|
||||||
|
|
||||||
@@ -147,6 +150,10 @@ public class OptimizationConfig {
|
|||||||
return convertValue("s2SQL.generation", SqlGenerationMode.class, sqlGenerationMode);
|
return convertValue("s2SQL.generation", SqlGenerationMode.class, sqlGenerationMode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Integer getParseShowCount() {
|
||||||
|
return convertValue("parse.show.count", Integer.class, parseShowCount);
|
||||||
|
}
|
||||||
|
|
||||||
public <T> T convertValue(String paramName, Class<T> targetType, T defaultValue) {
|
public <T> T convertValue(String paramName, Class<T> targetType, T defaultValue) {
|
||||||
try {
|
try {
|
||||||
String value = sysParameterService.getSysParameter().getParameterByName(paramName);
|
String value = sysParameterService.getSysParameter().getParameterByName(paramName);
|
||||||
@@ -1,22 +1,14 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticCorrector;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.apache.commons.lang3.tuple.Pair;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -24,6 +16,10 @@ import java.util.Map;
|
|||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.commons.lang3.tuple.Pair;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* basic semantic correction functionality, offering common methods and an
|
* basic semantic correction functionality, offering common methods and an
|
||||||
@@ -32,23 +28,23 @@ import java.util.stream.Collectors;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
||||||
|
|
||||||
public void correct(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void correct(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
try {
|
try {
|
||||||
if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getCorrectS2SQL())) {
|
if (StringUtils.isBlank(semanticParseInfo.getSqlInfo().getCorrectS2SQL())) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
doCorrect(queryReq, semanticParseInfo);
|
doCorrect(queryContext, semanticParseInfo);
|
||||||
log.info("sqlCorrection:{} sql:{}", this.getClass().getSimpleName(), semanticParseInfo.getSqlInfo());
|
log.info("sqlCorrection:{} sql:{}", this.getClass().getSimpleName(), semanticParseInfo.getSqlInfo());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error(String.format("correct error,sqlInfo:%s", semanticParseInfo.getSqlInfo()), e);
|
log.error(String.format("correct error,sqlInfo:%s", semanticParseInfo.getSqlInfo()), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo);
|
public abstract void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo);
|
||||||
|
|
||||||
protected Map<String, String> getFieldNameMap(Set<Long> modelIds) {
|
protected Map<String, String> getFieldNameMap(QueryContext queryContext, Set<Long> modelIds) {
|
||||||
|
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
|
|
||||||
List<SchemaElement> dbAllFields = new ArrayList<>();
|
List<SchemaElement> dbAllFields = new ArrayList<>();
|
||||||
dbAllFields.addAll(semanticSchema.getMetrics());
|
dbAllFields.addAll(semanticSchema.getMetrics());
|
||||||
@@ -101,12 +97,12 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
|||||||
semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceFields);
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(replaceFields);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void addAggregateToMetric(SemanticParseInfo semanticParseInfo) {
|
protected void addAggregateToMetric(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
//add aggregate to all metric
|
//add aggregate to all metric
|
||||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||||
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
||||||
|
|
||||||
List<SchemaElement> metrics = getMetricElements(modelIds);
|
List<SchemaElement> metrics = getMetricElements(queryContext, modelIds);
|
||||||
|
|
||||||
Map<String, String> metricToAggregate = metrics.stream()
|
Map<String, String> metricToAggregate = metrics.stream()
|
||||||
.map(schemaElement -> {
|
.map(schemaElement -> {
|
||||||
@@ -131,8 +127,8 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
|
|||||||
semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql);
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(aggregateSql);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<SchemaElement> getMetricElements(Set<Long> modelIds) {
|
protected List<SchemaElement> getMetricElements(QueryContext queryContext, Set<Long> modelIds) {
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
return semanticSchema.getMetrics(modelIds);
|
return semanticSchema.getMetrics(modelIds);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
/**
|
||||||
|
* Perform SQL corrections on the "From" section in S2SQL.
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class FromCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
String modelName = semanticParseInfo.getModel().getName();
|
||||||
|
String correctSql = SqlParserReplaceHelper
|
||||||
|
.replaceTable(semanticParseInfo.getSqlInfo().getCorrectS2SQL(), modelName);
|
||||||
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctSql);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -1,21 +1,18 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Perform SQL corrections on the "Group by" section in S2SQL.
|
* Perform SQL corrections on the "Group by" section in S2SQL.
|
||||||
@@ -24,19 +21,19 @@ import java.util.stream.Collectors;
|
|||||||
public class GroupByCorrector extends BaseSemanticCorrector {
|
public class GroupByCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
|
||||||
addGroupByFields(semanticParseInfo);
|
addGroupByFields(queryContext, semanticParseInfo);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addGroupByFields(SemanticParseInfo semanticParseInfo) {
|
private void addGroupByFields(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
||||||
|
|
||||||
//add dimension group by
|
//add dimension group by
|
||||||
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
|
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
|
||||||
String correctS2SQL = sqlInfo.getCorrectS2SQL();
|
String correctS2SQL = sqlInfo.getCorrectS2SQL();
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
//add alias field name
|
//add alias field name
|
||||||
Set<String> dimensions = semanticSchema.getDimensions(modelIds).stream()
|
Set<String> dimensions = semanticSchema.getDimensions(modelIds).stream()
|
||||||
.flatMap(
|
.flatMap(
|
||||||
@@ -77,15 +74,15 @@ public class GroupByCorrector extends BaseSemanticCorrector {
|
|||||||
.collect(Collectors.toSet());
|
.collect(Collectors.toSet());
|
||||||
semanticParseInfo.getSqlInfo().setCorrectS2SQL(SqlParserAddHelper.addGroupBy(correctS2SQL, groupByFields));
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(SqlParserAddHelper.addGroupBy(correctS2SQL, groupByFields));
|
||||||
|
|
||||||
addAggregate(semanticParseInfo);
|
addAggregate(queryContext, semanticParseInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addAggregate(SemanticParseInfo semanticParseInfo) {
|
private void addAggregate(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
List<String> sqlGroupByFields = SqlParserSelectHelper.getGroupByFields(
|
List<String> sqlGroupByFields = SqlParserSelectHelper.getGroupByFields(
|
||||||
semanticParseInfo.getSqlInfo().getCorrectS2SQL());
|
semanticParseInfo.getSqlInfo().getCorrectS2SQL());
|
||||||
if (CollectionUtils.isEmpty(sqlGroupByFields)) {
|
if (CollectionUtils.isEmpty(sqlGroupByFields)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
addAggregateToMetric(semanticParseInfo);
|
addAggregateToMetric(queryContext, semanticParseInfo);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,18 +1,14 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import net.sf.jsqlparser.expression.Expression;
|
import net.sf.jsqlparser.expression.Expression;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -24,20 +20,20 @@ import org.springframework.util.CollectionUtils;
|
|||||||
public class HavingCorrector extends BaseSemanticCorrector {
|
public class HavingCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
|
||||||
//add aggregate to all metric
|
//add aggregate to all metric
|
||||||
addHaving(semanticParseInfo);
|
addHaving(queryContext, semanticParseInfo);
|
||||||
|
|
||||||
//add having expression filed to select
|
//add having expression filed to select
|
||||||
addHavingToSelect(semanticParseInfo);
|
addHavingToSelect(semanticParseInfo);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addHaving(SemanticParseInfo semanticParseInfo) {
|
private void addHaving(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
||||||
|
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
|
|
||||||
Set<String> metrics = semanticSchema.getMetrics(modelIds).stream()
|
Set<String> metrics = semanticSchema.getMetrics(modelIds).stream()
|
||||||
.map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
|
.map(schemaElement -> schemaElement.getName()).collect(Collectors.toSet());
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.ParseResult;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.ParseResult;
|
||||||
import com.tencent.supersonic.chat.query.llm.s2sql.LLMReq.ElementValue;
|
import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq.ElementValue;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.AggregateEnum;
|
import com.tencent.supersonic.common.util.jsqlparser.AggregateEnum;
|
||||||
@@ -24,7 +24,7 @@ import org.springframework.util.CollectionUtils;
|
|||||||
public class SchemaCorrector extends BaseSemanticCorrector {
|
public class SchemaCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
|
||||||
correctAggFunction(semanticParseInfo);
|
correctAggFunction(semanticParseInfo);
|
||||||
|
|
||||||
@@ -34,7 +34,7 @@ public class SchemaCorrector extends BaseSemanticCorrector {
|
|||||||
|
|
||||||
updateFieldValueByLinkingValue(semanticParseInfo);
|
updateFieldValueByLinkingValue(semanticParseInfo);
|
||||||
|
|
||||||
correctFieldName(semanticParseInfo);
|
correctFieldName(queryContext, semanticParseInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void correctAggFunction(SemanticParseInfo semanticParseInfo) {
|
private void correctAggFunction(SemanticParseInfo semanticParseInfo) {
|
||||||
@@ -50,8 +50,8 @@ public class SchemaCorrector extends BaseSemanticCorrector {
|
|||||||
sqlInfo.setCorrectS2SQL(replaceAlias);
|
sqlInfo.setCorrectS2SQL(replaceAlias);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void correctFieldName(SemanticParseInfo semanticParseInfo) {
|
private void correctFieldName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
Map<String, String> fieldNameMap = getFieldNameMap(semanticParseInfo.getModel().getModelIds());
|
Map<String, String> fieldNameMap = getFieldNameMap(queryContext, semanticParseInfo.getModel().getModelIds());
|
||||||
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
|
SqlInfo sqlInfo = semanticParseInfo.getSqlInfo();
|
||||||
String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap);
|
String sql = SqlParserReplaceHelper.replaceFields(sqlInfo.getCorrectS2SQL(), fieldNameMap);
|
||||||
sqlInfo.setCorrectS2SQL(sql);
|
sqlInfo.setCorrectS2SQL(sql);
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -14,7 +14,7 @@ import org.springframework.util.CollectionUtils;
|
|||||||
public class SelectCorrector extends BaseSemanticCorrector {
|
public class SelectCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||||
List<String> aggregateFields = SqlParserSelectHelper.getAggregateFields(correctS2SQL);
|
List<String> aggregateFields = SqlParserSelectHelper.getAggregateFields(correctS2SQL);
|
||||||
List<String> selectFields = SqlParserSelectHelper.getSelectFields(correctS2SQL);
|
List<String> selectFields = SqlParserSelectHelper.getSelectFields(correctS2SQL);
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.api.component;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A semantic corrector checks validity of extracted semantic information and
|
* A semantic corrector checks validity of extracted semantic information and
|
||||||
@@ -9,5 +9,5 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
|||||||
*/
|
*/
|
||||||
public interface SemanticCorrector {
|
public interface SemanticCorrector {
|
||||||
|
|
||||||
void correct(QueryReq queryReq, SemanticParseInfo semanticParseInfo);
|
void correct(QueryContext queryContext, SemanticParseInfo semanticParseInfo);
|
||||||
}
|
}
|
||||||
@@ -1,20 +1,24 @@
|
|||||||
package com.tencent.supersonic.chat.corrector;
|
package com.tencent.supersonic.chat.core.corrector;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
|
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.core.parser.sql.llm.S2SqlDateHelper;
|
||||||
import com.tencent.supersonic.chat.parser.sql.llm.S2SqlDateHelper;
|
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.StringUtil;
|
import com.tencent.supersonic.common.util.StringUtil;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import net.sf.jsqlparser.JSQLParserException;
|
import net.sf.jsqlparser.JSQLParserException;
|
||||||
import net.sf.jsqlparser.expression.Expression;
|
import net.sf.jsqlparser.expression.Expression;
|
||||||
@@ -23,13 +27,6 @@ import org.apache.commons.lang3.StringUtils;
|
|||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Perform SQL corrections on the "Where" section in S2SQL.
|
* Perform SQL corrections on the "Where" section in S2SQL.
|
||||||
*/
|
*/
|
||||||
@@ -37,19 +34,19 @@ import java.util.stream.Collectors;
|
|||||||
public class WhereCorrector extends BaseSemanticCorrector {
|
public class WhereCorrector extends BaseSemanticCorrector {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doCorrect(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
public void doCorrect(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
|
|
||||||
addDateIfNotExist(semanticParseInfo);
|
addDateIfNotExist(queryContext, semanticParseInfo);
|
||||||
|
|
||||||
parserDateDiffFunction(semanticParseInfo);
|
parserDateDiffFunction(semanticParseInfo);
|
||||||
|
|
||||||
addQueryFilter(queryReq, semanticParseInfo);
|
addQueryFilter(queryContext, semanticParseInfo);
|
||||||
|
|
||||||
updateFieldValueByTechName(semanticParseInfo);
|
updateFieldValueByTechName(queryContext, semanticParseInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addQueryFilter(QueryReq queryReq, SemanticParseInfo semanticParseInfo) {
|
private void addQueryFilter(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
String queryFilter = getQueryFilter(queryReq.getQueryFilters());
|
String queryFilter = getQueryFilter(queryContext.getQueryFilters());
|
||||||
|
|
||||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||||
|
|
||||||
@@ -72,11 +69,11 @@ public class WhereCorrector extends BaseSemanticCorrector {
|
|||||||
semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL);
|
semanticParseInfo.getSqlInfo().setCorrectS2SQL(correctS2SQL);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addDateIfNotExist(SemanticParseInfo semanticParseInfo) {
|
private void addDateIfNotExist(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectS2SQL();
|
||||||
List<String> whereFields = SqlParserSelectHelper.getWhereFields(correctS2SQL);
|
List<String> whereFields = SqlParserSelectHelper.getWhereFields(correctS2SQL);
|
||||||
if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
|
if (CollectionUtils.isEmpty(whereFields) || !TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
|
||||||
String currentDate = S2SqlDateHelper.getReferenceDate(semanticParseInfo.getModelId());
|
String currentDate = S2SqlDateHelper.getReferenceDate(queryContext, semanticParseInfo.getModelId());
|
||||||
if (StringUtils.isNotBlank(currentDate)) {
|
if (StringUtils.isNotBlank(currentDate)) {
|
||||||
correctS2SQL = SqlParserAddHelper.addParenthesisToWhere(correctS2SQL);
|
correctS2SQL = SqlParserAddHelper.addParenthesisToWhere(correctS2SQL);
|
||||||
correctS2SQL = SqlParserAddHelper.addWhere(
|
correctS2SQL = SqlParserAddHelper.addWhere(
|
||||||
@@ -100,8 +97,8 @@ public class WhereCorrector extends BaseSemanticCorrector {
|
|||||||
.collect(Collectors.joining(Constants.AND_UPPER));
|
.collect(Collectors.joining(Constants.AND_UPPER));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void updateFieldValueByTechName(SemanticParseInfo semanticParseInfo) {
|
private void updateFieldValueByTechName(QueryContext queryContext, SemanticParseInfo semanticParseInfo) {
|
||||||
SemanticSchema semanticSchema = ContextUtils.getBean(SchemaService.class).getSemanticSchema();
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
Set<Long> modelIds = semanticParseInfo.getModel().getModelIds();
|
||||||
List<SchemaElement> dimensions = semanticSchema.getDimensions(modelIds);
|
List<SchemaElement> dimensions = semanticSchema.getDimensions(modelIds);
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
|
||||||
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
public enum DictUpdateMode {
|
public enum DictUpdateMode {
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge.dictionary;
|
||||||
|
|
||||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||||
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.hankcs.hanlp.corpus.io.IIOAdapter;
|
import com.hankcs.hanlp.corpus.io.IIOAdapter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.tencent.supersonic.knowledge.service;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
import com.tencent.supersonic.chat.core.utils.NatureHelper;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.core.config.LocalFileConfig;
|
||||||
import java.io.BufferedWriter;
|
import java.io.BufferedWriter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
@@ -10,7 +11,6 @@ import java.nio.file.Paths;
|
|||||||
import java.nio.file.StandardCopyOption;
|
import java.nio.file.StandardCopyOption;
|
||||||
import java.nio.file.StandardOpenOption;
|
import java.nio.file.StandardOpenOption;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import static com.hankcs.hanlp.utility.Predefine.logger;
|
import static com.hankcs.hanlp.utility.Predefine.logger;
|
||||||
|
|
||||||
@@ -15,8 +15,8 @@ import com.hankcs.hanlp.seg.common.Term;
|
|||||||
import com.hankcs.hanlp.utility.LexiconUtility;
|
import com.hankcs.hanlp.utility.LexiconUtility;
|
||||||
import com.hankcs.hanlp.utility.Predefine;
|
import com.hankcs.hanlp.utility.Predefine;
|
||||||
import com.hankcs.hanlp.utility.TextUtility;
|
import com.hankcs.hanlp.utility.TextUtility;
|
||||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
import com.tencent.supersonic.chat.core.knowledge.dictionary.DictionaryAttributeUtil;
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
import java.io.BufferedOutputStream;
|
import java.io.BufferedOutputStream;
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
@@ -353,7 +353,7 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean insert(String word, String natureWithFrequency) {
|
public synchronized boolean insert(String word, String natureWithFrequency) {
|
||||||
if (word == null) {
|
if (word == null) {
|
||||||
return false;
|
return false;
|
||||||
} else {
|
} else {
|
||||||
@@ -1,27 +1,27 @@
|
|||||||
package com.tencent.supersonic.knowledge.service;
|
package com.tencent.supersonic.chat.core.knowledge;
|
||||||
|
|
||||||
import com.hankcs.hanlp.collection.trie.bintrie.BaseNode;
|
import com.hankcs.hanlp.collection.trie.bintrie.BaseNode;
|
||||||
import com.hankcs.hanlp.collection.trie.bintrie.BinTrie;
|
import com.hankcs.hanlp.collection.trie.bintrie.BinTrie;
|
||||||
import com.hankcs.hanlp.corpus.tag.Nature;
|
import com.hankcs.hanlp.corpus.tag.Nature;
|
||||||
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
import com.hankcs.hanlp.dictionary.CoreDictionary;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.dictionary.DictionaryAttributeUtil;
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictionaryAttributeUtil;
|
import java.util.ArrayList;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.PriorityQueue;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@Service
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class SearchService {
|
public class SearchService {
|
||||||
|
|
||||||
@@ -156,5 +156,15 @@ public class SearchService {
|
|||||||
suffixTrie.remove(dictWord.getWord());
|
suffixTrie.remove(dictWord.getWord());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static List<String> getDimensionValue(DimensionValueReq dimensionValueReq) {
|
||||||
|
String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT
|
||||||
|
+ dimensionValueReq.getElementID();
|
||||||
|
PriorityQueue<Term> terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature);
|
||||||
|
if (org.apache.commons.collections.CollectionUtils.isEmpty(terms)) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
return terms.stream().map(term -> term.getWord()).collect(Collectors.toList());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -1,13 +1,11 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -1,13 +1,12 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -1,13 +1,11 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
@@ -1,13 +1,12 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DictWord;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DictWord;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.dictionary.builder;
|
package com.tencent.supersonic.chat.core.knowledge.builder;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||||
@@ -1,10 +1,9 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import com.google.common.cache.Cache;
|
import com.google.common.cache.Cache;
|
||||||
import com.google.common.cache.CacheBuilder;
|
import com.google.common.cache.CacheBuilder;
|
||||||
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
@@ -1,36 +1,35 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
|
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
|
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
|
import com.tencent.supersonic.headless.api.request.PageDimensionReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
|
import com.tencent.supersonic.headless.api.request.PageMetricReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
|
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
|
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
|
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
|
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
import com.tencent.supersonic.headless.api.response.DomainResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
|
import com.tencent.supersonic.headless.api.response.ExplainResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
|
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
|
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
|
||||||
import com.tencent.supersonic.semantic.model.domain.DimensionService;
|
import com.tencent.supersonic.headless.server.service.DimensionService;
|
||||||
import com.tencent.supersonic.semantic.model.domain.MetricService;
|
import com.tencent.supersonic.headless.server.service.MetricService;
|
||||||
import com.tencent.supersonic.semantic.query.service.QueryService;
|
import com.tencent.supersonic.headless.server.service.QueryService;
|
||||||
import com.tencent.supersonic.semantic.query.service.SchemaService;
|
import com.tencent.supersonic.headless.server.service.SchemaService;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
|
public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
|
||||||
|
|
||||||
@@ -41,20 +40,20 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
|
|||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
@Override
|
@Override
|
||||||
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
|
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
|
||||||
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
|
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
|
||||||
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
|
QuerySqlReq querySQLReq = new QuerySqlReq();
|
||||||
queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
|
querySQLReq.setSql(queryStructReq.getCorrectS2SQL());
|
||||||
queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet());
|
querySQLReq.setModelIds(queryStructReq.getModelIdSet());
|
||||||
queryS2SQLReq.setVariables(new HashMap<>());
|
querySQLReq.setParams(new ArrayList<>());
|
||||||
return queryByS2SQL(queryS2SQLReq, user);
|
return queryByS2SQL(querySQLReq, user);
|
||||||
}
|
}
|
||||||
queryService = ContextUtils.getBean(QueryService.class);
|
queryService = ContextUtils.getBean(QueryService.class);
|
||||||
return queryService.queryByStructWithAuth(queryStructReq, user);
|
return queryService.queryByStructWithAuth(queryStructReq, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
|
public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
|
||||||
try {
|
try {
|
||||||
queryService = ContextUtils.getBean(QueryService.class);
|
queryService = ContextUtils.getBean(QueryService.class);
|
||||||
return queryService.queryByMultiStruct(queryMultiStructReq, user);
|
return queryService.queryByMultiStruct(queryMultiStructReq, user);
|
||||||
@@ -66,15 +65,15 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) {
|
public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) {
|
||||||
queryService = ContextUtils.getBean(QueryService.class);
|
queryService = ContextUtils.getBean(QueryService.class);
|
||||||
Object object = queryService.queryBySql(queryS2SQLReq, user);
|
SemanticQueryResp object = queryService.queryBySql(querySQLReq, user);
|
||||||
return JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp.class);
|
return JsonUtil.toObject(JsonUtil.toString(object), SemanticQueryResp.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
|
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
|
||||||
queryService = ContextUtils.getBean(QueryService.class);
|
queryService = ContextUtils.getBean(QueryService.class);
|
||||||
return queryService.queryDimValue(queryDimValueReq, user);
|
return queryService.queryDimValue(queryDimValueReq, user);
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
@@ -6,12 +6,12 @@ import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
|
|||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
|
import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
|
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.RelateDimension;
|
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
|
||||||
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
|
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
|
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
|
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
@@ -1,4 +1,9 @@
|
|||||||
package com.tencent.supersonic.knowledge.semantic;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON;
|
import com.alibaba.fastjson.JSON;
|
||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
@@ -6,29 +11,36 @@ import com.google.gson.Gson;
|
|||||||
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
|
||||||
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.core.config.DefaultSemanticConfig;
|
||||||
import com.tencent.supersonic.common.pojo.ResultData;
|
import com.tencent.supersonic.common.pojo.ResultData;
|
||||||
import com.tencent.supersonic.common.pojo.enums.ReturnCode;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.ReturnCode;
|
||||||
import com.tencent.supersonic.common.pojo.exception.CommonException;
|
import com.tencent.supersonic.common.pojo.exception.CommonException;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
import com.tencent.supersonic.common.util.S2ThreadContext;
|
import com.tencent.supersonic.common.util.S2ThreadContext;
|
||||||
import com.tencent.supersonic.common.util.ThreadContext;
|
import com.tencent.supersonic.common.util.ThreadContext;
|
||||||
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
|
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
|
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
|
import com.tencent.supersonic.headless.api.request.PageDimensionReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
|
import com.tencent.supersonic.headless.api.request.PageMetricReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
|
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
|
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
|
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
|
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
import com.tencent.supersonic.headless.api.response.DomainResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
|
import com.tencent.supersonic.headless.api.response.ExplainResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
|
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
|
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
@@ -42,18 +54,6 @@ import org.springframework.http.ResponseEntity;
|
|||||||
import org.springframework.web.client.RestTemplate;
|
import org.springframework.web.client.RestTemplate;
|
||||||
import org.springframework.web.util.UriComponentsBuilder;
|
import org.springframework.web.util.UriComponentsBuilder;
|
||||||
|
|
||||||
import java.net.URI;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
|
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
|
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
|
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
|
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
|
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
|
public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
|
||||||
|
|
||||||
@@ -61,8 +61,8 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
|
|||||||
|
|
||||||
private AuthenticationConfig authenticationConfig;
|
private AuthenticationConfig authenticationConfig;
|
||||||
|
|
||||||
private ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>> structTypeRef =
|
private ParameterizedTypeReference<ResultData<SemanticQueryResp>> structTypeRef =
|
||||||
new ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>>() {
|
new ParameterizedTypeReference<ResultData<SemanticQueryResp>>() {
|
||||||
};
|
};
|
||||||
|
|
||||||
private ParameterizedTypeReference<ResultData<ExplainResp>> explainTypeRef =
|
private ParameterizedTypeReference<ResultData<ExplainResp>> explainTypeRef =
|
||||||
@@ -70,13 +70,13 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
|
|||||||
};
|
};
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
|
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
|
||||||
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
|
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
|
||||||
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
|
QuerySqlReq querySQLReq = new QuerySqlReq();
|
||||||
queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
|
querySQLReq.setSql(queryStructReq.getCorrectS2SQL());
|
||||||
queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet());
|
querySQLReq.setModelIds(queryStructReq.getModelIdSet());
|
||||||
queryS2SQLReq.setVariables(new HashMap<>());
|
querySQLReq.setParams(new ArrayList<>());
|
||||||
return queryByS2SQL(queryS2SQLReq, user);
|
return queryByS2SQL(querySQLReq, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||||
@@ -86,7 +86,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
|
public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
|
||||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||||
return searchByRestTemplate(
|
return searchByRestTemplate(
|
||||||
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByMultiStructPath(),
|
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByMultiStructPath(),
|
||||||
@@ -94,44 +94,44 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) {
|
public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) {
|
||||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||||
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
|
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
|
||||||
new Gson().toJson(queryS2SQLReq));
|
new Gson().toJson(querySQLReq));
|
||||||
}
|
}
|
||||||
|
|
||||||
public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) {
|
public SemanticQueryResp searchByRestTemplate(String url, String jsonReq) {
|
||||||
HttpHeaders headers = new HttpHeaders();
|
HttpHeaders headers = new HttpHeaders();
|
||||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||||
fillToken(headers);
|
fillToken(headers);
|
||||||
URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
|
URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
|
||||||
HttpEntity<String> entity = new HttpEntity<>(jsonReq, headers);
|
HttpEntity<String> entity = new HttpEntity<>(jsonReq, headers);
|
||||||
log.info("url:{},searchByRestTemplate:{}", url, entity.getBody());
|
log.info("url:{},searchByRestTemplate:{}", url, entity.getBody());
|
||||||
ResultData<QueryResultWithSchemaResp> responseBody;
|
ResultData<SemanticQueryResp> responseBody;
|
||||||
try {
|
try {
|
||||||
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
|
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
|
||||||
|
|
||||||
ResponseEntity<ResultData<QueryResultWithSchemaResp>> responseEntity = restTemplate.exchange(
|
ResponseEntity<ResultData<SemanticQueryResp>> responseEntity = restTemplate.exchange(
|
||||||
requestUrl, HttpMethod.POST, entity, structTypeRef);
|
requestUrl, HttpMethod.POST, entity, structTypeRef);
|
||||||
responseBody = responseEntity.getBody();
|
responseBody = responseEntity.getBody();
|
||||||
log.info("ApiResponse<QueryResultWithColumns> responseBody:{}", responseBody);
|
log.info("ApiResponse<QueryResultWithColumns> responseBody:{}", responseBody);
|
||||||
QueryResultWithSchemaResp semanticQuery = new QueryResultWithSchemaResp();
|
SemanticQueryResp schemaResp = new SemanticQueryResp();
|
||||||
if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) {
|
if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) {
|
||||||
QueryResultWithSchemaResp data = responseBody.getData();
|
SemanticQueryResp data = responseBody.getData();
|
||||||
semanticQuery.setColumns(data.getColumns());
|
schemaResp.setColumns(data.getColumns());
|
||||||
semanticQuery.setResultList(data.getResultList());
|
schemaResp.setResultList(data.getResultList());
|
||||||
semanticQuery.setSql(data.getSql());
|
schemaResp.setSql(data.getSql());
|
||||||
semanticQuery.setQueryAuthorization(data.getQueryAuthorization());
|
schemaResp.setQueryAuthorization(data.getQueryAuthorization());
|
||||||
return semanticQuery;
|
return schemaResp;
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new RuntimeException("search semantic interface error,url:" + url, e);
|
throw new RuntimeException("search headless interface error,url:" + url, e);
|
||||||
}
|
}
|
||||||
throw new CommonException(responseBody.getCode(), responseBody.getMsg());
|
throw new CommonException(responseBody.getCode(), responseBody.getMsg());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
|
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
|
||||||
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
|
||||||
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl()
|
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl()
|
||||||
+ defaultSemanticConfig.getQueryDimValuePath(),
|
+ defaultSemanticConfig.getQueryDimValuePath(),
|
||||||
@@ -1,23 +1,23 @@
|
|||||||
package com.tencent.supersonic.chat.api.component;
|
package com.tencent.supersonic.chat.core.knowledge.semantic;
|
||||||
|
|
||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
import com.tencent.supersonic.common.pojo.enums.AuthType;
|
||||||
import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
|
import com.tencent.supersonic.headless.api.request.PageDimensionReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
|
import com.tencent.supersonic.headless.api.request.PageMetricReq;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.DomainResp;
|
import com.tencent.supersonic.headless.api.response.DomainResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
|
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
|
import com.tencent.supersonic.headless.api.response.ExplainResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
|
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ModelResp;
|
import com.tencent.supersonic.headless.api.response.ModelResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
|
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
|
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
|
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryS2SQLReq;
|
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
|
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@@ -34,13 +34,13 @@ import java.util.List;
|
|||||||
*/
|
*/
|
||||||
public interface SemanticInterpreter {
|
public interface SemanticInterpreter {
|
||||||
|
|
||||||
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user);
|
SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user);
|
||||||
|
|
||||||
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
|
SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
|
||||||
|
|
||||||
QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user);
|
SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user);
|
||||||
|
|
||||||
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
|
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
|
||||||
|
|
||||||
List<ModelSchema> getModelSchema();
|
List<ModelSchema> getModelSchema();
|
||||||
|
|
||||||
@@ -1,14 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.service.SemanticService;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@@ -68,10 +66,10 @@ public abstract class BaseMapper implements SchemaMapper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public SchemaElement getSchemaElement(Long modelId, SchemaElementType elementType, Long elementID) {
|
public SchemaElement getSchemaElement(Long modelId, SchemaElementType elementType, Long elementID,
|
||||||
|
SemanticSchema semanticSchema) {
|
||||||
SchemaElement element = new SchemaElement();
|
SchemaElement element = new SchemaElement();
|
||||||
SemanticService schemaService = ContextUtils.getBean(SemanticService.class);
|
ModelSchema modelSchema = semanticSchema.getModelSchemaMap().get(modelId);
|
||||||
ModelSchema modelSchema = schemaService.getModelSchema(modelId);
|
|
||||||
if (Objects.isNull(modelSchema)) {
|
if (Objects.isNull(modelSchema)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
import com.tencent.supersonic.chat.core.utils.NatureHelper;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -28,7 +28,7 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Map<MatchText, List<T>> match(QueryContext queryContext, List<Term> terms, Set<Long> detectModelIds) {
|
public Map<MatchText, List<T>> match(QueryContext queryContext, List<Term> terms, Set<Long> detectModelIds) {
|
||||||
String text = queryContext.getRequest().getQueryText();
|
String text = queryContext.getQueryText();
|
||||||
if (Objects.isNull(terms) || StringUtils.isEmpty(text)) {
|
if (Objects.isNull(terms) || StringUtils.isEmpty(text)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -44,7 +44,7 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
|
|||||||
|
|
||||||
public List<T> detect(QueryContext queryContext, List<Term> terms, Set<Long> detectModelIds) {
|
public List<T> detect(QueryContext queryContext, List<Term> terms, Set<Long> detectModelIds) {
|
||||||
Map<Integer, Integer> regOffsetToLength = getRegOffsetToLength(terms);
|
Map<Integer, Integer> regOffsetToLength = getRegOffsetToLength(terms);
|
||||||
String text = queryContext.getRequest().getQueryText();
|
String text = queryContext.getQueryText();
|
||||||
Set<T> results = new HashSet<>();
|
Set<T> results = new HashSet<>();
|
||||||
|
|
||||||
Set<String> detectSegments = new HashSet<>();
|
Set<String> detectSegments = new HashSet<>();
|
||||||
@@ -102,7 +102,7 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public List<T> getMatches(QueryContext queryContext, List<Term> terms) {
|
public List<T> getMatches(QueryContext queryContext, List<Term> terms) {
|
||||||
Set<Long> detectModelIds = mapperHelper.getModelIds(queryContext.getRequest());
|
Set<Long> detectModelIds = mapperHelper.getModelIds(queryContext.getModelId(), queryContext.getAgent());
|
||||||
terms = filterByModelIds(terms, detectModelIds);
|
terms = filterByModelIds(terms, detectModelIds);
|
||||||
Map<MatchText, List<T>> matchResult = match(queryContext, terms, detectModelIds);
|
Map<MatchText, List<T>> matchResult = match(queryContext, terms, detectModelIds);
|
||||||
List<T> matches = new ArrayList<>();
|
List<T> matches = new ArrayList<>();
|
||||||
@@ -1,13 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DatabaseMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -33,14 +32,12 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy<DatabaseMapResult>
|
|||||||
private OptimizationConfig optimizationConfig;
|
private OptimizationConfig optimizationConfig;
|
||||||
@Autowired
|
@Autowired
|
||||||
private MapperHelper mapperHelper;
|
private MapperHelper mapperHelper;
|
||||||
@Autowired
|
|
||||||
private SchemaService schemaService;
|
|
||||||
private List<SchemaElement> allElements;
|
private List<SchemaElement> allElements;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Map<MatchText, List<DatabaseMapResult>> match(QueryContext queryContext, List<Term> terms,
|
public Map<MatchText, List<DatabaseMapResult>> match(QueryContext queryContext, List<Term> terms,
|
||||||
Set<Long> detectModelIds) {
|
Set<Long> detectModelIds) {
|
||||||
this.allElements = getSchemaElements();
|
this.allElements = getSchemaElements(queryContext);
|
||||||
return super.match(queryContext, terms, detectModelIds);
|
return super.match(queryContext, terms, detectModelIds);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,11 +55,11 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy<DatabaseMapResult>
|
|||||||
|
|
||||||
public void detectByStep(QueryContext queryContext, Set<DatabaseMapResult> existResults, Set<Long> detectModelIds,
|
public void detectByStep(QueryContext queryContext, Set<DatabaseMapResult> existResults, Set<Long> detectModelIds,
|
||||||
Integer startIndex, Integer index, int offset) {
|
Integer startIndex, Integer index, int offset) {
|
||||||
String detectSegment = queryContext.getRequest().getQueryText().substring(startIndex, index);
|
String detectSegment = queryContext.getQueryText().substring(startIndex, index);
|
||||||
if (StringUtils.isBlank(detectSegment)) {
|
if (StringUtils.isBlank(detectSegment)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Set<Long> modelIds = mapperHelper.getModelIds(queryContext.getRequest());
|
Set<Long> modelIds = mapperHelper.getModelIds(queryContext.getModelId(), queryContext.getAgent());
|
||||||
|
|
||||||
Double metricDimensionThresholdConfig = getThreshold(queryContext);
|
Double metricDimensionThresholdConfig = getThreshold(queryContext);
|
||||||
|
|
||||||
@@ -90,10 +87,10 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy<DatabaseMapResult>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<SchemaElement> getSchemaElements() {
|
private List<SchemaElement> getSchemaElements(QueryContext queryContext) {
|
||||||
List<SchemaElement> allElements = new ArrayList<>();
|
List<SchemaElement> allElements = new ArrayList<>();
|
||||||
allElements.addAll(schemaService.getSemanticSchema().getDimensions());
|
allElements.addAll(queryContext.getSemanticSchema().getDimensions());
|
||||||
allElements.addAll(schemaService.getSemanticSchema().getMetrics());
|
allElements.addAll(queryContext.getSemanticSchema().getMetrics());
|
||||||
return allElements;
|
return allElements;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
import com.alibaba.fastjson.JSONObject;
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.EmbeddingResult;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.EmbeddingResult;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder;
|
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
@@ -23,7 +23,7 @@ public class EmbeddingMapper extends BaseMapper {
|
|||||||
@Override
|
@Override
|
||||||
public void doMap(QueryContext queryContext) {
|
public void doMap(QueryContext queryContext) {
|
||||||
//1. query from embedding by queryText
|
//1. query from embedding by queryText
|
||||||
String queryText = queryContext.getRequest().getQueryText();
|
String queryText = queryContext.getQueryText();
|
||||||
List<Term> terms = HanlpHelper.getTerms(queryText);
|
List<Term> terms = HanlpHelper.getTerms(queryText);
|
||||||
|
|
||||||
EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class);
|
EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class);
|
||||||
@@ -43,7 +43,8 @@ public class EmbeddingMapper extends BaseMapper {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
long modelId = Long.parseLong(modelIdStr);
|
long modelId = Long.parseLong(modelIdStr);
|
||||||
schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId);
|
schemaElement = getSchemaElement(modelId, schemaElement.getType(), elementId,
|
||||||
|
queryContext.getSemanticSchema());
|
||||||
if (schemaElement == null) {
|
if (schemaElement == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -1,16 +1,16 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.knowledge.EmbeddingResult;
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
|
import com.tencent.supersonic.common.config.EmbeddingConfig;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.common.util.ComponentFactory;
|
import com.tencent.supersonic.common.util.ComponentFactory;
|
||||||
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
import com.tencent.supersonic.common.util.embedding.Retrieval;
|
||||||
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
|
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
|
||||||
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
|
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
|
||||||
import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
|
import com.tencent.supersonic.common.util.embedding.S2EmbeddingStore;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.EmbeddingResult;
|
|
||||||
import com.tencent.supersonic.semantic.model.domain.listener.MetaEmbeddingListener;
|
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -35,6 +35,9 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy<EmbeddingResult> {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private OptimizationConfig optimizationConfig;
|
private OptimizationConfig optimizationConfig;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private EmbeddingConfig embeddingConfig;
|
||||||
|
|
||||||
private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore();
|
private S2EmbeddingStore s2EmbeddingStore = ComponentFactory.getS2EmbeddingStore();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -86,7 +89,7 @@ public class EmbeddingMatchStrategy extends BaseMatchStrategy<EmbeddingResult> {
|
|||||||
.build();
|
.build();
|
||||||
// step2. retrieveQuery by detectSegment
|
// step2. retrieveQuery by detectSegment
|
||||||
List<RetrieveQueryResult> retrieveQueryResults = s2EmbeddingStore.retrieveQuery(
|
List<RetrieveQueryResult> retrieveQueryResults = s2EmbeddingStore.retrieveQuery(
|
||||||
MetaEmbeddingListener.COLLECTION_NAME, retrieveQuery, embeddingNumber);
|
embeddingConfig.getMetaCollectionName(), retrieveQuery, embeddingNumber);
|
||||||
|
|
||||||
if (CollectionUtils.isEmpty(retrieveQueryResults)) {
|
if (CollectionUtils.isEmpty(retrieveQueryResults)) {
|
||||||
return;
|
return;
|
||||||
@@ -1,21 +1,19 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.service.SemanticService;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A mapper capable of converting the VALUE of entity dimension values into ID types.
|
* A mapper capable of converting the VALUE of entity dimension values into ID types.
|
||||||
*/
|
*/
|
||||||
@@ -30,7 +28,7 @@ public class EntityMapper extends BaseMapper {
|
|||||||
if (CollectionUtils.isEmpty(schemaElementMatchList)) {
|
if (CollectionUtils.isEmpty(schemaElementMatchList)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
SchemaElement entity = getEntity(modelId);
|
SchemaElement entity = getEntity(modelId, queryContext);
|
||||||
if (entity == null || entity.getId() == null) {
|
if (entity == null || entity.getId() == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -66,9 +64,9 @@ public class EntityMapper extends BaseMapper {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private SchemaElement getEntity(Long modelId) {
|
private SchemaElement getEntity(Long modelId, QueryContext queryContext) {
|
||||||
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
ModelSchema modelSchema = semanticService.getModelSchema(modelId);
|
ModelSchema modelSchema = semanticSchema.getModelSchemaMap().get(modelId);
|
||||||
if (modelSchema != null && modelSchema.getEntity() != null) {
|
if (modelSchema != null && modelSchema.getEntity() != null) {
|
||||||
return modelSchema.getEntity();
|
return modelSchema.getEntity();
|
||||||
}
|
}
|
||||||
@@ -1,12 +1,11 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.knowledge.SearchService;
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.service.SearchService;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -38,8 +37,7 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
|
|||||||
@Override
|
@Override
|
||||||
public Map<MatchText, List<HanlpMapResult>> match(QueryContext queryContext, List<Term> terms,
|
public Map<MatchText, List<HanlpMapResult>> match(QueryContext queryContext, List<Term> terms,
|
||||||
Set<Long> detectModelIds) {
|
Set<Long> detectModelIds) {
|
||||||
QueryReq queryReq = queryContext.getRequest();
|
String text = queryContext.getQueryText();
|
||||||
String text = queryReq.getQueryText();
|
|
||||||
if (Objects.isNull(terms) || StringUtils.isEmpty(text)) {
|
if (Objects.isNull(terms) || StringUtils.isEmpty(text)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -61,16 +59,14 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
|
|||||||
|
|
||||||
public void detectByStep(QueryContext queryContext, Set<HanlpMapResult> existResults, Set<Long> detectModelIds,
|
public void detectByStep(QueryContext queryContext, Set<HanlpMapResult> existResults, Set<Long> detectModelIds,
|
||||||
Integer startIndex, Integer index, int offset) {
|
Integer startIndex, Integer index, int offset) {
|
||||||
QueryReq queryReq = queryContext.getRequest();
|
String text = queryContext.getQueryText();
|
||||||
String text = queryReq.getQueryText();
|
Integer agentId = queryContext.getAgentId();
|
||||||
Integer agentId = queryReq.getAgentId();
|
|
||||||
String detectSegment = text.substring(startIndex, index);
|
String detectSegment = text.substring(startIndex, index);
|
||||||
|
|
||||||
// step1. pre search
|
// step1. pre search
|
||||||
Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize();
|
Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize();
|
||||||
LinkedHashSet<HanlpMapResult> hanlpMapResults = SearchService.prefixSearch(detectSegment, oneDetectionMaxSize,
|
LinkedHashSet<HanlpMapResult> hanlpMapResults = SearchService.prefixSearch(detectSegment, oneDetectionMaxSize,
|
||||||
agentId,
|
agentId, detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
|
||||||
// step2. suffix search
|
// step2. suffix search
|
||||||
LinkedHashSet<HanlpMapResult> suffixHanlpMapResults = SearchService.suffixSearch(detectSegment,
|
LinkedHashSet<HanlpMapResult> suffixHanlpMapResults = SearchService.suffixSearch(detectSegment,
|
||||||
oneDetectionMaxSize, agentId, detectModelIds).stream()
|
oneDetectionMaxSize, agentId, detectModelIds).stream()
|
||||||
@@ -1,16 +1,16 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.HanlpHelper;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.NatureHelper;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.DatabaseMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.dictionary.HanlpMapResult;
|
|
||||||
import com.tencent.supersonic.knowledge.utils.HanlpHelper;
|
|
||||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@@ -29,13 +29,13 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void doMap(QueryContext queryContext) {
|
public void doMap(QueryContext queryContext) {
|
||||||
String queryText = queryContext.getRequest().getQueryText();
|
String queryText = queryContext.getQueryText();
|
||||||
//1.hanlpDict Match
|
//1.hanlpDict Match
|
||||||
List<Term> terms = HanlpHelper.getTerms(queryText);
|
List<Term> terms = HanlpHelper.getTerms(queryText);
|
||||||
HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class);
|
HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class);
|
||||||
|
|
||||||
List<HanlpMapResult> hanlpMapResults = hanlpMatchStrategy.getMatches(queryContext, terms);
|
List<HanlpMapResult> hanlpMapResults = hanlpMatchStrategy.getMatches(queryContext, terms);
|
||||||
convertHanlpMapResultToMapInfo(hanlpMapResults, queryContext.getMapInfo(), terms);
|
convertHanlpMapResultToMapInfo(hanlpMapResults, queryContext, terms);
|
||||||
|
|
||||||
//2.database Match
|
//2.database Match
|
||||||
DatabaseMatchStrategy databaseMatchStrategy = ContextUtils.getBean(DatabaseMatchStrategy.class);
|
DatabaseMatchStrategy databaseMatchStrategy = ContextUtils.getBean(DatabaseMatchStrategy.class);
|
||||||
@@ -44,7 +44,7 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
convertDatabaseMapResultToMapInfo(queryContext, databaseResults);
|
convertDatabaseMapResultToMapInfo(queryContext, databaseResults);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void convertHanlpMapResultToMapInfo(List<HanlpMapResult> mapResults, SchemaMapInfo schemaMap,
|
private void convertHanlpMapResultToMapInfo(List<HanlpMapResult> mapResults, QueryContext queryContext,
|
||||||
List<Term> terms) {
|
List<Term> terms) {
|
||||||
if (CollectionUtils.isEmpty(mapResults)) {
|
if (CollectionUtils.isEmpty(mapResults)) {
|
||||||
return;
|
return;
|
||||||
@@ -65,7 +65,8 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
Long elementID = NatureHelper.getElementID(nature);
|
Long elementID = NatureHelper.getElementID(nature);
|
||||||
SchemaElement element = getSchemaElement(modelId, elementType, elementID);
|
SchemaElement element = getSchemaElement(modelId, elementType, elementID,
|
||||||
|
queryContext.getSemanticSchema());
|
||||||
if (element == null) {
|
if (element == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -81,7 +82,7 @@ public class KeywordMapper extends BaseMapper {
|
|||||||
.detectWord(hanlpMapResult.getDetectWord())
|
.detectWord(hanlpMapResult.getDetectWord())
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
addToSchemaMap(schemaMap, modelId, schemaElementMatch);
|
addToSchemaMap(queryContext.getMapInfo(), modelId, schemaElementMatch);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,12 +1,10 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.algorithm.EditDistance;
|
import com.hankcs.hanlp.algorithm.EditDistance;
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.core.agent.Agent;
|
||||||
import com.tencent.supersonic.chat.config.OptimizationConfig;
|
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
|
||||||
import com.tencent.supersonic.chat.service.AgentService;
|
import com.tencent.supersonic.chat.core.utils.NatureHelper;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.knowledge.utils.NatureHelper;
|
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -83,15 +81,14 @@ public class MapperHelper {
|
|||||||
detectSegment.length());
|
detectSegment.length());
|
||||||
}
|
}
|
||||||
|
|
||||||
public Set<Long> getModelIds(QueryReq request) {
|
public Set<Long> getModelIds(Long modelId, Agent agent) {
|
||||||
|
|
||||||
Long modelId = request.getModelId();
|
Set<Long> detectModelIds = new HashSet<>();
|
||||||
|
if (Objects.nonNull(agent)) {
|
||||||
AgentService agentService = ContextUtils.getBean(AgentService.class);
|
detectModelIds = agent.getModelIds(null);
|
||||||
|
}
|
||||||
Set<Long> detectModelIds = agentService.getModelIds(request.getAgentId(), null);
|
|
||||||
//contains all
|
//contains all
|
||||||
if (agentService.containsAllModel(detectModelIds)) {
|
if (Agent.containsAllModel(detectModelIds)) {
|
||||||
if (Objects.nonNull(modelId) && modelId > 0) {
|
if (Objects.nonNull(modelId) && modelId > 0) {
|
||||||
Set<Long> result = new HashSet<>();
|
Set<Long> result = new HashSet<>();
|
||||||
result.add(modelId);
|
result.add(modelId);
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
@@ -1,16 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaModelClusterMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||||
import com.tencent.supersonic.chat.utils.ModelClusterBuilder;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.ModelClusterBuilder;
|
||||||
import com.tencent.supersonic.common.pojo.ModelCluster;
|
import com.tencent.supersonic.common.pojo.ModelCluster;
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.knowledge.service.SchemaService;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -27,8 +23,7 @@ public class ModelClusterMapper implements SchemaMapper {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void map(QueryContext queryContext) {
|
public void map(QueryContext queryContext) {
|
||||||
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
|
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
|
||||||
SemanticSchema semanticSchema = schemaService.getSemanticSchema();
|
|
||||||
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
|
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
|
||||||
List<ModelCluster> modelClusters = buildModelClusterMatched(schemaMapInfo, semanticSchema);
|
List<ModelCluster> modelClusters = buildModelClusterMatched(schemaMapInfo, semanticSchema);
|
||||||
Map<String, List<SchemaElementMatch>> modelClusterElementMatches = new HashMap<>();
|
Map<String, List<SchemaElementMatch>> modelClusterElementMatches = new HashMap<>();
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
@@ -10,10 +10,10 @@ import lombok.ToString;
|
|||||||
public class ModelWithSemanticType implements Serializable {
|
public class ModelWithSemanticType implements Serializable {
|
||||||
|
|
||||||
private Long model;
|
private Long model;
|
||||||
private SchemaElementType semanticType;
|
private SchemaElementType schemaElementType;
|
||||||
|
|
||||||
public ModelWithSemanticType(Long model, SchemaElementType semanticType) {
|
public ModelWithSemanticType(Long model, SchemaElementType schemaElementType) {
|
||||||
this.model = model;
|
this.model = model;
|
||||||
this.semanticType = semanticType;
|
this.schemaElementType = schemaElementType;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,31 +1,29 @@
|
|||||||
package com.tencent.supersonic.chat.mapper;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
|
||||||
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder;
|
||||||
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import com.tencent.supersonic.common.pojo.Constants;
|
import com.tencent.supersonic.common.pojo.Constants;
|
||||||
import com.tencent.supersonic.knowledge.dictionary.builder.BaseWordBuilder;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class QueryFilterMapper implements SchemaMapper {
|
public class QueryFilterMapper implements SchemaMapper {
|
||||||
|
|
||||||
private double similarity = 1.0;
|
private double similarity = 1.0;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void map(QueryContext queryContext) {
|
public void map(QueryContext queryContext) {
|
||||||
QueryReq queryReq = queryContext.getRequest();
|
Long modelId = queryContext.getModelId();
|
||||||
Long modelId = queryReq.getModelId();
|
|
||||||
if (modelId == null || modelId <= 0) {
|
if (modelId == null || modelId <= 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -36,7 +34,7 @@ public class QueryFilterMapper implements SchemaMapper {
|
|||||||
schemaElementMatches = Lists.newArrayList();
|
schemaElementMatches = Lists.newArrayList();
|
||||||
schemaMapInfo.setMatchedElements(modelId, schemaElementMatches);
|
schemaMapInfo.setMatchedElements(modelId, schemaElementMatches);
|
||||||
}
|
}
|
||||||
addValueSchemaElementMatch(queryContext, schemaElementMatches, queryReq.getQueryFilters());
|
addValueSchemaElementMatch(queryContext, schemaElementMatches);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void clearOtherSchemaElementMatch(Long modelId, SchemaMapInfo schemaMapInfo) {
|
private void clearOtherSchemaElementMatch(Long modelId, SchemaMapInfo schemaMapInfo) {
|
||||||
@@ -48,12 +46,12 @@ public class QueryFilterMapper implements SchemaMapper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private List<SchemaElementMatch> addValueSchemaElementMatch(QueryContext queryContext,
|
private List<SchemaElementMatch> addValueSchemaElementMatch(QueryContext queryContext,
|
||||||
List<SchemaElementMatch> candidateElementMatches,
|
List<SchemaElementMatch> candidateElementMatches) {
|
||||||
QueryFilters queryFilter) {
|
QueryFilters queryFilters = queryContext.getQueryFilters();
|
||||||
if (queryFilter == null || CollectionUtils.isEmpty(queryFilter.getFilters())) {
|
if (queryFilters == null || CollectionUtils.isEmpty(queryFilters.getFilters())) {
|
||||||
return candidateElementMatches;
|
return candidateElementMatches;
|
||||||
}
|
}
|
||||||
for (QueryFilter filter : queryFilter.getFilters()) {
|
for (QueryFilter filter : queryFilters.getFilters()) {
|
||||||
if (checkExistSameValueSchemaElementMatch(filter, candidateElementMatches)) {
|
if (checkExistSameValueSchemaElementMatch(filter, candidateElementMatches)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -62,7 +60,7 @@ public class QueryFilterMapper implements SchemaMapper {
|
|||||||
.name(String.valueOf(filter.getValue()))
|
.name(String.valueOf(filter.getValue()))
|
||||||
.type(SchemaElementType.VALUE)
|
.type(SchemaElementType.VALUE)
|
||||||
.bizName(filter.getBizName())
|
.bizName(filter.getBizName())
|
||||||
.model(queryContext.getRequest().getModelId())
|
.model(queryContext.getModelId())
|
||||||
.build();
|
.build();
|
||||||
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||||
.element(element)
|
.element(element)
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package com.tencent.supersonic.chat.api.component;
|
package com.tencent.supersonic.chat.core.mapper;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A schema mapper identifies references to schema elements(metrics/dimensions/entities/values)
|
* A schema mapper identifies references to schema elements(metrics/dimensions/entities/values)
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user