mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 11:07:06 +00:00
[improvement][project] global refactor , code format , support llm , support fuzzy detect ,support query filter and so on.
This commit is contained in:
23
assembly/bin/build-standalone.sh
Executable file
23
assembly/bin/build-standalone.sh
Executable file
@@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
sbinDir=$(cd "$(dirname "$0")"; pwd)
|
||||||
|
baseDir=$(readlink -f $sbinDir/../)
|
||||||
|
runtimeDir=$baseDir/runtime
|
||||||
|
buildDir=$baseDir/build
|
||||||
|
|
||||||
|
cd $baseDir
|
||||||
|
|
||||||
|
#1. build semantic chat service
|
||||||
|
rm -fr ${buildDir}/*.tar.gz
|
||||||
|
rm -fr dist
|
||||||
|
|
||||||
|
mvn -f $baseDir/../ clean package -DskipTests
|
||||||
|
|
||||||
|
#2. move package to build
|
||||||
|
cp $baseDir/../launchers/standalone/target/*.tar.gz ${buildDir}/supersonic.tar.gz
|
||||||
|
|
||||||
|
#3. build webapp
|
||||||
|
chmod +x $baseDir/../webapp/start-fe-prod.sh
|
||||||
|
cd ../webapp
|
||||||
|
sh ./start-fe-prod.sh
|
||||||
|
cp -fr ./supersonic-webapp.tar.gz ${buildDir}/
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
sbinDir=$(cd "$(dirname "$0")"; pwd)
|
sbinDir=$(cd "$(dirname "$0")"; pwd)
|
||||||
baseDir=$(readlink -f $sbinDir/../)
|
baseDir=$(readlink -f $sbinDir/../)
|
||||||
runtimeDir=$baseDir/runtime
|
runtimeDir=$baseDir/../runtime
|
||||||
buildDir=$baseDir/build
|
buildDir=$baseDir/build
|
||||||
|
|
||||||
cd $baseDir
|
cd $baseDir
|
||||||
|
|||||||
30
assembly/bin/start-standalone.sh
Executable file
30
assembly/bin/start-standalone.sh
Executable file
@@ -0,0 +1,30 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
sbinDir=$(cd "$(dirname "$0")"; pwd)
|
||||||
|
baseDir=$(readlink -f $sbinDir/../)
|
||||||
|
runtimeDir=$baseDir/../runtime
|
||||||
|
buildDir=$baseDir/build
|
||||||
|
|
||||||
|
cd $baseDir
|
||||||
|
|
||||||
|
#1. clear file
|
||||||
|
mkdir -p ${runtimeDir}
|
||||||
|
rm -fr ${runtimeDir}/*
|
||||||
|
|
||||||
|
#2. package lib
|
||||||
|
|
||||||
|
tar -zxvf ${buildDir}/supersonic.tar.gz -C ${runtimeDir}
|
||||||
|
|
||||||
|
mv ${runtimeDir}/launchers-standalone-* ${runtimeDir}/supersonic-standalone
|
||||||
|
|
||||||
|
tar -zxvf ${buildDir}/supersonic-webapp.tar.gz -C ${buildDir}
|
||||||
|
|
||||||
|
mkdir -p ${runtimeDir}/supersonic-standalone/webapp
|
||||||
|
|
||||||
|
cp -fr ${buildDir}/supersonic-webapp/* ${runtimeDir}/supersonic-standalone/webapp
|
||||||
|
|
||||||
|
rm -fr ${buildDir}/supersonic-webapp
|
||||||
|
|
||||||
|
#3. start service
|
||||||
|
sh ${runtimeDir}/supersonic-standalone/bin/service.sh restart
|
||||||
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>com.tencent.supersonic</groupId>
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.auth.authorization.domain.pojo;
|
package com.tencent.supersonic.auth.api.authorization.pojo;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.auth.authorization.domain.pojo;
|
package com.tencent.supersonic.auth.api.authorization.pojo;
|
||||||
|
|
||||||
import java.beans.Transient;
|
import java.beans.Transient;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package com.tencent.supersonic.auth.api.authorization.request;
|
package com.tencent.supersonic.auth.api.authorization.request;
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
|
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -11,6 +12,10 @@ import lombok.ToString;
|
|||||||
public class QueryAuthResReq {
|
public class QueryAuthResReq {
|
||||||
|
|
||||||
private String user;
|
private String user;
|
||||||
|
|
||||||
|
private List<String> departmentIds = new ArrayList<>();
|
||||||
|
|
||||||
private List<AuthRes> resources;
|
private List<AuthRes> resources;
|
||||||
|
|
||||||
private String domainId;
|
private String domainId;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,18 @@
|
|||||||
package com.tencent.supersonic.auth.api.authorization.service;
|
package com.tencent.supersonic.auth.api.authorization.service;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
|
||||||
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
||||||
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
public interface AuthService {
|
public interface AuthService {
|
||||||
|
|
||||||
AuthorizedResourceResp queryAuthorizedResources(HttpServletRequest request, QueryAuthResReq req);
|
List<AuthGroup> queryAuthGroups(String domainId, Integer groupId);
|
||||||
|
|
||||||
|
void updateAuthGroup(AuthGroup group);
|
||||||
|
|
||||||
|
void removeAuthGroup(AuthGroup group);
|
||||||
|
|
||||||
|
AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, HttpServletRequest request);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<parent>
|
<parent>
|
||||||
<artifactId>auth</artifactId>
|
<artifactId>auth</artifactId>
|
||||||
|
|||||||
@@ -37,13 +37,6 @@ public class UserDOExample {
|
|||||||
oredCriteria = new ArrayList<Criteria>();
|
oredCriteria = new ArrayList<Criteria>();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @mbg.generated
|
|
||||||
*/
|
|
||||||
public void setOrderByClause(String orderByClause) {
|
|
||||||
this.orderByClause = orderByClause;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @mbg.generated
|
* @mbg.generated
|
||||||
*/
|
*/
|
||||||
@@ -54,8 +47,8 @@ public class UserDOExample {
|
|||||||
/**
|
/**
|
||||||
* @mbg.generated
|
* @mbg.generated
|
||||||
*/
|
*/
|
||||||
public void setDistinct(boolean distinct) {
|
public void setOrderByClause(String orderByClause) {
|
||||||
this.distinct = distinct;
|
this.orderByClause = orderByClause;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -65,6 +58,13 @@ public class UserDOExample {
|
|||||||
return distinct;
|
return distinct;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @mbg.generated
|
||||||
|
*/
|
||||||
|
public void setDistinct(boolean distinct) {
|
||||||
|
this.distinct = distinct;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @mbg.generated
|
* @mbg.generated
|
||||||
*/
|
*/
|
||||||
@@ -116,13 +116,6 @@ public class UserDOExample {
|
|||||||
distinct = false;
|
distinct = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @mbg.generated
|
|
||||||
*/
|
|
||||||
public void setLimitStart(Integer limitStart) {
|
|
||||||
this.limitStart = limitStart;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @mbg.generated
|
* @mbg.generated
|
||||||
*/
|
*/
|
||||||
@@ -133,8 +126,8 @@ public class UserDOExample {
|
|||||||
/**
|
/**
|
||||||
* @mbg.generated
|
* @mbg.generated
|
||||||
*/
|
*/
|
||||||
public void setLimitEnd(Integer limitEnd) {
|
public void setLimitStart(Integer limitStart) {
|
||||||
this.limitEnd = limitEnd;
|
this.limitStart = limitStart;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -144,6 +137,13 @@ public class UserDOExample {
|
|||||||
return limitEnd;
|
return limitEnd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @mbg.generated
|
||||||
|
*/
|
||||||
|
public void setLimitEnd(Integer limitEnd) {
|
||||||
|
this.limitEnd = limitEnd;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* s2_user null
|
* s2_user null
|
||||||
*/
|
*/
|
||||||
@@ -561,38 +561,6 @@ public class UserDOExample {
|
|||||||
|
|
||||||
private String typeHandler;
|
private String typeHandler;
|
||||||
|
|
||||||
public String getCondition() {
|
|
||||||
return condition;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object getValue() {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object getSecondValue() {
|
|
||||||
return secondValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isNoValue() {
|
|
||||||
return noValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isSingleValue() {
|
|
||||||
return singleValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isBetweenValue() {
|
|
||||||
return betweenValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isListValue() {
|
|
||||||
return listValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getTypeHandler() {
|
|
||||||
return typeHandler;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected Criterion(String condition) {
|
protected Criterion(String condition) {
|
||||||
super();
|
super();
|
||||||
this.condition = condition;
|
this.condition = condition;
|
||||||
@@ -628,5 +596,37 @@ public class UserDOExample {
|
|||||||
protected Criterion(String condition, Object value, Object secondValue) {
|
protected Criterion(String condition, Object value, Object secondValue) {
|
||||||
this(condition, value, secondValue, null);
|
this(condition, value, secondValue, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getCondition() {
|
||||||
|
return condition;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Object getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Object getSecondValue() {
|
||||||
|
return secondValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isNoValue() {
|
||||||
|
return noValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isSingleValue() {
|
||||||
|
return singleValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isBetweenValue() {
|
||||||
|
return betweenValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isListValue() {
|
||||||
|
return listValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTypeHandler() {
|
||||||
|
return typeHandler;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -14,14 +14,11 @@ import javax.servlet.http.HttpServletRequest;
|
|||||||
import javax.servlet.http.HttpServletResponse;
|
import javax.servlet.http.HttpServletResponse;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import org.springframework.web.method.HandlerMethod;
|
import org.springframework.web.method.HandlerMethod;
|
||||||
|
|
||||||
@Component
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor {
|
public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor {
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler)
|
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler)
|
||||||
throws AccessException {
|
throws AccessException {
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ public class InterceptorFactory implements WebMvcConfigurer {
|
|||||||
public void addInterceptors(InterceptorRegistry registry) {
|
public void addInterceptors(InterceptorRegistry registry) {
|
||||||
for (AuthenticationInterceptor authenticationInterceptor : authenticationInterceptors) {
|
for (AuthenticationInterceptor authenticationInterceptor : authenticationInterceptors) {
|
||||||
registry.addInterceptor(authenticationInterceptor).addPathPatterns("/**")
|
registry.addInterceptor(authenticationInterceptor).addPathPatterns("/**")
|
||||||
.excludePathPatterns("/", "/webapp/**","/error");
|
.excludePathPatterns("/", "/webapp/**", "/error");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<parent>
|
<parent>
|
||||||
<artifactId>auth</artifactId>
|
<artifactId>auth</artifactId>
|
||||||
|
|||||||
@@ -1,118 +0,0 @@
|
|||||||
package com.tencent.supersonic.auth.authorization.application;
|
|
||||||
|
|
||||||
import com.google.common.base.Strings;
|
|
||||||
import com.google.gson.Gson;
|
|
||||||
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
|
|
||||||
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
|
|
||||||
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
|
|
||||||
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
|
||||||
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
|
||||||
import com.tencent.supersonic.auth.authorization.domain.pojo.AuthGroup;
|
|
||||||
import com.tencent.supersonic.auth.authorization.domain.pojo.AuthRule;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.jdbc.core.JdbcTemplate;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
@Slf4j
|
|
||||||
public class AuthApplicationService {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private JdbcTemplate jdbcTemplate;
|
|
||||||
|
|
||||||
private List<AuthGroup> load() {
|
|
||||||
List<String> rows = jdbcTemplate.queryForList("select config from s2_auth_groups", String.class);
|
|
||||||
Gson g = new Gson();
|
|
||||||
return rows.stream().map(row -> g.fromJson(row, AuthGroup.class)).collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<AuthGroup> queryAuthGroups(String domainId, Integer groupId) {
|
|
||||||
return load().stream()
|
|
||||||
.filter(group -> (Objects.isNull(groupId) || groupId.equals(group.getGroupId()))
|
|
||||||
&& domainId.equals(group.getDomainId()))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
public void updateAuthGroup(AuthGroup group) {
|
|
||||||
Gson g = new Gson();
|
|
||||||
if (group.getGroupId() == null) {
|
|
||||||
int nextGroupId = 1;
|
|
||||||
String sql = "select max(group_id) as group_id from s2_auth_groups";
|
|
||||||
Integer obj = jdbcTemplate.queryForObject(sql, Integer.class);
|
|
||||||
if (obj != null) {
|
|
||||||
nextGroupId = obj + 1;
|
|
||||||
}
|
|
||||||
group.setGroupId(nextGroupId);
|
|
||||||
jdbcTemplate.update("insert into s2_auth_groups (group_id, config) values (?, ?);", nextGroupId,
|
|
||||||
g.toJson(group));
|
|
||||||
} else {
|
|
||||||
jdbcTemplate.update("update s2_auth_groups set config = ? where group_id = ?;", g.toJson(group),
|
|
||||||
group.getGroupId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, HttpServletRequest request) {
|
|
||||||
List<AuthGroup> groups = load().stream().
|
|
||||||
filter(group -> group.getAuthorizedUsers().contains(req.getUser()) && req.getDomainId()
|
|
||||||
.equals(group.getDomainId())).
|
|
||||||
collect(Collectors.toList());
|
|
||||||
AuthorizedResourceResp resource = new AuthorizedResourceResp();
|
|
||||||
Map<String, List<AuthGroup>> authGroupsByDomainId = groups.stream()
|
|
||||||
.collect(Collectors.groupingBy(AuthGroup::getDomainId));
|
|
||||||
Map<String, List<AuthRes>> reqAuthRes = req.getResources().stream()
|
|
||||||
.collect(Collectors.groupingBy(AuthRes::getDomainId));
|
|
||||||
|
|
||||||
for (String domainId : reqAuthRes.keySet()) {
|
|
||||||
List<AuthRes> reqResourcesList = reqAuthRes.get(domainId);
|
|
||||||
AuthResGrp rg = new AuthResGrp();
|
|
||||||
if (authGroupsByDomainId.containsKey(domainId)) {
|
|
||||||
List<AuthGroup> authGroups = authGroupsByDomainId.get(domainId);
|
|
||||||
for (AuthRes reqRes : reqResourcesList) {
|
|
||||||
for (AuthGroup authRuleGroup : authGroups) {
|
|
||||||
List<AuthRule> authRules = authRuleGroup.getAuthRules();
|
|
||||||
List<String> allAuthItems = new ArrayList<>();
|
|
||||||
authRules.stream().forEach(authRule -> allAuthItems.addAll(authRule.resourceNames()));
|
|
||||||
|
|
||||||
if (allAuthItems.contains(reqRes.getName())) {
|
|
||||||
rg.getGroup().add(reqRes);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Objects.nonNull(rg) && !CollectionUtils.isEmpty(rg.getGroup())) {
|
|
||||||
resource.getResources().add(rg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (StringUtils.isNotEmpty(req.getDomainId())) {
|
|
||||||
List<AuthGroup> authGroups = authGroupsByDomainId.get(req.getDomainId());
|
|
||||||
if (!CollectionUtils.isEmpty(authGroups)) {
|
|
||||||
for (AuthGroup group : authGroups) {
|
|
||||||
if (group.getDimensionFilters() != null
|
|
||||||
&& group.getDimensionFilters().stream().anyMatch(expr -> !Strings.isNullOrEmpty(expr))) {
|
|
||||||
DimensionFilter df = new DimensionFilter();
|
|
||||||
df.setDescription(group.getDimensionFilterDescription());
|
|
||||||
df.setExpressions(group.getDimensionFilters());
|
|
||||||
resource.getFilters().add(df);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return resource;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void removeAuthGroup(AuthGroup group) {
|
|
||||||
jdbcTemplate.update("delete from s2_auth_groups where group_id = ?", group.getGroupId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,24 +1,148 @@
|
|||||||
package com.tencent.supersonic.auth.authorization.application;
|
package com.tencent.supersonic.auth.authorization.application;
|
||||||
|
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
|
||||||
|
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
|
||||||
|
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
|
||||||
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
||||||
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
||||||
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
|
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
|
||||||
|
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class AuthServiceImpl implements AuthService {
|
public class AuthServiceImpl implements AuthService {
|
||||||
|
|
||||||
private final AuthApplicationService authApplicationService;
|
private JdbcTemplate jdbcTemplate;
|
||||||
|
|
||||||
public AuthServiceImpl(AuthApplicationService authApplicationService) {
|
public AuthServiceImpl(JdbcTemplate jdbcTemplate) {
|
||||||
this.authApplicationService = authApplicationService;
|
this.jdbcTemplate = jdbcTemplate;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<AuthGroup> load() {
|
||||||
|
List<String> rows = jdbcTemplate.queryForList("select config from s2_auth_groups", String.class);
|
||||||
|
Gson g = new Gson();
|
||||||
|
return rows.stream().map(row -> g.fromJson(row, AuthGroup.class)).collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public AuthorizedResourceResp queryAuthorizedResources(HttpServletRequest request, QueryAuthResReq req) {
|
public List<AuthGroup> queryAuthGroups(String domainId, Integer groupId) {
|
||||||
return authApplicationService.queryAuthorizedResources(req, request);
|
return load().stream()
|
||||||
|
.filter(group -> (Objects.isNull(groupId) || groupId.equals(group.getGroupId()))
|
||||||
|
&& domainId.equals(group.getDomainId()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateAuthGroup(AuthGroup group) {
|
||||||
|
Gson g = new Gson();
|
||||||
|
if (group.getGroupId() == null) {
|
||||||
|
int nextGroupId = 1;
|
||||||
|
String sql = "select max(group_id) as group_id from s2_auth_groups";
|
||||||
|
Integer obj = jdbcTemplate.queryForObject(sql, Integer.class);
|
||||||
|
if (obj != null) {
|
||||||
|
nextGroupId = obj + 1;
|
||||||
|
}
|
||||||
|
group.setGroupId(nextGroupId);
|
||||||
|
jdbcTemplate.update("insert into s2_auth_groups (group_id, config) values (?, ?);", nextGroupId,
|
||||||
|
g.toJson(group));
|
||||||
|
} else {
|
||||||
|
jdbcTemplate.update("update s2_auth_groups set config = ? where group_id = ?;", g.toJson(group),
|
||||||
|
group.getGroupId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void removeAuthGroup(AuthGroup group) {
|
||||||
|
jdbcTemplate.update("delete from s2_auth_groups where group_id = ?", group.getGroupId());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, HttpServletRequest request) {
|
||||||
|
List<AuthGroup> groups = getAuthGroups(req);
|
||||||
|
AuthorizedResourceResp resource = new AuthorizedResourceResp();
|
||||||
|
Map<String, List<AuthGroup>> authGroupsByDomainId = groups.stream()
|
||||||
|
.collect(Collectors.groupingBy(AuthGroup::getDomainId));
|
||||||
|
Map<String, List<AuthRes>> reqAuthRes = req.getResources().stream()
|
||||||
|
.collect(Collectors.groupingBy(AuthRes::getDomainId));
|
||||||
|
|
||||||
|
for (String domainId : reqAuthRes.keySet()) {
|
||||||
|
List<AuthRes> reqResourcesList = reqAuthRes.get(domainId);
|
||||||
|
AuthResGrp rg = new AuthResGrp();
|
||||||
|
if (authGroupsByDomainId.containsKey(domainId)) {
|
||||||
|
List<AuthGroup> authGroups = authGroupsByDomainId.get(domainId);
|
||||||
|
for (AuthRes reqRes : reqResourcesList) {
|
||||||
|
for (AuthGroup authRuleGroup : authGroups) {
|
||||||
|
List<AuthRule> authRules = authRuleGroup.getAuthRules();
|
||||||
|
List<String> allAuthItems = new ArrayList<>();
|
||||||
|
authRules.forEach(authRule -> allAuthItems.addAll(authRule.resourceNames()));
|
||||||
|
|
||||||
|
if (allAuthItems.contains(reqRes.getName())) {
|
||||||
|
rg.getGroup().add(reqRes);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!CollectionUtils.isEmpty(rg.getGroup())) {
|
||||||
|
resource.getResources().add(rg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.isNotEmpty(req.getDomainId())) {
|
||||||
|
List<AuthGroup> authGroups = authGroupsByDomainId.get(req.getDomainId());
|
||||||
|
if (!CollectionUtils.isEmpty(authGroups)) {
|
||||||
|
for (AuthGroup group : authGroups) {
|
||||||
|
if (group.getDimensionFilters() != null
|
||||||
|
&& group.getDimensionFilters().stream().anyMatch(expr -> !Strings.isNullOrEmpty(expr))) {
|
||||||
|
DimensionFilter df = new DimensionFilter();
|
||||||
|
df.setDescription(group.getDimensionFilterDescription());
|
||||||
|
df.setExpressions(group.getDimensionFilters());
|
||||||
|
resource.getFilters().add(df);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return resource;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<AuthGroup> getAuthGroups(QueryAuthResReq req) {
|
||||||
|
List<AuthGroup> groups = load().stream().
|
||||||
|
filter(group -> {
|
||||||
|
if (!Objects.equals(group.getDomainId(), req.getDomainId())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers()
|
||||||
|
.contains(req.getUser())) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
for (String deparmentId : req.getDepartmentIds()) {
|
||||||
|
if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds())
|
||||||
|
&& group.getAuthorizedDepartmentIds().contains(deparmentId)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}).collect(Collectors.toList());
|
||||||
|
log.info("user:{} department:{} authGroups:{}", req.getUser(), req.getDepartmentIds(), groups);
|
||||||
|
return groups;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ package com.tencent.supersonic.auth.authorization.rest;
|
|||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
|
||||||
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
|
||||||
import com.tencent.supersonic.auth.authorization.application.AuthApplicationService;
|
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
|
||||||
import com.tencent.supersonic.auth.authorization.domain.pojo.AuthGroup;
|
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -19,16 +19,16 @@ import org.springframework.web.bind.annotation.RestController;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
public class AuthController {
|
public class AuthController {
|
||||||
|
|
||||||
private final AuthApplicationService service;
|
private final AuthService authService;
|
||||||
|
|
||||||
public AuthController(AuthApplicationService service) {
|
public AuthController(AuthService authService) {
|
||||||
this.service = service;
|
this.authService = authService;
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/queryGroup")
|
@GetMapping("/queryGroup")
|
||||||
public List<AuthGroup> queryAuthGroup(@RequestParam("domainId") String domainId,
|
public List<AuthGroup> queryAuthGroup(@RequestParam("domainId") String domainId,
|
||||||
@RequestParam(value = "groupId", required = false) Integer groupId) {
|
@RequestParam(value = "groupId", required = false) Integer groupId) {
|
||||||
return service.queryAuthGroups(domainId, groupId);
|
return authService.queryAuthGroups(domainId, groupId);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -37,12 +37,12 @@ public class AuthController {
|
|||||||
@PostMapping("/createGroup")
|
@PostMapping("/createGroup")
|
||||||
public void newAuthGroup(@RequestBody AuthGroup group) {
|
public void newAuthGroup(@RequestBody AuthGroup group) {
|
||||||
group.setGroupId(null);
|
group.setGroupId(null);
|
||||||
service.updateAuthGroup(group);
|
authService.updateAuthGroup(group);
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/removeGroup")
|
@PostMapping("/removeGroup")
|
||||||
public void removeAuthGroup(@RequestBody AuthGroup group) {
|
public void removeAuthGroup(@RequestBody AuthGroup group) {
|
||||||
service.removeAuthGroup(group);
|
authService.removeAuthGroup(group);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -55,7 +55,7 @@ public class AuthController {
|
|||||||
if (group.getGroupId() == null || group.getGroupId() == 0) {
|
if (group.getGroupId() == null || group.getGroupId() == 0) {
|
||||||
throw new RuntimeException("groupId is empty");
|
throw new RuntimeException("groupId is empty");
|
||||||
}
|
}
|
||||||
service.updateAuthGroup(group);
|
authService.updateAuthGroup(group);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -68,6 +68,6 @@ public class AuthController {
|
|||||||
@PostMapping("/queryAuthorizedRes")
|
@PostMapping("/queryAuthorizedRes")
|
||||||
public AuthorizedResourceResp queryAuthorizedResources(@RequestBody QueryAuthResReq req,
|
public AuthorizedResourceResp queryAuthorizedResources(@RequestBody QueryAuthResReq req,
|
||||||
HttpServletRequest request) {
|
HttpServletRequest request) {
|
||||||
return service.queryAuthorizedResources(req, request);
|
return authService.queryAuthorizedResources(req, request);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<parent>
|
<parent>
|
||||||
<artifactId>supersonic</artifactId>
|
<artifactId>supersonic</artifactId>
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.api.service;
|
package com.tencent.supersonic.chat.api.component;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
|
|
||||||
@@ -1,8 +1,9 @@
|
|||||||
package com.tencent.supersonic.chat.api.service;
|
package com.tencent.supersonic.chat.api.component;
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
|
||||||
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -23,6 +24,8 @@ public interface SemanticLayer {
|
|||||||
|
|
||||||
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user);
|
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user);
|
||||||
|
|
||||||
|
QueryResultWithSchemaResp queryBySql(QuerySqlReq querySqlReq, User user);
|
||||||
|
|
||||||
DomainSchemaResp getDomainSchemaInfo(Long domain);
|
DomainSchemaResp getDomainSchemaInfo(Long domain);
|
||||||
|
|
||||||
List<DomainSchemaResp> getDomainSchemaInfo(List<Long> ids);
|
List<DomainSchemaResp> getDomainSchemaInfo(List<Long> ids);
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.api.service;
|
package com.tencent.supersonic.chat.api.component;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
@@ -13,5 +13,5 @@ import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|||||||
*/
|
*/
|
||||||
public interface SemanticParser {
|
public interface SemanticParser {
|
||||||
|
|
||||||
boolean parse(QueryContextReq queryContext, ChatContext chatCtx);
|
void parse(QueryContextReq queryContext, ChatContext chatContext);
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package com.tencent.supersonic.chat.api.component;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class defines the contract for a semantic query that executes specific type of
|
||||||
|
* query based on the results of semantic parsing.
|
||||||
|
*/
|
||||||
|
public interface SemanticQuery {
|
||||||
|
|
||||||
|
String getQueryMode();
|
||||||
|
|
||||||
|
QueryResultResp execute(User user);
|
||||||
|
|
||||||
|
SemanticParseInfo getParseInfo();
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package com.tencent.supersonic.chat.api.pojo;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class QueryFilter {
|
||||||
|
|
||||||
|
private List<Filter> filters = new ArrayList<>();
|
||||||
|
|
||||||
|
private Map<String, Object> params = new HashMap<>();
|
||||||
|
|
||||||
|
}
|
||||||
@@ -3,8 +3,10 @@ package com.tencent.supersonic.chat.api.pojo;
|
|||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public class SchemaElementCount {
|
public class QueryMatchInfo {
|
||||||
|
|
||||||
|
SchemaElementType elementType;
|
||||||
|
String detectWord;
|
||||||
private Integer count = 0;
|
private Integer count = 0;
|
||||||
private double maxSimilarity;
|
private double maxSimilarity;
|
||||||
}
|
}
|
||||||
@@ -1,10 +1,16 @@
|
|||||||
package com.tencent.supersonic.chat.api.pojo;
|
package com.tencent.supersonic.chat.api.pojo;
|
||||||
|
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
import lombok.ToString;
|
import lombok.ToString;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@ToString
|
@ToString
|
||||||
|
@Builder
|
||||||
|
@AllArgsConstructor
|
||||||
|
@NoArgsConstructor
|
||||||
public class SchemaElementMatch {
|
public class SchemaElementMatch {
|
||||||
|
|
||||||
SchemaElementType elementType;
|
SchemaElementType elementType;
|
||||||
@@ -18,7 +24,4 @@ public class SchemaElementMatch {
|
|||||||
String word;
|
String word;
|
||||||
|
|
||||||
Long frequency;
|
Long frequency;
|
||||||
|
|
||||||
public SchemaElementMatch() {
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,12 +21,12 @@ public class SchemaMapInfo {
|
|||||||
return domainElementMatches;
|
return domainElementMatches;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setMatchedElements(Integer domain, List<SchemaElementMatch> elementMatches) {
|
|
||||||
domainElementMatches.put(domain, elementMatches);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDomainElementMatches(
|
public void setDomainElementMatches(
|
||||||
Map<Integer, List<SchemaElementMatch>> domainElementMatches) {
|
Map<Integer, List<SchemaElementMatch>> domainElementMatches) {
|
||||||
this.domainElementMatches = domainElementMatches;
|
this.domainElementMatches = domainElementMatches;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setMatchedElements(Integer domain, List<SchemaElementMatch> elementMatches) {
|
||||||
|
domainElementMatches.put(domain, elementMatches);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
|||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
import com.tencent.supersonic.common.pojo.Order;
|
import com.tencent.supersonic.common.pojo.Order;
|
||||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -27,4 +28,7 @@ public class SemanticParseInfo {
|
|||||||
private DateConf dateInfo;
|
private DateConf dateInfo;
|
||||||
private Long limit;
|
private Long limit;
|
||||||
private Boolean nativeQuery = false;
|
private Boolean nativeQuery = false;
|
||||||
|
private Double bonus = 0d;
|
||||||
|
private List<SchemaElementMatch> elementMatches = new ArrayList<>();
|
||||||
|
private Object info;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
package com.tencent.supersonic.chat.api.request;
|
package com.tencent.supersonic.chat.api.request;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.QueryFilter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public class QueryContextReq {
|
public class QueryContextReq {
|
||||||
|
|
||||||
@@ -13,7 +16,8 @@ public class QueryContextReq {
|
|||||||
private Integer chatId;
|
private Integer chatId;
|
||||||
private Integer domainId = 0;
|
private Integer domainId = 0;
|
||||||
private User user;
|
private User user;
|
||||||
private SemanticParseInfo parseInfo = new SemanticParseInfo();
|
private QueryFilter queryFilter;
|
||||||
|
private List<SemanticQuery> candidateQueries = new ArrayList<>();
|
||||||
private SchemaMapInfo mapInfo = new SchemaMapInfo();
|
private SchemaMapInfo mapInfo = new SchemaMapInfo();
|
||||||
private boolean saveAnswer = true;
|
private boolean saveAnswer = true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,13 +11,13 @@ import lombok.Data;
|
|||||||
@Data
|
@Data
|
||||||
public class QueryResultResp {
|
public class QueryResultResp {
|
||||||
|
|
||||||
|
public EntityInfo entityInfo;
|
||||||
private Long queryId;
|
private Long queryId;
|
||||||
private String queryMode;
|
private String queryMode;
|
||||||
private String querySql;
|
private String querySql;
|
||||||
private int queryState;
|
private int queryState;
|
||||||
private List<QueryColumn> queryColumns;
|
private List<QueryColumn> queryColumns;
|
||||||
private QueryAuthorization queryAuthorization;
|
private QueryAuthorization queryAuthorization;
|
||||||
public EntityInfo entityInfo;
|
|
||||||
private SemanticParseInfo chatContext;
|
private SemanticParseInfo chatContext;
|
||||||
private Object response;
|
private Object response;
|
||||||
private List<Map<String, Object>> queryResults;
|
private List<Map<String, Object>> queryResults;
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.api.service;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementCount;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This interface defines the contract for a semantic query that executes specific type of
|
|
||||||
* query based on the results of semantic parsing.
|
|
||||||
*/
|
|
||||||
public interface SemanticQuery extends Serializable {
|
|
||||||
|
|
||||||
String getQueryMode();
|
|
||||||
|
|
||||||
QueryResultResp execute(QueryContextReq queryCtx, ChatContext chatCtx) throws Exception;
|
|
||||||
|
|
||||||
SchemaElementCount match(List<SchemaElementMatch> elementMatches, QueryContextReq queryCtx);
|
|
||||||
|
|
||||||
void updateContext(QueryResultResp queryResponse, ChatContext chatCtx, QueryContextReq queryCtx);
|
|
||||||
|
|
||||||
SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx);
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -113,6 +113,12 @@
|
|||||||
<artifactId>semantic-api</artifactId>
|
<artifactId>semantic-api</artifactId>
|
||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<!-- <dependency>-->
|
||||||
|
<!-- <groupId>com.tencent.supersonic</groupId>-->
|
||||||
|
<!-- <artifactId>semantic-query</artifactId>-->
|
||||||
|
<!-- <version>${project.version}</version>-->
|
||||||
|
<!-- <scope>compile</scope>-->
|
||||||
|
<!-- </dependency>-->
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.tencent.supersonic</groupId>
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
<artifactId>semantic-query</artifactId>
|
<artifactId>semantic-query</artifactId>
|
||||||
@@ -125,6 +131,12 @@
|
|||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
<scope>compile</scope>
|
<scope>compile</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.tencent.supersonic</groupId>
|
||||||
|
<artifactId>semantic-query</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
@@ -18,20 +18,19 @@ import com.tencent.supersonic.chat.domain.service.ChatService;
|
|||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.context.annotation.Primary;
|
import org.springframework.context.annotation.Primary;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@Service("ChatService")
|
@Service("ChatService")
|
||||||
@Primary
|
@Primary
|
||||||
|
@Slf4j
|
||||||
public class ChatServiceImpl implements ChatService {
|
public class ChatServiceImpl implements ChatService {
|
||||||
|
|
||||||
private ChatContextRepository chatContextRepository;
|
private ChatContextRepository chatContextRepository;
|
||||||
private ChatRepository chatRepository;
|
private ChatRepository chatRepository;
|
||||||
private ChatQueryRepository chatQueryRepository;
|
private ChatQueryRepository chatQueryRepository;
|
||||||
private final Logger logger = LoggerFactory.getLogger(ChatService.class);
|
|
||||||
|
|
||||||
|
|
||||||
public ChatServiceImpl(ChatContextRepository chatContextRepository, ChatRepository chatRepository,
|
public ChatServiceImpl(ChatContextRepository chatContextRepository, ChatRepository chatRepository,
|
||||||
ChatQueryRepository chatQueryRepository) {
|
ChatQueryRepository chatQueryRepository) {
|
||||||
@@ -64,27 +63,32 @@ public class ChatServiceImpl implements ChatService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void updateContext(ChatContext chatCtx) {
|
public void updateContext(ChatContext chatCtx) {
|
||||||
logger.debug("save ChatContext {}", chatCtx);
|
log.debug("save ChatContext {}", chatCtx);
|
||||||
chatContextRepository.updateContext(chatCtx);
|
chatContextRepository.updateContext(chatCtx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateContext(ChatContext chatCtx, QueryContextReq queryCtx, SemanticParseInfo semanticParseInfo) {
|
||||||
|
chatCtx.setParseInfo(semanticParseInfo);
|
||||||
|
chatCtx.setQueryText(queryCtx.getQueryText());
|
||||||
|
updateContext(chatCtx);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void switchContext(ChatContext chatCtx) {
|
public void switchContext(ChatContext chatCtx) {
|
||||||
logger.debug("switchContext ChatContext {}", chatCtx);
|
log.debug("switchContext ChatContext {}", chatCtx);
|
||||||
chatCtx.setParseInfo(new SemanticParseInfo());
|
chatCtx.setParseInfo(new SemanticParseInfo());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Boolean addChat(User user, String chatName) {
|
public Boolean addChat(User user, String chatName) {
|
||||||
SimpleDateFormat tempDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
|
||||||
String datetime = tempDate.format(new java.util.Date());
|
|
||||||
ChatDO intelligentConversionDO = new ChatDO();
|
ChatDO intelligentConversionDO = new ChatDO();
|
||||||
intelligentConversionDO.setChatName(chatName);
|
intelligentConversionDO.setChatName(chatName);
|
||||||
intelligentConversionDO.setCreator(user.getName());
|
intelligentConversionDO.setCreator(user.getName());
|
||||||
intelligentConversionDO.setCreateTime(datetime);
|
intelligentConversionDO.setCreateTime(getCurrentTime());
|
||||||
intelligentConversionDO.setIsDelete(0);
|
intelligentConversionDO.setIsDelete(0);
|
||||||
intelligentConversionDO.setLastTime(datetime);
|
intelligentConversionDO.setLastTime(getCurrentTime());
|
||||||
intelligentConversionDO.setLastQuestion("Hello, welcome to using supersonic");
|
intelligentConversionDO.setLastQuestion("Hello, welcome to using supersonic");
|
||||||
intelligentConversionDO.setIsTop(0);
|
intelligentConversionDO.setIsTop(0);
|
||||||
return chatRepository.createChat(intelligentConversionDO);
|
return chatRepository.createChat(intelligentConversionDO);
|
||||||
@@ -97,9 +101,7 @@ public class ChatServiceImpl implements ChatService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean updateChatName(Long chatId, String chatName, String userName) {
|
public boolean updateChatName(Long chatId, String chatName, String userName) {
|
||||||
SimpleDateFormat tempDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
return chatRepository.updateChatName(chatId, chatName, getCurrentTime(), userName);
|
||||||
String lastTime = tempDate.format(new java.util.Date());
|
|
||||||
return chatRepository.updateChatName(chatId, chatName, lastTime, userName);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -129,6 +131,8 @@ public class ChatServiceImpl implements ChatService {
|
|||||||
@Override
|
@Override
|
||||||
public void addQuery(QueryResultResp queryResponse, QueryContextReq queryContext, ChatContext chatCtx) {
|
public void addQuery(QueryResultResp queryResponse, QueryContextReq queryContext, ChatContext chatCtx) {
|
||||||
chatQueryRepository.createChatQuery(queryResponse, queryContext, chatCtx);
|
chatQueryRepository.createChatQuery(queryResponse, queryContext, chatCtx);
|
||||||
|
chatRepository.updateLastQuestion(chatCtx.getChatId().longValue(), queryContext.getQueryText(),
|
||||||
|
getCurrentTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -141,4 +145,9 @@ public class ChatServiceImpl implements ChatService {
|
|||||||
return chatQueryRepository.updateChatQuery(chatQueryDO);
|
return chatQueryRepository.updateChatQuery(chatQueryDO);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String getCurrentTime() {
|
||||||
|
SimpleDateFormat tempDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||||
|
return tempDate.format(new java.util.Date());
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,9 +2,7 @@ package com.tencent.supersonic.chat.application;
|
|||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfig;
|
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfig;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigBase;
|
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigBase;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigEditReq;
|
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigEditReq;
|
||||||
@@ -14,14 +12,22 @@ import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
|||||||
import com.tencent.supersonic.chat.domain.pojo.config.DefaultMetric;
|
import com.tencent.supersonic.chat.domain.pojo.config.DefaultMetric;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ItemVisibilityInfo;
|
import com.tencent.supersonic.chat.domain.pojo.config.ItemVisibilityInfo;
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.config.KnowledgeInfo;
|
||||||
import com.tencent.supersonic.chat.domain.repository.ChatConfigRepository;
|
import com.tencent.supersonic.chat.domain.repository.ChatConfigRepository;
|
||||||
import com.tencent.supersonic.chat.domain.service.ConfigService;
|
import com.tencent.supersonic.chat.domain.service.ConfigService;
|
||||||
import com.tencent.supersonic.chat.domain.utils.ChatConfigUtils;
|
import com.tencent.supersonic.chat.domain.utils.ChatConfigUtils;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
|
||||||
import com.tencent.supersonic.common.util.json.JsonUtil;
|
import com.tencent.supersonic.common.util.json.JsonUtil;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.DomainResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
@@ -35,15 +41,16 @@ import org.springframework.util.CollectionUtils;
|
|||||||
public class ConfigServiceImpl implements ConfigService {
|
public class ConfigServiceImpl implements ConfigService {
|
||||||
|
|
||||||
private final ChatConfigRepository chaConfigRepository;
|
private final ChatConfigRepository chaConfigRepository;
|
||||||
private final SemanticLayer semanticLayer;
|
|
||||||
private final ChatConfigUtils chatConfigUtils;
|
private final ChatConfigUtils chatConfigUtils;
|
||||||
|
private final DefaultSemanticInternalUtils defaultSemanticUtils;
|
||||||
|
|
||||||
|
|
||||||
public ConfigServiceImpl(ChatConfigRepository chaConfigRepository,
|
public ConfigServiceImpl(ChatConfigRepository chaConfigRepository,
|
||||||
@Lazy SemanticLayer semanticLayer,
|
ChatConfigUtils chatConfigUtils,
|
||||||
ChatConfigUtils chatConfigUtils) {
|
@Lazy DefaultSemanticInternalUtils defaultSemanticUtils) {
|
||||||
this.chaConfigRepository = chaConfigRepository;
|
this.chaConfigRepository = chaConfigRepository;
|
||||||
this.semanticLayer = semanticLayer;
|
|
||||||
this.chatConfigUtils = chatConfigUtils;
|
this.chatConfigUtils = chatConfigUtils;
|
||||||
|
this.defaultSemanticUtils = defaultSemanticUtils;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -101,19 +108,19 @@ public class ConfigServiceImpl implements ConfigService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public EntityRichInfo fetchEntityDescByDomainId(Long domainId) {
|
public EntityRichInfo fetchEntityDescByDomainId(Long domainId) {
|
||||||
|
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
ChatConfigInfo chaConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
|
ChatConfigInfo chaConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
|
||||||
return fetchEntityDescByConfig(chaConfigDesc);
|
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domainId);
|
||||||
|
return fetchEntityDescByConfig(chaConfigDesc, domainSchemaDesc);
|
||||||
}
|
}
|
||||||
|
|
||||||
public EntityRichInfo fetchEntityDescByConfig(ChatConfigInfo chatConfigDesc) {
|
public EntityRichInfo fetchEntityDescByConfig(ChatConfigInfo chatConfigDesc, DomainSchemaResp domain) {
|
||||||
Long domainId = chatConfigDesc.getDomainId();
|
Long domainId = chatConfigDesc.getDomainId();
|
||||||
EntityRichInfo entityDesc = new EntityRichInfo();
|
EntityRichInfo entityDesc = new EntityRichInfo();
|
||||||
if (Objects.isNull(chatConfigDesc) || Objects.isNull(chatConfigDesc.getEntity())) {
|
if (Objects.isNull(chatConfigDesc) || Objects.isNull(chatConfigDesc.getEntity())) {
|
||||||
log.info("domainId:{}, entityDesc info is null", domainId);
|
log.info("domainId:{}, entityDesc info is null", domainId);
|
||||||
return entityDesc;
|
return entityDesc;
|
||||||
}
|
}
|
||||||
DomainSchemaResp domain = semanticLayer.getDomainSchemaInfo(domainId);
|
|
||||||
|
|
||||||
entityDesc.setDomainId(domain.getId());
|
entityDesc.setDomainId(domain.getId());
|
||||||
entityDesc.setDomainBizName(domain.getBizName());
|
entityDesc.setDomainBizName(domain.getBizName());
|
||||||
@@ -128,13 +135,14 @@ public class ConfigServiceImpl implements ConfigService {
|
|||||||
|
|
||||||
|
|
||||||
public List<DefaultMetric> fetchDefaultMetricDescByDomainId(Long domainId) {
|
public List<DefaultMetric> fetchDefaultMetricDescByDomainId(Long domainId) {
|
||||||
|
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
|
ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
|
||||||
return fetchDefaultMetricDescByConfig(chatConfigDesc);
|
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domainId);
|
||||||
|
return fetchDefaultMetricDescByConfig(chatConfigDesc, domainSchemaDesc);
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<DefaultMetric> fetchDefaultMetricDescByConfig(ChatConfigInfo chatConfigDesc) {
|
public List<DefaultMetric> fetchDefaultMetricDescByConfig(ChatConfigInfo chatConfigDesc, DomainSchemaResp domain) {
|
||||||
Long domainId = chatConfigDesc.getDomainId();
|
Long domainId = chatConfigDesc.getDomainId();
|
||||||
DomainSchemaResp domain = semanticLayer.getDomainSchemaInfo(domainId);
|
|
||||||
List<DefaultMetric> defaultMetricDescList = new ArrayList<>();
|
List<DefaultMetric> defaultMetricDescList = new ArrayList<>();
|
||||||
if (Objects.isNull(chatConfigDesc) || CollectionUtils.isEmpty(chatConfigDesc.getDefaultMetrics())) {
|
if (Objects.isNull(chatConfigDesc) || CollectionUtils.isEmpty(chatConfigDesc.getDefaultMetrics())) {
|
||||||
log.info("domainId:{}, defaultMetricDescList info is null", domainId);
|
log.info("domainId:{}, defaultMetricDescList info is null", domainId);
|
||||||
@@ -158,14 +166,17 @@ public class ConfigServiceImpl implements ConfigService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public ItemVisibilityInfo fetchVisibilityDescByDomainId(Long domainId) {
|
public ItemVisibilityInfo fetchVisibilityDescByDomainId(Long domainId) {
|
||||||
|
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
|
ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
|
||||||
return fetchVisibilityDescByConfig(chatConfigDesc);
|
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domainId);
|
||||||
|
return fetchVisibilityDescByConfig(chatConfigDesc, domainSchemaDesc);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ItemVisibilityInfo fetchVisibilityDescByConfig(ChatConfigInfo chatConfigDesc) {
|
private ItemVisibilityInfo fetchVisibilityDescByConfig(ChatConfigInfo chatConfigDesc,
|
||||||
|
DomainSchemaResp domainSchemaDesc) {
|
||||||
ItemVisibilityInfo itemVisibilityDesc = new ItemVisibilityInfo();
|
ItemVisibilityInfo itemVisibilityDesc = new ItemVisibilityInfo();
|
||||||
Long domainId = chatConfigDesc.getDomainId();
|
Long domainId = chatConfigDesc.getDomainId();
|
||||||
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domainId);
|
|
||||||
List<Long> dimIdAllList = chatConfigUtils.generateAllDimIdList(domainSchemaDesc);
|
List<Long> dimIdAllList = chatConfigUtils.generateAllDimIdList(domainSchemaDesc);
|
||||||
List<Long> metricIdAllList = chatConfigUtils.generateAllMetricIdList(domainSchemaDesc);
|
List<Long> metricIdAllList = chatConfigUtils.generateAllMetricIdList(domainSchemaDesc);
|
||||||
|
|
||||||
@@ -194,18 +205,65 @@ public class ConfigServiceImpl implements ConfigService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ChatConfigRichInfo getConfigRichInfo(Long domainId) {
|
public ChatConfigRichInfo getConfigRichInfo(Long domainId) {
|
||||||
|
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
ChatConfigRichInfo chaConfigRichDesc = new ChatConfigRichInfo();
|
ChatConfigRichInfo chaConfigRichDesc = new ChatConfigRichInfo();
|
||||||
ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
|
ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
|
||||||
|
if (Objects.isNull(chatConfigDesc)) {
|
||||||
|
log.info("there is no chatConfigDesc for domainId:{}", domainId);
|
||||||
|
return chaConfigRichDesc;
|
||||||
|
}
|
||||||
BeanUtils.copyProperties(chatConfigDesc, chaConfigRichDesc);
|
BeanUtils.copyProperties(chatConfigDesc, chaConfigRichDesc);
|
||||||
|
|
||||||
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domainId);
|
DomainSchemaResp domainSchemaInfo = semanticLayer.getDomainSchemaInfo(domainId);
|
||||||
chaConfigRichDesc.setBizName(domainSchemaDesc.getBizName());
|
chaConfigRichDesc.setBizName(domainSchemaInfo.getBizName());
|
||||||
chaConfigRichDesc.setName(domainSchemaDesc.getName());
|
chaConfigRichDesc.setName(domainSchemaInfo.getName());
|
||||||
|
|
||||||
chaConfigRichDesc.setDefaultMetrics(fetchDefaultMetricDescByConfig(chatConfigDesc));
|
chaConfigRichDesc.setKnowledgeInfos(
|
||||||
chaConfigRichDesc.setVisibility(fetchVisibilityDescByConfig(chatConfigDesc));
|
fillKnowledgeBizName(chaConfigRichDesc.getKnowledgeInfos(), domainSchemaInfo));
|
||||||
chaConfigRichDesc.setEntity(fetchEntityDescByConfig(chatConfigDesc));
|
chaConfigRichDesc.setDefaultMetrics(fetchDefaultMetricDescByConfig(chatConfigDesc, domainSchemaInfo));
|
||||||
|
chaConfigRichDesc.setVisibility(fetchVisibilityDescByConfig(chatConfigDesc, domainSchemaInfo));
|
||||||
|
chaConfigRichDesc.setEntity(fetchEntityDescByConfig(chatConfigDesc, domainSchemaInfo));
|
||||||
|
|
||||||
return chaConfigRichDesc;
|
return chaConfigRichDesc;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private List<KnowledgeInfo> fillKnowledgeBizName(List<KnowledgeInfo> knowledgeInfos,
|
||||||
|
DomainSchemaResp domainSchemaInfo) {
|
||||||
|
if (CollectionUtils.isEmpty(knowledgeInfos)) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
Map<Long, DimSchemaResp> dimIdAndRespPair = domainSchemaInfo.getDimensions().stream()
|
||||||
|
.collect(Collectors.toMap(DimSchemaResp::getId, Function.identity()));
|
||||||
|
knowledgeInfos.stream().forEach(knowledgeInfo -> {
|
||||||
|
if (Objects.nonNull(knowledgeInfo)) {
|
||||||
|
DimSchemaResp dimSchemaResp = dimIdAndRespPair.get(knowledgeInfo.getItemId());
|
||||||
|
if (Objects.nonNull(dimSchemaResp)) {
|
||||||
|
knowledgeInfo.setBizName(dimSchemaResp.getBizName());
|
||||||
|
}
|
||||||
|
if (CollectionUtils.isEmpty(knowledgeInfo.getBlackList())) {
|
||||||
|
knowledgeInfo.setBlackList(new ArrayList<>());
|
||||||
|
}
|
||||||
|
if (CollectionUtils.isEmpty(knowledgeInfo.getRuleList())) {
|
||||||
|
knowledgeInfo.setRuleList(new ArrayList<>());
|
||||||
|
}
|
||||||
|
if (CollectionUtils.isEmpty(knowledgeInfo.getWhiteList())) {
|
||||||
|
knowledgeInfo.setWhiteList(new ArrayList<>());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return knowledgeInfos;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<ChatConfigRichInfo> getAllChatRichConfig() {
|
||||||
|
List<ChatConfigRichInfo> chatConfigRichInfoList = new ArrayList<>();
|
||||||
|
List<DomainResp> domainRespList = defaultSemanticUtils.getDomainListForAdmin();
|
||||||
|
domainRespList.stream().forEach(domainResp -> {
|
||||||
|
ChatConfigRichInfo chatConfigRichInfo = getConfigRichInfo(domainResp.getId());
|
||||||
|
if (Objects.nonNull(chatConfigRichInfo)) {
|
||||||
|
chatConfigRichInfoList.add(chatConfigRichInfo);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return chatConfigRichInfoList;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -2,49 +2,44 @@ package com.tencent.supersonic.chat.application;
|
|||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
import com.tencent.supersonic.chat.api.pojo.DataInfo;
|
import com.tencent.supersonic.chat.api.pojo.DataInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.DomainInfo;
|
import com.tencent.supersonic.chat.api.pojo.DomainInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
|
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
import com.tencent.supersonic.chat.api.pojo.Filter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||||
import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
|
import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
|
||||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import org.slf4j.Logger;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
|
@Slf4j
|
||||||
public class DomainEntityService {
|
public class DomainEntityService {
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(DomainEntityService.class);
|
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
@Autowired
|
|
||||||
private SemanticLayer semanticLayer;
|
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private DefaultSemanticInternalUtils defaultSemanticUtils;
|
private DefaultSemanticInternalUtils defaultSemanticUtils;
|
||||||
|
|
||||||
public EntityInfo getEntityInfo(QueryContextReq queryCtx, ChatContext chatCtx, User user) {
|
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
|
||||||
SemanticParseInfo parseInfo = queryCtx.getParseInfo();
|
|
||||||
|
|
||||||
if (parseInfo != null && parseInfo.getDomainId() > 0) {
|
if (parseInfo != null && parseInfo.getDomainId() > 0) {
|
||||||
EntityInfo entityInfo = getEntityInfo(parseInfo.getDomainId());
|
EntityInfo entityInfo = getEntityInfo(parseInfo.getDomainId());
|
||||||
if (parseInfo.getDimensionFilters().size() <= 0) {
|
if (parseInfo.getDimensionFilters().size() <= 0) {
|
||||||
@@ -56,7 +51,8 @@ public class DomainEntityService {
|
|||||||
String domainInfoPrimaryName = entityInfo.getDomainInfo().getPrimaryEntityBizName();
|
String domainInfoPrimaryName = entityInfo.getDomainInfo().getPrimaryEntityBizName();
|
||||||
String domainInfoId = "";
|
String domainInfoId = "";
|
||||||
for (Filter chatFilter : parseInfo.getDimensionFilters()) {
|
for (Filter chatFilter : parseInfo.getDimensionFilters()) {
|
||||||
if (chatFilter.getBizName().equals(domainInfoPrimaryName)) {
|
if (chatFilter != null && chatFilter.getBizName() != null && chatFilter.getBizName()
|
||||||
|
.equals(domainInfoPrimaryName)) {
|
||||||
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
|
if (chatFilter.getOperator().equals(FilterOperatorEnum.EQUALS)) {
|
||||||
domainInfoId = chatFilter.getValue().toString();
|
domainInfoId = chatFilter.getValue().toString();
|
||||||
}
|
}
|
||||||
@@ -72,7 +68,7 @@ public class DomainEntityService {
|
|||||||
|
|
||||||
return entityInfo;
|
return entityInfo;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("setMaintDomain error {}", e);
|
log.error("setMaintDomain error {}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -88,7 +84,7 @@ public class DomainEntityService {
|
|||||||
private EntityInfo getEntityInfo(EntityRichInfo entityDesc) {
|
private EntityInfo getEntityInfo(EntityRichInfo entityDesc) {
|
||||||
EntityInfo entityInfo = new EntityInfo();
|
EntityInfo entityInfo = new EntityInfo();
|
||||||
|
|
||||||
if (entityDesc != null) {
|
if (entityDesc != null && Objects.nonNull(entityDesc.getDomainId())) {
|
||||||
DomainInfo domainInfo = new DomainInfo();
|
DomainInfo domainInfo = new DomainInfo();
|
||||||
domainInfo.setItemId(Integer.valueOf(entityDesc.getDomainId().intValue()));
|
domainInfo.setItemId(Integer.valueOf(entityDesc.getDomainId().intValue()));
|
||||||
domainInfo.setName(entityDesc.getDomainName());
|
domainInfo.setName(entityDesc.getDomainName());
|
||||||
@@ -148,7 +144,7 @@ public class DomainEntityService {
|
|||||||
queryResultWithColumns = semanticLayer.queryByStruct(SchemaInfoConverter.convertTo(semanticParseInfo),
|
queryResultWithColumns = semanticLayer.queryByStruct(SchemaInfoConverter.convertTo(semanticParseInfo),
|
||||||
user);
|
user);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.warn("setMainDomain queryByStruct error, e:", e);
|
log.warn("setMainDomain queryByStruct error, e:", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (queryResultWithColumns != null) {
|
if (queryResultWithColumns != null) {
|
||||||
|
|||||||
@@ -2,52 +2,44 @@ package com.tencent.supersonic.chat.application;
|
|||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.api.component.*;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||||
import com.tencent.supersonic.chat.api.service.SchemaMapper;
|
import com.tencent.supersonic.chat.application.query.QuerySelector;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticParser;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
|
||||||
import com.tencent.supersonic.chat.application.query.SemanticQueryFactory;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.QueryData;
|
import com.tencent.supersonic.chat.domain.pojo.chat.QueryData;
|
||||||
import com.tencent.supersonic.chat.domain.service.ChatService;
|
import com.tencent.supersonic.chat.domain.pojo.search.QueryState;
|
||||||
import com.tencent.supersonic.chat.domain.service.QueryService;
|
import com.tencent.supersonic.chat.domain.service.QueryService;
|
||||||
|
import com.tencent.supersonic.chat.domain.service.ChatService;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||||
import com.tencent.supersonic.common.util.json.JsonUtil;
|
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.core.io.support.SpringFactoriesLoader;
|
import org.springframework.context.annotation.Primary;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
|
@Component("chatQueryService")
|
||||||
|
@Primary
|
||||||
|
@Slf4j
|
||||||
public class QueryServiceImpl implements QueryService {
|
public class QueryServiceImpl implements QueryService {
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(QueryServiceImpl.class);
|
|
||||||
private List<SchemaMapper> schemaMappers;
|
|
||||||
private List<SemanticParser> semanticParsers;
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private ChatService chatService;
|
private ChatService chatService;
|
||||||
@Autowired
|
|
||||||
private SemanticLayer semanticLayer;
|
|
||||||
|
|
||||||
public QueryServiceImpl() {
|
private List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
|
||||||
schemaMappers = SpringFactoriesLoader.loadFactories(SchemaMapper.class,
|
private List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
|
||||||
Thread.currentThread().getContextClassLoader());
|
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
semanticParsers = SpringFactoriesLoader.loadFactories(SemanticParser.class,
|
private QuerySelector querySelector = ComponentFactory.getQuerySelector();
|
||||||
Thread.currentThread().getContextClassLoader());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
@Override
|
||||||
public QueryResultResp executeQuery(QueryContextReq queryCtx) throws Exception {
|
public QueryResultResp executeQuery(QueryContextReq queryCtx) throws Exception {
|
||||||
schemaMappers.stream().forEach(s -> s.map(queryCtx));
|
schemaMappers.stream().forEach(s -> s.map(queryCtx));
|
||||||
|
|
||||||
@@ -55,24 +47,29 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
ChatContext chatCtx = chatService.getOrCreateContext(queryCtx.getChatId());
|
ChatContext chatCtx = chatService.getOrCreateContext(queryCtx.getChatId());
|
||||||
|
|
||||||
for (SemanticParser semanticParser : semanticParsers) {
|
for (SemanticParser semanticParser : semanticParsers) {
|
||||||
logger.info("semanticParser processing:{}", JsonUtil.prettyToString(semanticParser));
|
log.info("semanticParser processing:[{}]", semanticParser.getClass().getName());
|
||||||
boolean isFinish = semanticParser.parse(queryCtx, chatCtx);
|
semanticParser.parse(queryCtx, chatCtx);
|
||||||
if (isFinish) {
|
}
|
||||||
logger.info("semanticParser is finish ,semanticParser:{}", semanticParser.getClass().getName());
|
|
||||||
break;
|
if (queryCtx.getCandidateQueries().size() > 0) {
|
||||||
|
log.info("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
|
||||||
|
Collectors.toList()));
|
||||||
|
SemanticQuery semanticQuery = querySelector.select(queryCtx.getCandidateQueries());
|
||||||
|
log.info("pick after [{}]", semanticQuery);
|
||||||
|
|
||||||
|
QueryResultResp queryResponse = semanticQuery.execute(queryCtx.getUser());
|
||||||
|
if (queryResponse != null) {
|
||||||
|
// update chat context after a successful semantic query
|
||||||
|
if (queryCtx.isSaveAnswer() && queryResponse.getQueryState() == QueryState.NORMAL.getState()) {
|
||||||
|
chatService.updateContext(chatCtx, queryCtx, semanticQuery.getParseInfo());
|
||||||
|
}
|
||||||
|
queryResponse.setChatContext(chatCtx.getParseInfo());
|
||||||
|
chatService.addQuery(queryResponse, queryCtx, chatCtx);
|
||||||
|
return queryResponse;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// submit semantic query based on the result of semantic parsing
|
|
||||||
SemanticQuery query = SemanticQueryFactory.get(queryCtx.getParseInfo().getQueryMode());
|
|
||||||
|
|
||||||
QueryResultResp queryResponse = query.execute(queryCtx, chatCtx);
|
return null;
|
||||||
|
|
||||||
// update chat context after a successful semantic query
|
|
||||||
query.updateContext(queryResponse, chatCtx, queryCtx);
|
|
||||||
|
|
||||||
chatService.addQuery(queryResponse, queryCtx, chatCtx);
|
|
||||||
|
|
||||||
return queryResponse;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -82,7 +79,7 @@ public class QueryServiceImpl implements QueryService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QueryResultResp queryData(QueryData queryData, User user) throws Exception {
|
public QueryResultResp executeDirectQuery(QueryData queryData, User user) throws Exception {
|
||||||
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
||||||
QueryResultResp queryResponse = new QueryResultResp();
|
QueryResultResp queryResponse = new QueryResultResp();
|
||||||
BeanUtils.copyProperties(queryData, semanticParseInfo);
|
BeanUtils.copyProperties(queryData, semanticParseInfo);
|
||||||
|
|||||||
@@ -2,14 +2,15 @@ package com.tencent.supersonic.chat.application;
|
|||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.RecommendResponse;
|
import com.tencent.supersonic.chat.domain.pojo.chat.RecommendResponse;
|
||||||
import com.tencent.supersonic.chat.domain.service.RecommendService;
|
import com.tencent.supersonic.chat.domain.service.RecommendService;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
/***
|
/***
|
||||||
@@ -18,8 +19,7 @@ import org.springframework.stereotype.Service;
|
|||||||
@Service
|
@Service
|
||||||
public class RecommendServiceImpl implements RecommendService {
|
public class RecommendServiceImpl implements RecommendService {
|
||||||
|
|
||||||
@Autowired
|
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
private SemanticLayer semanticLayer;
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RecommendResponse recommend(QueryContextReq queryCtx) {
|
public RecommendResponse recommend(QueryContextReq queryCtx) {
|
||||||
|
|||||||
@@ -32,9 +32,9 @@ import java.util.Objects;
|
|||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -43,9 +43,11 @@ import org.springframework.stereotype.Service;
|
|||||||
* search service impl
|
* search service impl
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
|
@Slf4j
|
||||||
public class SearchServiceImpl implements SearchService {
|
public class SearchServiceImpl implements SearchService {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(SearchServiceImpl.class);
|
private static final int RESULT_SIZE = 10;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private WordNatureService wordNatureService;
|
private WordNatureService wordNatureService;
|
||||||
@Autowired
|
@Autowired
|
||||||
@@ -53,9 +55,6 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private SearchMatchStrategy searchMatchStrategy;
|
private SearchMatchStrategy searchMatchStrategy;
|
||||||
|
|
||||||
private static final int RESULT_SIZE = 10;
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<SearchResult> search(QueryContextReq queryCtx) {
|
public List<SearchResult> search(QueryContextReq queryCtx) {
|
||||||
String queryText = queryCtx.getQueryText();
|
String queryText = queryCtx.getQueryText();
|
||||||
@@ -67,7 +66,7 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
// 2.detect by segment
|
// 2.detect by segment
|
||||||
List<Term> originals = HanlpHelper.getSegment().seg(queryText.toLowerCase()).stream()
|
List<Term> originals = HanlpHelper.getSegment().seg(queryText.toLowerCase()).stream()
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
Map<MatchText, List<MapResult>> regTextMap = searchMatchStrategy.matchWithMatchText(queryText, originals,
|
Map<MatchText, List<MapResult>> regTextMap = searchMatchStrategy.match(queryText, originals,
|
||||||
queryCtx.getDomainId());
|
queryCtx.getDomainId());
|
||||||
regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue()));
|
regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue()));
|
||||||
// 3.get the most matching data
|
// 3.get the most matching data
|
||||||
@@ -77,14 +76,14 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
.reduce((entry1, entry2) ->
|
.reduce((entry1, entry2) ->
|
||||||
entry1.getKey().getDetectSegment().length() >= entry2.getKey().getDetectSegment().length()
|
entry1.getKey().getDetectSegment().length() >= entry2.getKey().getDetectSegment().length()
|
||||||
? entry1 : entry2);
|
? entry1 : entry2);
|
||||||
LOGGER.debug("mostSimilarSearchResult:{}", mostSimilarSearchResult);
|
log.debug("mostSimilarSearchResult:{}", mostSimilarSearchResult);
|
||||||
// 4.optimize the results after the query
|
// 4.optimize the results after the query
|
||||||
if (!mostSimilarSearchResult.isPresent()) {
|
if (!mostSimilarSearchResult.isPresent()) {
|
||||||
LOGGER.info("unable to find any information through search , queryCtx:{}", queryCtx);
|
log.info("unable to find any information through search , queryCtx:{}", queryCtx);
|
||||||
return Lists.newArrayList();
|
return Lists.newArrayList();
|
||||||
}
|
}
|
||||||
Map.Entry<MatchText, List<MapResult>> searchTextEntry = mostSimilarSearchResult.get();
|
Map.Entry<MatchText, List<MapResult>> searchTextEntry = mostSimilarSearchResult.get();
|
||||||
LOGGER.info("searchTextEntry:{},queryCtx:{}", searchTextEntry, queryCtx);
|
log.info("searchTextEntry:{},queryCtx:{}", searchTextEntry, queryCtx);
|
||||||
|
|
||||||
Set<SearchResult> searchResults = new LinkedHashSet();
|
Set<SearchResult> searchResults = new LinkedHashSet();
|
||||||
DomainInfoStat domainStat = NatureHelper.getDomainStat(originals);
|
DomainInfoStat domainStat = NatureHelper.getDomainStat(originals);
|
||||||
@@ -98,7 +97,7 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
// 4.2 process based on dimension values
|
// 4.2 process based on dimension values
|
||||||
MatchText matchText = searchTextEntry.getKey();
|
MatchText matchText = searchTextEntry.getKey();
|
||||||
Map<String, String> natureToNameMap = getNatureToNameMap(searchTextEntry, new HashSet<>(possibleDomains));
|
Map<String, String> natureToNameMap = getNatureToNameMap(searchTextEntry, new HashSet<>(possibleDomains));
|
||||||
LOGGER.debug("possibleDomains:{},natureToNameMap:{}", possibleDomains, natureToNameMap);
|
log.debug("possibleDomains:{},natureToNameMap:{}", possibleDomains, natureToNameMap);
|
||||||
|
|
||||||
for (Map.Entry<String, String> natureToNameEntry : natureToNameMap.entrySet()) {
|
for (Map.Entry<String, String> natureToNameEntry : natureToNameMap.entrySet()) {
|
||||||
searchDimensionValue(metricsDb, domainToName, domainStat.getMetricDomainCount(), searchResults,
|
searchDimensionValue(metricsDb, domainToName, domainStat.getMetricDomainCount(), searchResults,
|
||||||
@@ -120,7 +119,7 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
|
|
||||||
Long contextDomain = chatService.getContextDomain(queryCtx.getChatId());
|
Long contextDomain = chatService.getContextDomain(queryCtx.getChatId());
|
||||||
|
|
||||||
LOGGER.debug("possibleDomains:{},domainStat:{},contextDomain:{}", possibleDomains, domainStat, contextDomain);
|
log.debug("possibleDomains:{},domainStat:{},contextDomain:{}", possibleDomains, domainStat, contextDomain);
|
||||||
|
|
||||||
// If nothing is recognized or only metric are present, then add the contextDomain.
|
// If nothing is recognized or only metric are present, then add the contextDomain.
|
||||||
if (nothingOrOnlyMetric(domainStat) && effectiveDomain(contextDomain)) {
|
if (nothingOrOnlyMetric(domainStat) && effectiveDomain(contextDomain)) {
|
||||||
@@ -249,7 +248,7 @@ public class SearchServiceImpl implements SearchService {
|
|||||||
domainToName.get(domain), domain, semanticType));
|
domainToName.get(domain), domain, semanticType));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
LOGGER.info("parseResult:{},dimensionMetricClassIds:{},possibleDomains:{}", mapResult,
|
log.info("parseResult:{},dimensionMetricClassIds:{},possibleDomains:{}", mapResult,
|
||||||
dimensionMetricClassIds, possibleDomains);
|
dimensionMetricClassIds, possibleDomains);
|
||||||
}
|
}
|
||||||
return existMetric;
|
return existMetric;
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ public class ApplicationStartedInit implements ApplicationListener<ApplicationSt
|
|||||||
*/
|
*/
|
||||||
@Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}")
|
@Scheduled(cron = "${reload.knowledge.corn:0 0/1 * * * ?}")
|
||||||
public void reloadKnowledge() {
|
public void reloadKnowledge() {
|
||||||
log.debug("reloadKnowledge start");
|
log.info("reloadKnowledge start");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
List<WordNature> wordNatures = wordNatureService.getAllWordNature();
|
List<WordNature> wordNatures = wordNatureService.getAllWordNature();
|
||||||
@@ -55,6 +55,7 @@ public class ApplicationStartedInit implements ApplicationListener<ApplicationSt
|
|||||||
log.debug("wordNatures is not change, reloadKnowledge end");
|
log.debug("wordNatures is not change, reloadKnowledge end");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
log.info("wordNatures is change");
|
||||||
wordNatureService.setPreWordNatures(wordNatures);
|
wordNatureService.setPreWordNatures(wordNatures);
|
||||||
onlineKnowledgeService.updateOnlineKnowledge(wordNatureService.getAllWordNature());
|
onlineKnowledgeService.updateOnlineKnowledge(wordNatureService.getAllWordNature());
|
||||||
wordNatureService.getCache().refresh("");
|
wordNatureService.getCache().refresh("");
|
||||||
@@ -63,6 +64,6 @@ public class ApplicationStartedInit implements ApplicationListener<ApplicationSt
|
|||||||
log.error("reloadKnowledge error", e);
|
log.error("reloadKnowledge error", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
log.debug("reloadKnowledge end");
|
log.info("reloadKnowledge end");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -29,19 +29,16 @@ import org.springframework.util.CollectionUtils;
|
|||||||
@Service
|
@Service
|
||||||
public class DictApplicationService {
|
public class DictApplicationService {
|
||||||
|
|
||||||
@Value("${dict.flush.enable:true}")
|
|
||||||
private Boolean dictFlushEnable;
|
|
||||||
|
|
||||||
@Value("${dict.file.type:txt}")
|
|
||||||
private String dictFileType;
|
|
||||||
private String dimValue = "DimValue_%d_%d";
|
|
||||||
private String dateTimeFormatter = "yyyyMMddHHmmss";
|
|
||||||
|
|
||||||
|
|
||||||
private final DictMetaUtils metaUtils;
|
private final DictMetaUtils metaUtils;
|
||||||
private final DictQueryUtils dictQueryUtils;
|
private final DictQueryUtils dictQueryUtils;
|
||||||
private final FileHandler fileHandler;
|
private final FileHandler fileHandler;
|
||||||
private final DictRepository dictRepository;
|
private final DictRepository dictRepository;
|
||||||
|
@Value("${dict.flush.enable:true}")
|
||||||
|
private Boolean dictFlushEnable;
|
||||||
|
@Value("${dict.file.type:txt}")
|
||||||
|
private String dictFileType;
|
||||||
|
private String dimValue = "DimValue_%d_%d";
|
||||||
|
private String dateTimeFormatter = "yyyyMMddHHmmss";
|
||||||
|
|
||||||
public DictApplicationService(DictMetaUtils metaUtils,
|
public DictApplicationService(DictMetaUtils metaUtils,
|
||||||
DictQueryUtils dictQueryUtils,
|
DictQueryUtils dictQueryUtils,
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
@@ -20,10 +22,9 @@ import org.slf4j.LoggerFactory;
|
|||||||
/**
|
/**
|
||||||
* nature parse helper
|
* nature parse helper
|
||||||
*/
|
*/
|
||||||
|
@Slf4j
|
||||||
public class NatureHelper {
|
public class NatureHelper {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(NatureHelper.class);
|
|
||||||
|
|
||||||
private static boolean isDomainOrEntity(Term term, Integer domain) {
|
private static boolean isDomainOrEntity(Term term, Integer domain) {
|
||||||
return (NatureType.NATURE_SPILT + domain).equals(term.nature.toString()) || term.nature.toString()
|
return (NatureType.NATURE_SPILT + domain).equals(term.nature.toString()) || term.nature.toString()
|
||||||
.endsWith(NatureType.ENTITY.getType());
|
.endsWith(NatureType.ENTITY.getType());
|
||||||
@@ -47,7 +48,7 @@ public class NatureHelper {
|
|||||||
}
|
}
|
||||||
return Integer.valueOf(split[1]);
|
return Integer.valueOf(split[1]);
|
||||||
} catch (NumberFormatException e) {
|
} catch (NumberFormatException e) {
|
||||||
LOGGER.error("", e);
|
log.error("", e);
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,8 +3,9 @@ package com.tencent.supersonic.chat.application.knowledge;
|
|||||||
import com.google.common.cache.CacheBuilder;
|
import com.google.common.cache.CacheBuilder;
|
||||||
import com.google.common.cache.CacheLoader;
|
import com.google.common.cache.CacheLoader;
|
||||||
import com.google.common.cache.LoadingCache;
|
import com.google.common.cache.LoadingCache;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||||
import com.tencent.supersonic.common.nlp.ItemDO;
|
import com.tencent.supersonic.common.nlp.ItemDO;
|
||||||
import com.tencent.supersonic.common.nlp.NatureType;
|
import com.tencent.supersonic.common.nlp.NatureType;
|
||||||
@@ -13,9 +14,10 @@ import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFac
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
|
||||||
@@ -23,14 +25,11 @@ import org.springframework.stereotype.Service;
|
|||||||
* word nature service
|
* word nature service
|
||||||
**/
|
**/
|
||||||
@Service
|
@Service
|
||||||
|
@Slf4j
|
||||||
public class WordNatureService {
|
public class WordNatureService {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(WordNatureService.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private SemanticLayer semanticLayer;
|
|
||||||
private static final Integer META_CACHE_TIME = 5;
|
private static final Integer META_CACHE_TIME = 5;
|
||||||
|
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
private List<WordNature> preWordNatures = new ArrayList<>();
|
private List<WordNature> preWordNatures = new ArrayList<>();
|
||||||
|
|
||||||
private LoadingCache<String, DomainInfos> cache = CacheBuilder.newBuilder()
|
private LoadingCache<String, DomainInfos> cache = CacheBuilder.newBuilder()
|
||||||
@@ -39,14 +38,14 @@ public class WordNatureService {
|
|||||||
new CacheLoader<String, DomainInfos>() {
|
new CacheLoader<String, DomainInfos>() {
|
||||||
@Override
|
@Override
|
||||||
public DomainInfos load(String key) {
|
public DomainInfos load(String key) {
|
||||||
LOGGER.info("load getDomainSchemaInfo cache [{}]", key);
|
log.info("load getDomainSchemaInfo cache [{}]", key);
|
||||||
return SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
|
return SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
public List<WordNature> getAllWordNature() {
|
public List<WordNature> getAllWordNature() {
|
||||||
|
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
DomainInfos domainInfos = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
|
DomainInfos domainInfos = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
|
||||||
|
|
||||||
List<WordNature> natures = new ArrayList<>();
|
List<WordNature> natures = new ArrayList<>();
|
||||||
@@ -64,7 +63,7 @@ public class WordNatureService {
|
|||||||
|
|
||||||
private void addNatureToResult(NatureType value, List<ItemDO> metas, List<WordNature> natures) {
|
private void addNatureToResult(NatureType value, List<ItemDO> metas, List<WordNature> natures) {
|
||||||
List<WordNature> natureList = WordNatureStrategyFactory.get(value).getWordNatureList(metas);
|
List<WordNature> natureList = WordNatureStrategyFactory.get(value).getWordNatureList(metas);
|
||||||
LOGGER.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
|
log.debug("nature type:{} , nature size:{}", value.name(), natureList.size());
|
||||||
natures.addAll(natureList);
|
natures.addAll(natureList);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,144 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.mapper;
|
||||||
|
|
||||||
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
|
import com.tencent.supersonic.chat.application.knowledge.WordNatureService;
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||||
|
import com.tencent.supersonic.common.nlp.ItemDO;
|
||||||
|
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||||
|
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class DatabaseSchemaMapper implements SchemaMapper {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void map(QueryContextReq queryContext) {
|
||||||
|
|
||||||
|
log.debug("before db mapper,mapInfo:{}", queryContext.getMapInfo());
|
||||||
|
|
||||||
|
List<Term> terms = HanlpHelper.getSegment().seg(queryContext.getQueryText().toLowerCase()).stream()
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
WordNatureService wordNatureService = ContextUtils.getBean(WordNatureService.class);
|
||||||
|
|
||||||
|
DomainInfos domainInfos = wordNatureService.getCache().getUnchecked("");
|
||||||
|
|
||||||
|
detectAndAddToSchema(queryContext, terms, domainInfos.getDimensions(),
|
||||||
|
SchemaElementType.DIMENSION);
|
||||||
|
detectAndAddToSchema(queryContext, terms, domainInfos.getMetrics(), SchemaElementType.METRIC);
|
||||||
|
|
||||||
|
log.debug("after db mapper,mapInfo:{}", queryContext.getMapInfo());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void detectAndAddToSchema(QueryContextReq queryContext, List<Term> terms, List<ItemDO> domains,
|
||||||
|
SchemaElementType schemaElementType) {
|
||||||
|
try {
|
||||||
|
String queryText = queryContext.getQueryText();
|
||||||
|
|
||||||
|
Map<String, Set<ItemDO>> domainResultSet = getResultSet(queryText, terms, domains);
|
||||||
|
|
||||||
|
addToSchemaMapInfo(domainResultSet, queryContext.getMapInfo(), schemaElementType);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("detectAndAddToSchema error", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Set<ItemDO>> getResultSet(String queryText, List<Term> terms, List<ItemDO> domains) {
|
||||||
|
|
||||||
|
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
|
||||||
|
|
||||||
|
Map<String, Set<ItemDO>> nameToItems = getNameToItems(domains);
|
||||||
|
|
||||||
|
Map<Integer, Integer> regOffsetToLength = terms.stream().sorted(Comparator.comparing(Term::length))
|
||||||
|
.collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2));
|
||||||
|
|
||||||
|
Map<String, Set<ItemDO>> domainResultSet = new HashMap<>();
|
||||||
|
for (Integer index = 0; index <= queryText.length() - 1; ) {
|
||||||
|
for (Integer i = index; i <= queryText.length(); ) {
|
||||||
|
i = mapperHelper.getStepIndex(regOffsetToLength, i);
|
||||||
|
if (i <= queryText.length()) {
|
||||||
|
String detectSegment = queryText.substring(index, i);
|
||||||
|
nameToItems.forEach(
|
||||||
|
(name, newItemDOs) -> {
|
||||||
|
if (name.contains(detectSegment)
|
||||||
|
&& mapperHelper.getSimilarity(detectSegment, name)
|
||||||
|
>= mapperHelper.getMetricDimensionThresholdConfig()) {
|
||||||
|
Set<ItemDO> preItemDOS = domainResultSet.putIfAbsent(detectSegment, newItemDOs);
|
||||||
|
if (Objects.nonNull(preItemDOS)) {
|
||||||
|
preItemDOS.addAll(newItemDOs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
index = mapperHelper.getStepIndex(regOffsetToLength, index);
|
||||||
|
}
|
||||||
|
return domainResultSet;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Set<ItemDO>> getNameToItems(List<ItemDO> domains) {
|
||||||
|
return domains.stream()
|
||||||
|
.collect(Collectors.toMap(ItemDO::getName, a -> {
|
||||||
|
Set<ItemDO> result = new HashSet<>();
|
||||||
|
result.add(a);
|
||||||
|
return result;
|
||||||
|
}, (k1, k2) -> {
|
||||||
|
k1.addAll(k2);
|
||||||
|
return k1;
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addToSchemaMapInfo(Map<String, Set<ItemDO>> mapResultRowSet, SchemaMapInfo schemaMap,
|
||||||
|
SchemaElementType schemaElementType) {
|
||||||
|
if (Objects.isNull(mapResultRowSet) || mapResultRowSet.size() <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
|
||||||
|
|
||||||
|
for (Map.Entry<String, Set<ItemDO>> entry : mapResultRowSet.entrySet()) {
|
||||||
|
String detectWord = entry.getKey();
|
||||||
|
Set<ItemDO> itemDOS = entry.getValue();
|
||||||
|
for (ItemDO itemDO : itemDOS) {
|
||||||
|
|
||||||
|
List<SchemaElementMatch> elements = schemaMap.getMatchedElements(itemDO.getDomain());
|
||||||
|
if (CollectionUtils.isEmpty(elements)) {
|
||||||
|
elements = new ArrayList<>();
|
||||||
|
schemaMap.setMatchedElements(itemDO.getDomain(), elements);
|
||||||
|
}
|
||||||
|
Set<Integer> regElementSet = elements.stream()
|
||||||
|
.filter(elementMatch -> schemaElementType.equals(elementMatch.getElementType()))
|
||||||
|
.map(elementMatch -> elementMatch.getElementID())
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
|
if (regElementSet.contains(itemDO.getItemId())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||||
|
.elementID(itemDO.getItemId()).word(itemDO.getName()).frequency(10000L)
|
||||||
|
.elementType(schemaElementType).detectWord(detectWord)
|
||||||
|
.similarity(mapperHelper.getSimilarity(detectWord, itemDO.getName()))
|
||||||
|
.build();
|
||||||
|
log.info("schemaElementType:{},add to schema, elementMatch {}", schemaElementType, schemaElementMatch);
|
||||||
|
elements.add(schemaElementMatch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,17 +1,17 @@
|
|||||||
package com.tencent.supersonic.chat.application.mapper;
|
package com.tencent.supersonic.chat.application.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.service.SchemaMapper;
|
|
||||||
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
|
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
|
||||||
import com.tencent.supersonic.chat.domain.utils.NatureConverter;
|
import com.tencent.supersonic.chat.domain.utils.NatureConverter;
|
||||||
import com.tencent.supersonic.common.nlp.MapResult;
|
import com.tencent.supersonic.common.nlp.MapResult;
|
||||||
import com.tencent.supersonic.common.nlp.NatureType;
|
import com.tencent.supersonic.common.nlp.NatureType;
|
||||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||||
import com.tencent.supersonic.common.util.json.JsonUtil;
|
|
||||||
import com.tencent.supersonic.knowledge.application.online.BaseWordNature;
|
import com.tencent.supersonic.knowledge.application.online.BaseWordNature;
|
||||||
import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory;
|
import com.tencent.supersonic.knowledge.application.online.WordNatureStrategyFactory;
|
||||||
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
|
import com.tencent.supersonic.knowledge.infrastructure.nlp.HanlpHelper;
|
||||||
@@ -19,15 +19,14 @@ import java.util.ArrayList;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.Optional;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
public class HanlpSchemaMapper implements SchemaMapper {
|
public class HanlpSchemaMapper implements SchemaMapper {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(HanlpSchemaMapper.class);
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void map(QueryContextReq queryContext) {
|
public void map(QueryContextReq queryContext) {
|
||||||
|
|
||||||
@@ -35,22 +34,37 @@ public class HanlpSchemaMapper implements SchemaMapper {
|
|||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
terms.forEach(
|
terms.forEach(
|
||||||
item -> LOGGER.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(),
|
item -> log.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(),
|
||||||
item.getFrequency())
|
item.getFrequency())
|
||||||
);
|
);
|
||||||
QueryMatchStrategy matchStrategy = ContextUtils.getBean(QueryMatchStrategy.class);
|
QueryMatchStrategy matchStrategy = ContextUtils.getBean(QueryMatchStrategy.class);
|
||||||
|
|
||||||
List<MapResult> matches = matchStrategy.match(queryContext.getQueryText(), terms, queryContext.getDomainId());
|
Map<MatchText, List<MapResult>> matchResult = matchStrategy.match(queryContext.getQueryText(), terms,
|
||||||
|
queryContext.getDomainId());
|
||||||
|
List<MapResult> matches = new ArrayList<>();
|
||||||
|
if (Objects.nonNull(matchResult)) {
|
||||||
|
Optional<List<MapResult>> first = matchResult.entrySet().stream()
|
||||||
|
.filter(entry -> CollectionUtils.isNotEmpty(entry.getValue()))
|
||||||
|
.map(entry -> entry.getValue()).findFirst();
|
||||||
|
if (first.isPresent()) {
|
||||||
|
matches = first.get();
|
||||||
|
}
|
||||||
|
}
|
||||||
HanlpHelper.transLetterOriginal(matches);
|
HanlpHelper.transLetterOriginal(matches);
|
||||||
LOGGER.info("queryContext:{},matches:{}", queryContext, matches);
|
log.info("queryContext:{},matches:{}", queryContext, matches);
|
||||||
|
|
||||||
convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo());
|
convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo(), terms);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void convertTermsToSchemaMapInfo(List<MapResult> mapResults, SchemaMapInfo schemaMap) {
|
private void convertTermsToSchemaMapInfo(List<MapResult> mapResults, SchemaMapInfo schemaMap, List<Term> terms) {
|
||||||
if (CollectionUtils.isEmpty(mapResults)) {
|
if (CollectionUtils.isEmpty(mapResults)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Map<String, Long> wordNatureToFrequency = terms.stream().collect(
|
||||||
|
Collectors.toMap(entry -> entry.getWord() + entry.getNature(),
|
||||||
|
term -> Long.valueOf(term.getFrequency()), (value1, value2) -> value2));
|
||||||
|
|
||||||
for (MapResult mapResult : mapResults) {
|
for (MapResult mapResult : mapResults) {
|
||||||
for (String nature : mapResult.getNatures()) {
|
for (String nature : mapResult.getNatures()) {
|
||||||
Integer domain = NatureHelper.getDomain(nature);
|
Integer domain = NatureHelper.getDomain(nature);
|
||||||
@@ -61,17 +75,18 @@ public class HanlpSchemaMapper implements SchemaMapper {
|
|||||||
if (Objects.isNull(elementType)) {
|
if (Objects.isNull(elementType)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
SchemaElementMatch schemaElementMatch = new SchemaElementMatch();
|
|
||||||
|
|
||||||
schemaElementMatch.setElementType(elementType);
|
|
||||||
BaseWordNature baseWordNature = WordNatureStrategyFactory.get(NatureType.getNatureType(nature));
|
BaseWordNature baseWordNature = WordNatureStrategyFactory.get(NatureType.getNatureType(nature));
|
||||||
Integer elementID = baseWordNature.getElementID(nature);
|
Integer elementID = baseWordNature.getElementID(nature);
|
||||||
schemaElementMatch.setElementID(elementID);
|
Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature);
|
||||||
Long frequency = baseWordNature.getFrequency(nature);
|
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||||
schemaElementMatch.setFrequency(frequency);
|
.elementType(elementType)
|
||||||
schemaElementMatch.setWord(mapResult.getName());
|
.elementID(elementID)
|
||||||
schemaElementMatch.setSimilarity(mapResult.getSimilarity());
|
.frequency(frequency)
|
||||||
schemaElementMatch.setDetectWord(mapResult.getDetectWord());
|
.word(mapResult.getName())
|
||||||
|
.similarity(mapResult.getSimilarity())
|
||||||
|
.detectWord(mapResult.getDetectWord())
|
||||||
|
.build();
|
||||||
|
|
||||||
Map<Integer, List<SchemaElementMatch>> domainElementMatches = schemaMap.getDomainElementMatches();
|
Map<Integer, List<SchemaElementMatch>> domainElementMatches = schemaMap.getDomainElementMatches();
|
||||||
List<SchemaElementMatch> schemaElementMatches = domainElementMatches.putIfAbsent(domain,
|
List<SchemaElementMatch> schemaElementMatches = domainElementMatches.putIfAbsent(domain,
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.mapper;
|
||||||
|
|
||||||
|
import com.hankcs.hanlp.algorithm.EditDistance;
|
||||||
|
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mapper helper
|
||||||
|
*/
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Service
|
||||||
|
@Slf4j
|
||||||
|
public class MapperHelper {
|
||||||
|
|
||||||
|
@Value("${one.detection.size:6}")
|
||||||
|
private Integer oneDetectionSize;
|
||||||
|
@Value("${one.detection.max.size:20}")
|
||||||
|
private Integer oneDetectionMaxSize;
|
||||||
|
@Value("${metric.dimension.threshold:0.3}")
|
||||||
|
private Double metricDimensionThresholdConfig;
|
||||||
|
@Value("${dimension.value.threshold:0.5}")
|
||||||
|
private Double dimensionValueThresholdConfig;
|
||||||
|
|
||||||
|
public Integer getStepIndex(Map<Integer, Integer> regOffsetToLength, Integer index) {
|
||||||
|
Integer subRegLength = regOffsetToLength.get(index);
|
||||||
|
if (Objects.nonNull(subRegLength)) {
|
||||||
|
index = index + subRegLength;
|
||||||
|
} else {
|
||||||
|
index++;
|
||||||
|
}
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getStepOffset(List<Integer> termList, Integer index) {
|
||||||
|
for (int j = 0; j < termList.size() - 1; j++) {
|
||||||
|
if (termList.get(j) <= index && termList.get(j + 1) > index) {
|
||||||
|
return termList.get(j);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
|
public double getThresholdMatch(List<String> natures) {
|
||||||
|
if (existDimensionValues(natures)) {
|
||||||
|
return dimensionValueThresholdConfig;
|
||||||
|
}
|
||||||
|
return metricDimensionThresholdConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
/***
|
||||||
|
* exist dimension values
|
||||||
|
* @param natures
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public boolean existDimensionValues(List<String> natures) {
|
||||||
|
for (String nature : natures) {
|
||||||
|
if (NatureHelper.isDimensionValueClassId(nature)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/***
|
||||||
|
* get similarity
|
||||||
|
* @param detectSegment
|
||||||
|
* @param matchName
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public double getSimilarity(String detectSegment, String matchName) {
|
||||||
|
String detectSegmentLower = detectSegment == null ? null : detectSegment.toLowerCase();
|
||||||
|
String matchNameLower = matchName == null ? null : matchName.toLowerCase();
|
||||||
|
return 1 - (double) EditDistance.compute(detectSegmentLower, matchNameLower) / Math.max(matchName.length(),
|
||||||
|
detectSegment.length());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,6 @@
|
|||||||
package com.tencent.supersonic.chat.application.mapper;
|
package com.tencent.supersonic.chat.application.mapper;
|
||||||
|
|
||||||
import com.hankcs.hanlp.algorithm.EditDistance;
|
|
||||||
import com.hankcs.hanlp.seg.common.Term;
|
import com.hankcs.hanlp.seg.common.Term;
|
||||||
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
|
import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
|
||||||
import com.tencent.supersonic.common.nlp.MapResult;
|
import com.tencent.supersonic.common.nlp.MapResult;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -13,33 +11,6 @@ import java.util.Map;
|
|||||||
*/
|
*/
|
||||||
public interface MatchStrategy {
|
public interface MatchStrategy {
|
||||||
|
|
||||||
List<MapResult> match(String text, List<Term> terms, Integer detectDomainId);
|
Map<MatchText, List<MapResult>> match(String text, List<Term> terms, Integer detectDomainId);
|
||||||
|
|
||||||
|
|
||||||
Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals, Integer detectDomainId);
|
|
||||||
|
|
||||||
/***
|
|
||||||
* exist dimension values
|
|
||||||
* @param natures
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
default boolean existDimensionValues(List<String> natures) {
|
|
||||||
for (String nature : natures) {
|
|
||||||
if (NatureHelper.isDimensionValueClassId(nature)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/***
|
|
||||||
* get similarity
|
|
||||||
* @param detectSegment
|
|
||||||
* @param matchName
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
default double getSimilarity(String detectSegment, String matchName) {
|
|
||||||
return 1 - (double) EditDistance.compute(detectSegment, matchName) / Math.max(matchName.length(),
|
|
||||||
detectSegment.length());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.mapper;
|
||||||
|
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.*;
|
||||||
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class QueryFilterMapper implements SchemaMapper {
|
||||||
|
|
||||||
|
private Long FREQUENCY = 9999999L;
|
||||||
|
|
||||||
|
private double SIMILARITY = 1.0;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void map(QueryContextReq queryContext) {
|
||||||
|
Integer domainId = queryContext.getDomainId();
|
||||||
|
if (domainId == null || domainId <= 0 || queryContext.getQueryFilter() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
QueryFilter queryFilter = queryContext.getQueryFilter();
|
||||||
|
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
|
||||||
|
List<SchemaElementMatch> schemaElementMatches = schemaMapInfo.getMatchedElements(domainId);
|
||||||
|
convertFilterToSchemaMapInfo(queryFilter.getFilters(), schemaElementMatches);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void convertFilterToSchemaMapInfo(List<Filter> filters, List<SchemaElementMatch> schemaElementMatches) {
|
||||||
|
log.info("schemaElementMatches before queryFilerMapper:{}", schemaElementMatches);
|
||||||
|
if (CollectionUtils.isEmpty(schemaElementMatches)) {
|
||||||
|
schemaElementMatches = Lists.newArrayList();
|
||||||
|
}
|
||||||
|
for (Filter filter : filters) {
|
||||||
|
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
|
||||||
|
.elementType(SchemaElementType.VALUE)
|
||||||
|
.elementID(filter.getElementID().intValue())
|
||||||
|
.frequency(FREQUENCY)
|
||||||
|
.word(String.valueOf(filter.getValue()))
|
||||||
|
.similarity(SIMILARITY)
|
||||||
|
.detectWord(filter.getName())
|
||||||
|
.build();
|
||||||
|
schemaElementMatches.add(schemaElementMatch);
|
||||||
|
}
|
||||||
|
log.info("schemaElementMatches after queryFilerMapper:{}", schemaElementMatches);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -5,42 +5,34 @@ import com.tencent.supersonic.chat.domain.pojo.search.MatchText;
|
|||||||
import com.tencent.supersonic.common.nlp.MapResult;
|
import com.tencent.supersonic.common.nlp.MapResult;
|
||||||
import com.tencent.supersonic.common.nlp.NatureType;
|
import com.tencent.supersonic.common.nlp.NatureType;
|
||||||
import com.tencent.supersonic.knowledge.infrastructure.nlp.Suggester;
|
import com.tencent.supersonic.knowledge.infrastructure.nlp.Suggester;
|
||||||
import org.apache.commons.collections.CollectionUtils;
|
|
||||||
import org.apache.commons.compress.utils.Lists;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
import org.apache.commons.compress.utils.Lists;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* match strategy implement
|
* match strategy implement
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
|
@Slf4j
|
||||||
public class QueryMatchStrategy implements MatchStrategy {
|
public class QueryMatchStrategy implements MatchStrategy {
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(QueryMatchStrategy.class);
|
@Autowired
|
||||||
public static final double STEP = 0.1;
|
private MapperHelper mapperHelper;
|
||||||
@Value("${one.detection.size:6}")
|
|
||||||
private Integer oneDetectionSize;
|
|
||||||
@Value("${one.detection.max.size:20}")
|
|
||||||
private Integer oneDetectionMaxSize;
|
|
||||||
@Value("${metric.dimension.threshold:0.3}")
|
|
||||||
private Double metricDimensionThresholdConfig;
|
|
||||||
@Value("${dimension.value.threshold:0.5}")
|
|
||||||
private Double dimensionValueThresholdConfig;
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<MapResult> match(String text, List<Term> terms, Integer detectDomainId) {
|
public Map<MatchText, List<MapResult>> match(String text, List<Term> terms, Integer detectDomainId) {
|
||||||
if (CollectionUtils.isEmpty(terms) || StringUtils.isEmpty(text)) {
|
if (CollectionUtils.isEmpty(terms) || StringUtils.isEmpty(text)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -50,17 +42,14 @@ public class QueryMatchStrategy implements MatchStrategy {
|
|||||||
List<Integer> offsetList = terms.stream().sorted(Comparator.comparing(Term::getOffset))
|
List<Integer> offsetList = terms.stream().sorted(Comparator.comparing(Term::getOffset))
|
||||||
.map(term -> term.getOffset()).collect(Collectors.toList());
|
.map(term -> term.getOffset()).collect(Collectors.toList());
|
||||||
|
|
||||||
LOGGER.debug("retryCount:{},terms:{},regOffsetToLength:{},offsetList:{},detectDomainId:{}", terms,
|
log.debug("retryCount:{},terms:{},regOffsetToLength:{},offsetList:{},detectDomainId:{}", terms,
|
||||||
regOffsetToLength, offsetList, detectDomainId);
|
regOffsetToLength, offsetList, detectDomainId);
|
||||||
|
|
||||||
return detect(text, regOffsetToLength, offsetList, detectDomainId);
|
List<MapResult> detects = detect(text, regOffsetToLength, offsetList, detectDomainId);
|
||||||
}
|
Map<MatchText, List<MapResult>> result = new HashMap<>();
|
||||||
|
MatchText matchText = new MatchText(text, text);
|
||||||
@Override
|
result.put(matchText, detects);
|
||||||
public Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals,
|
return result;
|
||||||
Integer detectDomainId) {
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<MapResult> detect(String text, Map<Integer, Integer> regOffsetToLength, List<Integer> offsetList,
|
private List<MapResult> detect(String text, Map<Integer, Integer> regOffsetToLength, List<Integer> offsetList,
|
||||||
@@ -72,15 +61,15 @@ public class QueryMatchStrategy implements MatchStrategy {
|
|||||||
Set<MapResult> mapResultRowSet = new LinkedHashSet();
|
Set<MapResult> mapResultRowSet = new LinkedHashSet();
|
||||||
|
|
||||||
for (Integer i = index; i <= text.length(); ) {
|
for (Integer i = index; i <= text.length(); ) {
|
||||||
int offset = getStepOffset(offsetList, index);
|
int offset = mapperHelper.getStepOffset(offsetList, index);
|
||||||
i = getStepIndex(regOffsetToLength, i);
|
i = mapperHelper.getStepIndex(regOffsetToLength, i);
|
||||||
if (i <= text.length()) {
|
if (i <= text.length()) {
|
||||||
List<MapResult> mapResults = detectByStep(text, detectDomainId, index, i, offset);
|
List<MapResult> mapResults = detectByStep(text, detectDomainId, index, i, offset);
|
||||||
mapResultRowSet.addAll(mapResults);
|
mapResultRowSet.addAll(mapResults);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
index = getStepIndex(regOffsetToLength, index);
|
index = mapperHelper.getStepIndex(regOffsetToLength, index);
|
||||||
results.addAll(mapResultRowSet);
|
results.addAll(mapResultRowSet);
|
||||||
}
|
}
|
||||||
return results;
|
return results;
|
||||||
@@ -88,11 +77,13 @@ public class QueryMatchStrategy implements MatchStrategy {
|
|||||||
|
|
||||||
private List<MapResult> detectByStep(String text, Integer detectDomainId, Integer index, Integer i, int offset) {
|
private List<MapResult> detectByStep(String text, Integer detectDomainId, Integer index, Integer i, int offset) {
|
||||||
String detectSegment = text.substring(index, i);
|
String detectSegment = text.substring(index, i);
|
||||||
|
Integer oneDetectionSize = mapperHelper.getOneDetectionSize();
|
||||||
// step1. pre search
|
// step1. pre search
|
||||||
LinkedHashSet<MapResult> mapResults = Suggester.prefixSearch(detectSegment, oneDetectionMaxSize)
|
LinkedHashSet<MapResult> mapResults = Suggester.prefixSearch(detectSegment,
|
||||||
|
mapperHelper.getOneDetectionMaxSize())
|
||||||
.stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
.stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
// step2. suffix search
|
// step2. suffix search
|
||||||
LinkedHashSet<MapResult> suffixMapResults = Suggester.suffixSearch(detectSegment, oneDetectionMaxSize)
|
LinkedHashSet<MapResult> suffixMapResults = Suggester.suffixSearch(detectSegment, oneDetectionSize)
|
||||||
.stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
.stream().collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
|
|
||||||
mapResults.addAll(suffixMapResults);
|
mapResults.addAll(suffixMapResults);
|
||||||
@@ -105,7 +96,7 @@ public class QueryMatchStrategy implements MatchStrategy {
|
|||||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
// step4. filter by classId
|
// step4. filter by classId
|
||||||
if (Objects.nonNull(detectDomainId) && detectDomainId > 0) {
|
if (Objects.nonNull(detectDomainId) && detectDomainId > 0) {
|
||||||
LOGGER.debug("detectDomainId:{}, before parseResults:{}", mapResults);
|
log.debug("detectDomainId:{}, before parseResults:{}", mapResults);
|
||||||
mapResults = mapResults.stream().map(entry -> {
|
mapResults = mapResults.stream().map(entry -> {
|
||||||
List<String> natures = entry.getNatures().stream().filter(
|
List<String> natures = entry.getNatures().stream().filter(
|
||||||
nature -> nature.startsWith(NatureType.NATURE_SPILT + detectDomainId) || (nature.startsWith(
|
nature -> nature.startsWith(NatureType.NATURE_SPILT + detectDomainId) || (nature.startsWith(
|
||||||
@@ -114,25 +105,27 @@ public class QueryMatchStrategy implements MatchStrategy {
|
|||||||
entry.setNatures(natures);
|
entry.setNatures(natures);
|
||||||
return entry;
|
return entry;
|
||||||
}).collect(Collectors.toCollection(LinkedHashSet::new));
|
}).collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
LOGGER.info("after domainId parseResults:{}", mapResults);
|
log.info("after domainId parseResults:{}", mapResults);
|
||||||
}
|
}
|
||||||
// step5. filter by similarity
|
// step5. filter by similarity
|
||||||
mapResults = mapResults.stream()
|
mapResults = mapResults.stream()
|
||||||
.filter(term -> getSimilarity(detectSegment, term.getName()) >= getThresholdMatch(term.getNatures()))
|
.filter(term -> mapperHelper.getSimilarity(detectSegment, term.getName())
|
||||||
|
>= mapperHelper.getThresholdMatch(term.getNatures()))
|
||||||
.filter(term -> CollectionUtils.isNotEmpty(term.getNatures()))
|
.filter(term -> CollectionUtils.isNotEmpty(term.getNatures()))
|
||||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
|
|
||||||
LOGGER.debug("metricDimensionThreshold:{},dimensionValueThreshold:{},after isSimilarity parseResults:{}",
|
log.debug("metricDimensionThreshold:{},dimensionValueThreshold:{},after isSimilarity parseResults:{}",
|
||||||
mapResults);
|
mapResults);
|
||||||
|
|
||||||
mapResults = mapResults.stream().map(parseResult -> {
|
mapResults = mapResults.stream().map(parseResult -> {
|
||||||
parseResult.setOffset(offset);
|
parseResult.setOffset(offset);
|
||||||
parseResult.setSimilarity(getSimilarity(detectSegment, parseResult.getName()));
|
parseResult.setSimilarity(mapperHelper.getSimilarity(detectSegment, parseResult.getName()));
|
||||||
return parseResult;
|
return parseResult;
|
||||||
}).collect(Collectors.toCollection(LinkedHashSet::new));
|
}).collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
|
|
||||||
// step6. take only one dimension or 10 metric/dimension value per rond.
|
// step6. take only one dimension or 10 metric/dimension value per rond.
|
||||||
List<MapResult> dimensionMetrics = mapResults.stream().filter(entry -> existDimensionValues(entry.getNatures()))
|
List<MapResult> dimensionMetrics = mapResults.stream()
|
||||||
|
.filter(entry -> mapperHelper.existDimensionValues(entry.getNatures()))
|
||||||
.collect(Collectors.toList())
|
.collect(Collectors.toList())
|
||||||
.stream()
|
.stream()
|
||||||
.limit(1)
|
.limit(1)
|
||||||
@@ -144,31 +137,4 @@ public class QueryMatchStrategy implements MatchStrategy {
|
|||||||
return mapResults.stream().limit(oneDetectionSize).collect(Collectors.toList());
|
return mapResults.stream().limit(oneDetectionSize).collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Integer getStepIndex(Map<Integer, Integer> regOffsetToLength, Integer index) {
|
|
||||||
Integer subRegLength = regOffsetToLength.get(index);
|
|
||||||
if (Objects.nonNull(subRegLength)) {
|
|
||||||
index = index + subRegLength;
|
|
||||||
} else {
|
|
||||||
index++;
|
|
||||||
}
|
|
||||||
return index;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Integer getStepOffset(List<Integer> termList, Integer index) {
|
|
||||||
for (int j = 0; j < termList.size() - 1; j++) {
|
|
||||||
if (termList.get(j) <= index && termList.get(j + 1) > index) {
|
|
||||||
return termList.get(j);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return index;
|
|
||||||
}
|
|
||||||
|
|
||||||
private double getThresholdMatch(List<String> natures) {
|
|
||||||
if (existDimensionValues(natures)) {
|
|
||||||
return dimensionValueThresholdConfig;
|
|
||||||
}
|
|
||||||
return metricDimensionThresholdConfig;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -23,15 +23,8 @@ public class SearchMatchStrategy implements MatchStrategy {
|
|||||||
|
|
||||||
private static final int SEARCH_SIZE = 3;
|
private static final int SEARCH_SIZE = 3;
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<MapResult> match(String text, List<Term> terms, Integer detectDomainId) {
|
public Map<MatchText, List<MapResult>> match(String text, List<Term> originals,
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Map<MatchText, List<MapResult>> matchWithMatchText(String text, List<Term> originals,
|
|
||||||
Integer detectDomainId) {
|
Integer detectDomainId) {
|
||||||
|
|
||||||
Map<Integer, Integer> regOffsetToLength = originals.stream()
|
Map<Integer, Integer> regOffsetToLength = originals.stream()
|
||||||
|
|||||||
@@ -1,59 +1,82 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser;
|
package com.tencent.supersonic.chat.application.parser;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticParser;
|
import com.tencent.supersonic.chat.application.query.EntityListFilter;
|
||||||
import com.tencent.supersonic.chat.application.parser.resolver.AggregateTypeResolver;
|
|
||||||
import com.tencent.supersonic.chat.application.query.MetricCompare;
|
|
||||||
import com.tencent.supersonic.chat.application.query.MetricFilter;
|
|
||||||
import com.tencent.supersonic.chat.application.query.MetricGroupBy;
|
import com.tencent.supersonic.chat.application.query.MetricGroupBy;
|
||||||
import com.tencent.supersonic.chat.application.query.MetricOrderBy;
|
import com.tencent.supersonic.chat.application.query.MetricOrderBy;
|
||||||
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
||||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
import java.util.HashMap;
|
||||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
import java.util.Map;
|
||||||
import java.util.List;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.regex.Matcher;
|
||||||
import org.slf4j.Logger;
|
import java.util.regex.Pattern;
|
||||||
import org.slf4j.LoggerFactory;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
@Component
|
@Slf4j
|
||||||
public class AggregateSemanticParser implements SemanticParser {
|
public class AggregateSemanticParser implements SemanticParser {
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(AggregateSemanticParser.class);
|
|
||||||
public static final Integer TOPN_LIMIT = 1000;
|
public static final Integer TOPN_LIMIT = 1000;
|
||||||
|
|
||||||
private AggregateTypeResolver aggregateTypeResolver;
|
private static Map<AggregateTypeEnum, Pattern> aggregateRegexMap = new HashMap<>();
|
||||||
|
|
||||||
|
static {
|
||||||
|
aggregateRegexMap.put(AggregateTypeEnum.MAX, Pattern.compile("(?i)(最大值|最大|max|峰值|最高|最多)"));
|
||||||
|
aggregateRegexMap.put(AggregateTypeEnum.MIN, Pattern.compile("(?i)(最小值|最小|min|最低|最少)"));
|
||||||
|
aggregateRegexMap.put(AggregateTypeEnum.SUM, Pattern.compile("(?i)(汇总|总和|sum)"));
|
||||||
|
aggregateRegexMap.put(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|日均|平均|avg)"));
|
||||||
|
aggregateRegexMap.put(AggregateTypeEnum.TOPN, Pattern.compile("(?i)(top)"));
|
||||||
|
aggregateRegexMap.put(AggregateTypeEnum.DISTINCT, Pattern.compile("(?i)(uv)"));
|
||||||
|
aggregateRegexMap.put(AggregateTypeEnum.COUNT, Pattern.compile("(?i)(总数|pv)"));
|
||||||
|
aggregateRegexMap.put(AggregateTypeEnum.NONE, Pattern.compile("(?i)(明细)"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static AggregateTypeEnum resolveAggregateType(String queryText) {
|
||||||
|
|
||||||
|
Map<AggregateTypeEnum, Integer> aggregateCount = new HashMap<>(aggregateRegexMap.size());
|
||||||
|
for (Map.Entry<AggregateTypeEnum, Pattern> entry : aggregateRegexMap.entrySet()) {
|
||||||
|
Matcher matcher = entry.getValue().matcher(queryText);
|
||||||
|
int count = 0;
|
||||||
|
while (matcher.find()) {
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
if (count > 0) {
|
||||||
|
aggregateCount.put(entry.getKey(), count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return aggregateCount.entrySet().stream().max(Map.Entry.comparingByValue()).map(entry -> entry.getKey())
|
||||||
|
.orElse(null);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean parse(QueryContextReq queryContext, ChatContext chatCtx) {
|
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
|
||||||
aggregateTypeResolver = ContextUtils.getBean(AggregateTypeResolver.class);
|
AggregateTypeEnum aggregateType = resolveAggregateType(queryContext.getQueryText());
|
||||||
|
|
||||||
AggregateTypeEnum aggregateType = aggregateTypeResolver.resolve(queryContext.getQueryText());
|
for (SemanticQuery semanticQuery : queryContext.getCandidateQueries()) {
|
||||||
|
SemanticParseInfo semanticParse = semanticQuery.getParseInfo();
|
||||||
|
|
||||||
SemanticParseInfo semanticParse = queryContext.getParseInfo();
|
semanticParse.setNativeQuery(getNativeQuery(aggregateType, semanticParse));
|
||||||
|
semanticParse.setAggType(aggregateType);
|
||||||
Set<SchemaItem> metrics = semanticParse.getMetrics();
|
if (Objects.isNull(semanticParse.getLimit()) || semanticParse.getLimit() <= 0) {
|
||||||
|
semanticParse.setLimit(Long.valueOf(TOPN_LIMIT));
|
||||||
semanticParse.setNativeQuery(getNativeQuery(aggregateType, queryContext));
|
}
|
||||||
|
resetQueryModeByAggregateType(semanticParse, aggregateType);
|
||||||
semanticParse.setAggType(aggregateType);
|
}
|
||||||
//semanticParse.setOrders(getOrder(aggregateType, metrics));
|
|
||||||
semanticParse.setLimit(Long.valueOf(TOPN_LIMIT));
|
|
||||||
resetQueryModeByAggregateType(queryContext, aggregateType);
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* query mode reset by the AggregateType
|
* query mode reset by the AggregateType
|
||||||
*
|
*
|
||||||
* @param queryContext
|
* @param parseInfo
|
||||||
* @param aggregateType
|
* @param aggregateType
|
||||||
*/
|
*/
|
||||||
private void resetQueryModeByAggregateType(QueryContextReq queryContext, AggregateTypeEnum aggregateType) {
|
private void resetQueryModeByAggregateType(SemanticParseInfo parseInfo,
|
||||||
|
AggregateTypeEnum aggregateType) {
|
||||||
|
|
||||||
SemanticParseInfo parseInfo = queryContext.getParseInfo();
|
|
||||||
String queryMode = parseInfo.getQueryMode();
|
String queryMode = parseInfo.getQueryMode();
|
||||||
if (MetricGroupBy.QUERY_MODE.equals(queryMode) || MetricGroupBy.QUERY_MODE.equals(queryMode)) {
|
if (MetricGroupBy.QUERY_MODE.equals(queryMode) || MetricGroupBy.QUERY_MODE.equals(queryMode)) {
|
||||||
if (AggregateTypeEnum.MAX.equals(aggregateType) || AggregateTypeEnum.MIN.equals(aggregateType)
|
if (AggregateTypeEnum.MAX.equals(aggregateType) || AggregateTypeEnum.MIN.equals(aggregateType)
|
||||||
@@ -62,23 +85,18 @@ public class AggregateSemanticParser implements SemanticParser {
|
|||||||
} else {
|
} else {
|
||||||
parseInfo.setQueryMode(MetricGroupBy.QUERY_MODE);
|
parseInfo.setQueryMode(MetricGroupBy.QUERY_MODE);
|
||||||
}
|
}
|
||||||
|
log.info("queryMode mode [{}]->[{}]", queryMode, parseInfo.getQueryMode());
|
||||||
}
|
}
|
||||||
if (MetricFilter.QUERY_MODE.equals(queryMode) || MetricCompare.QUERY_MODE.equals(queryMode)) {
|
|
||||||
if (aggregateTypeResolver.hasCompareIntentionalWords(queryContext.getQueryText())) {
|
|
||||||
parseInfo.setQueryMode(MetricCompare.QUERY_MODE);
|
|
||||||
} else {
|
|
||||||
parseInfo.setQueryMode(MetricFilter.QUERY_MODE);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
logger.info("queryMode mode [{}]->[{}]", queryMode, parseInfo.getQueryMode());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean getNativeQuery(AggregateTypeEnum aggregateType, QueryContextReq queryContext) {
|
private boolean getNativeQuery(AggregateTypeEnum aggregateType, SemanticParseInfo semanticParse) {
|
||||||
if (AggregateTypeEnum.TOPN.equals(aggregateType)) {
|
if (AggregateTypeEnum.TOPN.equals(aggregateType)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return queryContext.getParseInfo().getNativeQuery();
|
if (EntityListFilter.QUERY_MODE.equals(semanticParse.getQueryMode()) && (semanticParse.getMetrics() == null
|
||||||
|
|| semanticParse.getMetrics().isEmpty())) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return semanticParse.getNativeQuery();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser.resolver;
|
package com.tencent.supersonic.chat.application.parser;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
public interface DomainResolver {
|
public interface DomainResolver {
|
||||||
@@ -12,6 +14,6 @@ public interface DomainResolver {
|
|||||||
Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq queryCtx, ChatContext chatCtx,
|
Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq queryCtx, ChatContext chatCtx,
|
||||||
SchemaMapInfo schemaMap);
|
SchemaMapInfo schemaMap);
|
||||||
|
|
||||||
boolean isDomainSwitch(ChatContext chatCtx, QueryContextReq queryCtx);
|
boolean isDomainSwitch(ChatContext chatCtx, SemanticParseInfo semanticParseInfo);
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -1,125 +1,111 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser;
|
package com.tencent.supersonic.chat.application.parser;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.*;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
import com.tencent.supersonic.chat.application.query.*;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticParser;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.chat.application.parser.resolver.DomainResolver;
|
|
||||||
import com.tencent.supersonic.chat.application.parser.resolver.SemanticQueryResolver;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
||||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
import com.tencent.supersonic.chat.domain.utils.*;
|
||||||
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
import java.util.*;
|
||||||
import java.util.HashMap;
|
import java.util.function.Function;
|
||||||
import java.util.LinkedList;
|
import java.util.stream.Collectors;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||||
import java.util.Objects;
|
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
||||||
import java.util.Optional;
|
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||||
import org.slf4j.Logger;
|
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||||
import org.slf4j.LoggerFactory;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.core.io.support.SpringFactoriesLoader;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@Component
|
@Slf4j
|
||||||
public class DomainSemanticParser implements SemanticParser {
|
public class DomainSemanticParser implements SemanticParser {
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(DomainSemanticParser.class);
|
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
private List<DomainResolver> domainResolverList;
|
|
||||||
|
|
||||||
private SemanticQueryResolver semanticQueryResolver;
|
|
||||||
|
|
||||||
public DomainSemanticParser() {
|
|
||||||
domainResolverList = SpringFactoriesLoader.loadFactories(DomainResolver.class,
|
|
||||||
Thread.currentThread().getContextClassLoader());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean parse(QueryContextReq queryContext, ChatContext chatCtx) {
|
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
|
||||||
DomainInfos domainInfosDb = SchemaInfoConverter.convert(
|
DomainInfos domainInfosDb = SchemaInfoConverter.convert(semanticLayer.getDomainSchemaInfo(new ArrayList<>()));
|
||||||
ContextUtils.getBean(SemanticLayer.class).getDomainSchemaInfo(new ArrayList<>()));
|
|
||||||
Map<Integer, String> domainToName = domainInfosDb.getDomainToName();
|
Map<Integer, String> domainToName = domainInfosDb.getDomainToName();
|
||||||
|
|
||||||
SchemaMapInfo mapInfo = queryContext.getMapInfo();
|
SchemaMapInfo mapInfo = queryContext.getMapInfo();
|
||||||
SemanticParseInfo parseInfo = queryContext.getParseInfo();
|
|
||||||
|
|
||||||
//domainResolver = ContextUtils.getBean(DomainResolver.class);
|
// iterate all schemaElementMatches to resolve semantic query
|
||||||
semanticQueryResolver = ContextUtils.getBean(SemanticQueryResolver.class);
|
for (Integer domainId : mapInfo.getMatchedDomains()) {
|
||||||
|
List<SchemaElementMatch> elementMatches = mapInfo.getMatchedElements(domainId);
|
||||||
Map<Integer, SemanticQuery> domainSemanticQuery = new HashMap<>();
|
Map<RuleSemanticQuery, List<SchemaElementMatch>> queryMatches = resolveQuery(elementMatches, queryContext);
|
||||||
// Round 1: find all domains that can be resolved to any query mode
|
for (Map.Entry<RuleSemanticQuery, List<SchemaElementMatch>> match : queryMatches.entrySet()) {
|
||||||
|
addCandidateQuery(queryContext, chatContext, domainId.longValue(),
|
||||||
for (Integer domain : mapInfo.getMatchedDomains()) {
|
domainToName.get(domainId), match.getKey(), match.getValue());
|
||||||
List<SchemaElementMatch> elementMatches = mapInfo.getMatchedElements(domain);
|
|
||||||
|
|
||||||
SemanticQuery query = semanticQueryResolver.resolve(elementMatches, queryContext);
|
|
||||||
|
|
||||||
if (Objects.nonNull(query)) {
|
|
||||||
domainSemanticQuery.put(domain, query);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// only one domain is found, no need to rank
|
|
||||||
if (domainSemanticQuery.size() == 1) {
|
// if no candidates have been found yet, count in chat context and try again
|
||||||
Optional<Map.Entry<Integer, SemanticQuery>> match = domainSemanticQuery.entrySet().stream().findFirst();
|
if (queryContext.getCandidateQueries().size() <= 0) {
|
||||||
if (match.isPresent()) {
|
if (chatContext.getParseInfo() != null && chatContext.getParseInfo().getDomainId() > 0) {
|
||||||
logger.info("select by only one [{}:{}]", match.get().getKey(), match.get().getValue());
|
Integer chatDomainId = Integer.valueOf(chatContext.getParseInfo().getDomainId().intValue());
|
||||||
parseInfo.setDomainId(Long.valueOf(match.get().getKey()));
|
if (mapInfo.getMatchedDomains().contains(chatDomainId)) {
|
||||||
parseInfo.setDomainName(domainToName.get(Integer.valueOf(match.get().getKey())));
|
List<SchemaElementMatch> elementMatches = mapInfo.getMatchedElements(chatDomainId);
|
||||||
parseInfo.setQueryMode(match.get().getValue().getQueryMode());
|
detectionContext(chatContext);
|
||||||
return false;
|
Map<RuleSemanticQuery, List<SchemaElementMatch>> queryMatches = tryParseByContext(elementMatches,
|
||||||
}
|
chatContext, queryContext);
|
||||||
} else if (domainSemanticQuery.size() > 1) {
|
for (Map.Entry<RuleSemanticQuery, List<SchemaElementMatch>> match : queryMatches.entrySet()) {
|
||||||
// will choose one by the domain select
|
addCandidateQuery(queryContext, chatContext, chatDomainId.longValue(),
|
||||||
Optional<Integer> domainId = domainResolverList.stream()
|
domainToName.get(chatDomainId), match.getKey(), match.getValue());
|
||||||
.map(domainResolver -> domainResolver.resolve(domainSemanticQuery, queryContext, chatCtx, mapInfo))
|
|
||||||
.filter(d -> d > 0).findFirst();
|
|
||||||
if (domainId.isPresent() && domainId.get() > 0) {
|
|
||||||
for (Map.Entry<Integer, SemanticQuery> match : domainSemanticQuery.entrySet()) {
|
|
||||||
if (match.getKey().equals(domainId.get())) {
|
|
||||||
logger.info("select by selectStrategy [{}:{}]", domainId.get(), match.getValue());
|
|
||||||
parseInfo.setDomainId(Long.valueOf(match.getKey()));
|
|
||||||
parseInfo.setDomainName(domainToName.get(Integer.valueOf(match.getKey())));
|
|
||||||
parseInfo.setQueryMode(match.getValue().getQueryMode());
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Round 2: no domains can be found yet, count in chat context
|
|
||||||
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getDomainId() > 0) {
|
|
||||||
Integer chatDomain = Integer.valueOf(chatCtx.getParseInfo().getDomainId().intValue());
|
|
||||||
if (mapInfo.getMatchedDomains().contains(chatDomain) || CollectionUtils.isEmpty(
|
|
||||||
mapInfo.getMatchedDomains())) {
|
|
||||||
List<SchemaElementMatch> elementMatches = mapInfo.getMatchedElements(chatDomain);
|
|
||||||
if (CollectionUtils.isEmpty(elementMatches)) {
|
|
||||||
parseInfo.setDomainId(Long.valueOf(chatDomain));
|
|
||||||
parseInfo.setDomainName(domainToName.get(chatDomain));
|
|
||||||
parseInfo.setQueryMode(chatCtx.getParseInfo().getQueryMode());
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
SemanticQuery query = tryParseByContext(elementMatches, chatCtx, queryContext);
|
|
||||||
if (Objects.nonNull(query)) {
|
|
||||||
logger.info("select by context count [{}:{}]", chatDomain, query);
|
|
||||||
parseInfo.setDomainId(Long.valueOf(chatDomain));
|
|
||||||
parseInfo.setDomainName(domainToName.get(chatDomain));
|
|
||||||
parseInfo.setQueryMode(query.getQueryMode());
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Round 3: no domains can be found yet, count in default metric
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void addCandidateQuery(QueryContextReq queryContext, ChatContext chatContext,
|
||||||
|
Long domainId, String domainName,
|
||||||
|
RuleSemanticQuery semanticQuery, List<SchemaElementMatch> elementMatches) {
|
||||||
|
if (semanticQuery != null) {
|
||||||
|
fillParseInfo(semanticQuery, domainId, domainName, elementMatches);
|
||||||
|
// inherit from context
|
||||||
|
inheritContext(semanticQuery, chatContext);
|
||||||
|
// default metric, date, dimension
|
||||||
|
injectDefaultMetric(semanticQuery, queryContext, chatContext);
|
||||||
|
queryContext.getCandidateQueries().add(semanticQuery);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void inheritContext(RuleSemanticQuery semanticQuery, ChatContext chatContext) {
|
||||||
|
// is domain switch
|
||||||
|
SemanticParseInfo semanticParse = semanticQuery.getParseInfo();
|
||||||
|
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
|
||||||
|
if (!domainResolver.isDomainSwitch(chatContext, semanticParse)) {
|
||||||
|
semanticQuery.inheritContext(chatContext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void injectDefaultMetric(RuleSemanticQuery semanticQuery, QueryContextReq queryContext,
|
||||||
|
ChatContext chatContext) {
|
||||||
|
DefaultMetricUtils defaultMetricUtils = ContextUtils.getBean(DefaultMetricUtils.class);
|
||||||
|
defaultMetricUtils.injectDefaultMetric(semanticQuery.getParseInfo(), queryContext, chatContext);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get the chatContext for the tryParseByContext
|
||||||
|
*
|
||||||
|
* @param chatContext
|
||||||
|
*/
|
||||||
|
protected void detectionContext(ChatContext chatContext) {
|
||||||
|
if (chatContext.getParseInfo() != null) {
|
||||||
|
SemanticParseInfo semanticParseInfo = chatContext.getParseInfo();
|
||||||
|
if (semanticParseInfo.getQueryMode().equals(EntityDetail.QUERY_MODE)) {
|
||||||
|
// EntityDetail model will unset some items
|
||||||
|
semanticParseInfo.setDateInfo(null);
|
||||||
|
semanticParseInfo.setMetrics(new HashSet<>());
|
||||||
|
semanticParseInfo.setDimensions(new HashSet<>());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* try to add ChatContext to SchemaElementMatch and look if match QueryMode
|
* try to add ChatContext to SchemaElementMatch and look if match QueryMode
|
||||||
@@ -128,8 +114,8 @@ public class DomainSemanticParser implements SemanticParser {
|
|||||||
* @param chatCtx
|
* @param chatCtx
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
private SemanticQuery tryParseByContext(List<SchemaElementMatch> elementMatches, ChatContext chatCtx,
|
private Map<RuleSemanticQuery, List<SchemaElementMatch>> tryParseByContext(List<SchemaElementMatch> elementMatches,
|
||||||
QueryContextReq searchCt) {
|
ChatContext chatCtx, QueryContextReq queryCtx) {
|
||||||
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getEntity() > 0) {
|
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getEntity() > 0) {
|
||||||
Long entityCount = elementMatches.stream().filter(i -> SchemaElementType.ENTITY.equals(i.getElementType()))
|
Long entityCount = elementMatches.stream().filter(i -> SchemaElementType.ENTITY.equals(i.getElementType()))
|
||||||
.count();
|
.count();
|
||||||
@@ -137,27 +123,29 @@ public class DomainSemanticParser implements SemanticParser {
|
|||||||
.count();
|
.count();
|
||||||
if (entityCount <= 0 && metricCount <= 0 && ContextHelper.hasEntityId(chatCtx)) {
|
if (entityCount <= 0 && metricCount <= 0 && ContextHelper.hasEntityId(chatCtx)) {
|
||||||
// try entity parse
|
// try entity parse
|
||||||
SchemaElementMatch entityElementMatch = new SchemaElementMatch();
|
SchemaElementMatch entityElementMatch = SchemaElementMatch.builder()
|
||||||
entityElementMatch.setElementType(SchemaElementType.ENTITY);
|
.elementType(SchemaElementType.ENTITY).build();
|
||||||
List<SchemaElementMatch> newSchemaElementMatch = new ArrayList<>();
|
List<SchemaElementMatch> newSchemaElementMatch = new ArrayList<>();
|
||||||
if (!CollectionUtils.isEmpty(elementMatches)) {
|
if (!CollectionUtils.isEmpty(elementMatches)) {
|
||||||
newSchemaElementMatch.addAll(elementMatches);
|
newSchemaElementMatch.addAll(elementMatches);
|
||||||
}
|
}
|
||||||
newSchemaElementMatch.add(entityElementMatch);
|
newSchemaElementMatch.add(entityElementMatch);
|
||||||
SemanticQuery semanticQuery = doParseByContext(newSchemaElementMatch, chatCtx, searchCt);
|
Map<RuleSemanticQuery, List<SchemaElementMatch>> queryMatches = doParseByContext(newSchemaElementMatch,
|
||||||
if (Objects.nonNull(semanticQuery)) {
|
chatCtx, queryCtx);
|
||||||
return semanticQuery;
|
if (queryMatches.size() > 0) {
|
||||||
|
return queryMatches;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return doParseByContext(elementMatches, chatCtx, searchCt);
|
return doParseByContext(elementMatches, chatCtx, queryCtx);
|
||||||
}
|
}
|
||||||
|
|
||||||
private SemanticQuery doParseByContext(List<SchemaElementMatch> elementMatches, ChatContext chatCtx,
|
|
||||||
QueryContextReq searchCt) {
|
private Map<RuleSemanticQuery, List<SchemaElementMatch>> doParseByContext(List<SchemaElementMatch> elementMatches,
|
||||||
SemanticParseInfo contextSemanticParseInfo = chatCtx.getParseInfo();
|
ChatContext chatCtx, QueryContextReq queryCtx) {
|
||||||
if (contextSemanticParseInfo != null) {
|
SemanticParseInfo contextSemanticParse = chatCtx.getParseInfo();
|
||||||
List<SchemaElementMatch> newSchemaElementMatch = new ArrayList<>();
|
if (contextSemanticParse != null) {
|
||||||
|
List<SchemaElementMatch> newElementMatches = new ArrayList<>();
|
||||||
List<List<SchemaElementType>> trySchemaElementTypes = new LinkedList<>();
|
List<List<SchemaElementType>> trySchemaElementTypes = new LinkedList<>();
|
||||||
// try DIMENSION+METRIC+VALUE
|
// try DIMENSION+METRIC+VALUE
|
||||||
// try DIMENSION+METRIC METRIC+VALUE DIMENSION+VALUE
|
// try DIMENSION+METRIC METRIC+VALUE DIMENSION+VALUE
|
||||||
@@ -175,18 +163,122 @@ public class DomainSemanticParser implements SemanticParser {
|
|||||||
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION)));
|
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION)));
|
||||||
|
|
||||||
for (List<SchemaElementType> schemaElementTypes : trySchemaElementTypes) {
|
for (List<SchemaElementType> schemaElementTypes : trySchemaElementTypes) {
|
||||||
newSchemaElementMatch.clear();
|
newElementMatches.clear();
|
||||||
if (!CollectionUtils.isEmpty(elementMatches)) {
|
if (!CollectionUtils.isEmpty(elementMatches)) {
|
||||||
newSchemaElementMatch.addAll(elementMatches);
|
newElementMatches.addAll(elementMatches);
|
||||||
}
|
}
|
||||||
ContextHelper.mergeContextSchemaElementMatch(newSchemaElementMatch, elementMatches, schemaElementTypes,
|
ContextHelper.mergeContextSchemaElementMatch(newElementMatches, elementMatches, schemaElementTypes,
|
||||||
contextSemanticParseInfo);
|
contextSemanticParse);
|
||||||
SemanticQuery semanticQuery = semanticQueryResolver.resolve(newSchemaElementMatch, searchCt);
|
Map<RuleSemanticQuery, List<SchemaElementMatch>> queryMatches = resolveQuery(newElementMatches,
|
||||||
if (semanticQuery != null) {
|
queryCtx);
|
||||||
return semanticQuery;
|
if (queryMatches.size() > 0) {
|
||||||
|
return queryMatches;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return new HashMap<>();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
private Map<RuleSemanticQuery, List<SchemaElementMatch>> resolveQuery(List<SchemaElementMatch> elementMatches,
|
||||||
|
QueryContextReq queryCtx) {
|
||||||
|
Map<RuleSemanticQuery, List<SchemaElementMatch>> matchMap = new HashMap<>();
|
||||||
|
|
||||||
|
for (RuleSemanticQuery semanticQuery : RuleSemanticQueryManager.getSemanticQueries()) {
|
||||||
|
List<SchemaElementMatch> matches = semanticQuery.match(elementMatches, queryCtx);
|
||||||
|
|
||||||
|
if (matches.size() > 0) {
|
||||||
|
log.info("resolve match [{}:{}] ", semanticQuery.getQueryMode(), matches.size());
|
||||||
|
matchMap.put(RuleSemanticQueryManager.create(semanticQuery.getQueryMode()), matches);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return matchMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void fillParseInfo(SemanticQuery query, Long domainId, String domainName,
|
||||||
|
List<SchemaElementMatch> elementMatches) {
|
||||||
|
SemanticParseInfo parseInfo = query.getParseInfo();
|
||||||
|
parseInfo.setDomainId(domainId);
|
||||||
|
parseInfo.setDomainName(domainName);
|
||||||
|
parseInfo.setQueryMode(query.getQueryMode());
|
||||||
|
parseInfo.getElementMatches().addAll(elementMatches);
|
||||||
|
|
||||||
|
DefaultSemanticInternalUtils defaultSemanticUtils = ContextUtils.getBean(DefaultSemanticInternalUtils.class);
|
||||||
|
SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
|
|
||||||
|
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(parseInfo.getDomainId());
|
||||||
|
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(parseInfo.getDomainId());
|
||||||
|
Map<Long, DimSchemaResp> dimensionDescMap = domainSchemaDesc.getDimensions().stream()
|
||||||
|
.collect(Collectors.toMap(DimSchemaResp::getId, Function.identity()));
|
||||||
|
Map<Long, MetricSchemaResp> metricDescMap = domainSchemaDesc.getMetrics().stream()
|
||||||
|
.collect(Collectors.toMap(MetricSchemaResp::getId, Function.identity()));
|
||||||
|
Map<Long, List<SchemaElementMatch>> dim2Values = new HashMap<>();
|
||||||
|
|
||||||
|
for (SchemaElementMatch schemaElementMatch : elementMatches) {
|
||||||
|
Long elementID = Long.valueOf(schemaElementMatch.getElementID());
|
||||||
|
switch (schemaElementMatch.getElementType()) {
|
||||||
|
case ID:
|
||||||
|
case VALUE:
|
||||||
|
if (dimensionDescMap.containsKey(elementID)) {
|
||||||
|
if (dim2Values.containsKey(elementID)) {
|
||||||
|
dim2Values.get(elementID).add(schemaElementMatch);
|
||||||
|
} else {
|
||||||
|
dim2Values.put(elementID, new ArrayList<>(Arrays.asList(schemaElementMatch)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case DIMENSION:
|
||||||
|
DimSchemaResp dimensionDesc = dimensionDescMap.get(elementID);
|
||||||
|
if (dimensionDesc != null) {
|
||||||
|
SchemaItem dimensionParseInfo = new SchemaItem();
|
||||||
|
dimensionParseInfo.setBizName(dimensionDesc.getBizName());
|
||||||
|
dimensionParseInfo.setName(dimensionDesc.getName());
|
||||||
|
dimensionParseInfo.setId(dimensionDesc.getId());
|
||||||
|
parseInfo.getDimensions().add(dimensionParseInfo);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case METRIC:
|
||||||
|
MetricSchemaResp metricDesc = metricDescMap.get(elementID);
|
||||||
|
if (metricDesc != null) {
|
||||||
|
SchemaItem metricItem = new SchemaItem();
|
||||||
|
metricItem.setBizName(metricDesc.getBizName());
|
||||||
|
metricItem.setName(metricDesc.getName());
|
||||||
|
metricItem.setId(metricDesc.getId());
|
||||||
|
metricItem.setCreatedAt(null);
|
||||||
|
metricItem.setUpdatedAt(null);
|
||||||
|
parseInfo.getMetrics().add(metricItem);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!dim2Values.isEmpty()) {
|
||||||
|
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dim2Values.entrySet()) {
|
||||||
|
DimSchemaResp dimensionDesc = dimensionDescMap.get(entry.getKey());
|
||||||
|
if (entry.getValue().size() == 1) {
|
||||||
|
SchemaElementMatch schemaElementMatch = entry.getValue().get(0);
|
||||||
|
Filter dimensionFilter = new Filter();
|
||||||
|
dimensionFilter.setValue(schemaElementMatch.getWord());
|
||||||
|
dimensionFilter.setBizName(dimensionDesc.getBizName());
|
||||||
|
dimensionFilter.setName(dimensionDesc.getName());
|
||||||
|
dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
|
||||||
|
dimensionFilter.setElementID(Long.valueOf(schemaElementMatch.getElementID()));
|
||||||
|
parseInfo.getDimensionFilters().add(dimensionFilter);
|
||||||
|
ContextHelper.setEntityId(entry.getKey(), schemaElementMatch.getWord(), chaConfigRichDesc,
|
||||||
|
parseInfo);
|
||||||
|
} else {
|
||||||
|
Filter dimensionFilter = new Filter();
|
||||||
|
List<String> vals = new ArrayList<>();
|
||||||
|
entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
|
||||||
|
dimensionFilter.setValue(vals);
|
||||||
|
dimensionFilter.setBizName(dimensionDesc.getBizName());
|
||||||
|
dimensionFilter.setName(dimensionDesc.getName());
|
||||||
|
dimensionFilter.setOperator(FilterOperatorEnum.IN);
|
||||||
|
dimensionFilter.setElementID(entry.getKey());
|
||||||
|
parseInfo.getDimensionFilters().add(dimensionFilter);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,36 +1,117 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser.resolver;
|
package com.tencent.supersonic.chat.application.parser;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.*;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementCount;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
@Slf4j
|
import java.util.*;
|
||||||
public abstract class BaseDomainResolver implements DomainResolver {
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Override
|
@Slf4j
|
||||||
public boolean isDomainSwitch(ChatContext chatCtx, QueryContextReq searchCtx) {
|
public class HeuristicDomainResolver implements DomainResolver {
|
||||||
Long contextDomain = chatCtx.getParseInfo().getDomainId();
|
|
||||||
Long currentDomain = searchCtx.getParseInfo().getDomainId();
|
protected static Integer selectDomainBySchemaElementCount(Map<Integer, SemanticQuery> domainQueryModes,
|
||||||
boolean noSwitch = currentDomain == null || contextDomain == null || contextDomain.equals(currentDomain);
|
SchemaMapInfo schemaMap) {
|
||||||
log.info("ChatContext isDomainSwitch [{}]", !noSwitch);
|
Map<Integer, QueryMatchInfo> domainTypeMap = getDomainTypeMap(schemaMap);
|
||||||
return !noSwitch;
|
if (domainTypeMap.size() == 1) {
|
||||||
|
Integer domainSelect = domainTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey();
|
||||||
|
if (domainQueryModes.containsKey(domainSelect)) {
|
||||||
|
log.info("selectDomain from domainTypeMap not order [{}]", domainSelect);
|
||||||
|
return domainSelect;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Map.Entry<Integer, QueryMatchInfo> maxDomain = domainTypeMap.entrySet().stream()
|
||||||
|
.filter(entry -> domainQueryModes.containsKey(entry.getKey()))
|
||||||
|
.sorted(ContextHelper.DomainStatComparator).findFirst().orElse(null);
|
||||||
|
if (maxDomain != null) {
|
||||||
|
log.info("selectDomain from domainTypeMap order [{}]", maxDomain.getKey());
|
||||||
|
return maxDomain.getKey();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract Integer selectDomain(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
|
/**
|
||||||
ChatContext chatCtx, SchemaMapInfo schemaMap);
|
* to check can switch domain if context exit domain
|
||||||
|
*
|
||||||
|
* @return false will use context domain, true will use other domain , maybe include context domain
|
||||||
|
*/
|
||||||
|
protected static boolean isAllowSwitch(Map<Integer, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap,
|
||||||
|
ChatContext chatCtx, QueryContextReq searchCtx, Integer domainId) {
|
||||||
|
if (!Objects.nonNull(domainId) || domainId <= 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// except content domain, calculate the number of types for each domain, if numbers<=1 will not switch
|
||||||
|
Map<Integer, QueryMatchInfo> domainTypeMap = getDomainTypeMap(schemaMap);
|
||||||
|
log.info("isAllowSwitch domainTypeMap [{}]", domainTypeMap);
|
||||||
|
long otherDomainTypeNumBigOneCount = domainTypeMap.entrySet().stream()
|
||||||
|
.filter(entry -> domainQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(domainId))
|
||||||
|
.filter(entry -> entry.getValue().getCount() > 1).count();
|
||||||
|
if (otherDomainTypeNumBigOneCount >= 1) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// if query text only contain time , will not switch
|
||||||
|
for (SemanticQuery semanticQuery : domainQueryModes.values()) {
|
||||||
|
SemanticParseInfo semanticParseInfo = semanticQuery.getParseInfo();
|
||||||
|
if (semanticParseInfo == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (searchCtx.getQueryText() != null && semanticParseInfo.getDateInfo() != null) {
|
||||||
|
if (semanticParseInfo.getDateInfo().getText() != null) {
|
||||||
|
if (semanticParseInfo.getDateInfo().getText().equalsIgnoreCase(searchCtx.getQueryText())) {
|
||||||
|
log.info("timeParseResults is not null , can not switch context , timeParseResults:{},",
|
||||||
|
semanticParseInfo.getDateInfo());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// if context domain not in schemaMap , will switch
|
||||||
|
if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) {
|
||||||
|
log.info("domainId not in schemaMap ");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// other will not switch
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Map<Integer, QueryMatchInfo> getDomainTypeMap(SchemaMapInfo schemaMap) {
|
||||||
|
Map<Integer, QueryMatchInfo> domainCount = new HashMap<>();
|
||||||
|
for (Map.Entry<Integer, List<SchemaElementMatch>> entry : schemaMap.getDomainElementMatches().entrySet()) {
|
||||||
|
List<SchemaElementMatch> schemaElementMatches = schemaMap.getMatchedElements(entry.getKey());
|
||||||
|
if (schemaElementMatches != null && schemaElementMatches.size() > 0) {
|
||||||
|
if (!domainCount.containsKey(entry.getKey())) {
|
||||||
|
domainCount.put(entry.getKey(), new QueryMatchInfo());
|
||||||
|
}
|
||||||
|
QueryMatchInfo queryMatchInfo = domainCount.get(entry.getKey());
|
||||||
|
Set<SchemaElementType> schemaElementTypes = new HashSet<>();
|
||||||
|
schemaElementMatches.stream()
|
||||||
|
.forEach(schemaElementMatch -> schemaElementTypes.add(schemaElementMatch.getElementType()));
|
||||||
|
SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream()
|
||||||
|
.sorted(ContextHelper.schemaElementMatchComparatorBySimilarity).findFirst().orElse(null);
|
||||||
|
if (schemaElementMatchMax != null) {
|
||||||
|
queryMatchInfo.setMaxSimilarity(schemaElementMatchMax.getSimilarity());
|
||||||
|
}
|
||||||
|
queryMatchInfo.setCount(schemaElementTypes.size());
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return domainCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isDomainSwitch(ChatContext chatCtx, SemanticParseInfo semanticParseInfo) {
|
||||||
|
Long contextDomain = chatCtx.getParseInfo().getDomainId();
|
||||||
|
Long currentDomain = semanticParseInfo.getDomainId();
|
||||||
|
boolean noSwitch =
|
||||||
|
currentDomain == null || contextDomain == null || contextDomain.equals(currentDomain);
|
||||||
|
log.debug("ChatContext isDomainSwitch [{}] [{}]",
|
||||||
|
semanticParseInfo.getQueryMode(), !noSwitch);
|
||||||
|
return !noSwitch;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
|
public Integer resolve(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
|
||||||
@@ -44,88 +125,23 @@ public abstract class BaseDomainResolver implements DomainResolver {
|
|||||||
return selectDomainBySchemaElementCount(domainQueryModes, schemaMap);
|
return selectDomainBySchemaElementCount(domainQueryModes, schemaMap);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected static Integer selectDomainBySchemaElementCount(Map<Integer, SemanticQuery> domainQueryModes,
|
public Integer selectDomain(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
|
||||||
|
ChatContext chatCtx,
|
||||||
SchemaMapInfo schemaMap) {
|
SchemaMapInfo schemaMap) {
|
||||||
Map<Integer, SchemaElementCount> domainTypeMap = getDomainTypeMap(schemaMap);
|
// if QueryContext has domainId and in domainQueryModes
|
||||||
if (domainTypeMap.size() == 1) {
|
if (domainQueryModes.containsKey(searchCtx.getDomainId())) {
|
||||||
Integer domainSelect = domainTypeMap.entrySet().stream().collect(Collectors.toList()).get(0).getKey();
|
log.info("selectDomain from QueryContext [{}]", searchCtx.getDomainId());
|
||||||
if (domainQueryModes.containsKey(domainSelect)) {
|
return searchCtx.getDomainId();
|
||||||
log.info("selectDomain from domainTypeMap not order [{}]", domainSelect);
|
}
|
||||||
return domainSelect;
|
// if ChatContext has domainId and in domainQueryModes
|
||||||
}
|
if (chatCtx.getParseInfo().getDomainId() > 0) {
|
||||||
} else {
|
Integer domainId = Integer.valueOf(chatCtx.getParseInfo().getDomainId().intValue());
|
||||||
Map.Entry<Integer, SchemaElementCount> maxDomain = domainTypeMap.entrySet().stream()
|
if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, searchCtx, domainId)) {
|
||||||
.filter(entry -> domainQueryModes.containsKey(entry.getKey()))
|
log.info("selectDomain from ChatContext [{}]", domainId);
|
||||||
.sorted(ContextHelper.DomainStatComparator).findFirst().orElse(null);
|
return domainId;
|
||||||
if (maxDomain != null) {
|
|
||||||
log.info("selectDomain from domainTypeMap order [{}]", maxDomain.getKey());
|
|
||||||
return maxDomain.getKey();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// default 0
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* to check can switch domain if context exit domain
|
|
||||||
*
|
|
||||||
* @return false will use context domain, true will use other domain , maybe include context domain
|
|
||||||
*/
|
|
||||||
protected static boolean isAllowSwitch(Map<Integer, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap,
|
|
||||||
ChatContext chatCtx, QueryContextReq searchCtx, Integer domainId) {
|
|
||||||
if (!Objects.nonNull(domainId) || domainId <= 0) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
// except content domain, calculate the number of types for each domain, if numbers<=1 will not switch
|
|
||||||
Map<Integer, SchemaElementCount> domainTypeMap = getDomainTypeMap(schemaMap);
|
|
||||||
log.info("isAllowSwitch domainTypeMap [{}]", domainTypeMap);
|
|
||||||
long otherDomainTypeNumBigOneCount = domainTypeMap.entrySet().stream()
|
|
||||||
.filter(entry -> domainQueryModes.containsKey(entry.getKey()) && !entry.getKey().equals(domainId))
|
|
||||||
.filter(entry -> entry.getValue().getCount() > 1).count();
|
|
||||||
if (otherDomainTypeNumBigOneCount >= 1) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
// if query text only contain time , will not switch
|
|
||||||
if (searchCtx.getQueryText() != null && searchCtx.getParseInfo().getDateInfo() != null) {
|
|
||||||
if (searchCtx.getParseInfo().getDateInfo().getText() != null) {
|
|
||||||
if (searchCtx.getParseInfo().getDateInfo().getText().equalsIgnoreCase(searchCtx.getQueryText())) {
|
|
||||||
log.info("timeParseResults is not null , can not switch context , timeParseResults:{},",
|
|
||||||
searchCtx.getParseInfo().getDateInfo());
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// if context domain not in schemaMap , will switch
|
|
||||||
if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) {
|
|
||||||
log.info("domainId not in schemaMap ");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
// other will not switch
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected static Map<Integer, SchemaElementCount> getDomainTypeMap(SchemaMapInfo schemaMap) {
|
|
||||||
Map<Integer, SchemaElementCount> domainCount = new HashMap<>();
|
|
||||||
for (Map.Entry<Integer, List<SchemaElementMatch>> entry : schemaMap.getDomainElementMatches().entrySet()) {
|
|
||||||
List<SchemaElementMatch> schemaElementMatches = schemaMap.getMatchedElements(entry.getKey());
|
|
||||||
if (schemaElementMatches != null && schemaElementMatches.size() > 0) {
|
|
||||||
if (!domainCount.containsKey(entry.getKey())) {
|
|
||||||
domainCount.put(entry.getKey(), new SchemaElementCount());
|
|
||||||
}
|
|
||||||
SchemaElementCount schemaElementCount = domainCount.get(entry.getKey());
|
|
||||||
Set<SchemaElementType> schemaElementTypes = new HashSet<>();
|
|
||||||
schemaElementMatches.stream()
|
|
||||||
.forEach(schemaElementMatch -> schemaElementTypes.add(schemaElementMatch.getElementType()));
|
|
||||||
SchemaElementMatch schemaElementMatchMax = schemaElementMatches.stream()
|
|
||||||
.sorted(ContextHelper.schemaElementMatchComparatorBySimilarity).findFirst().orElse(null);
|
|
||||||
if (schemaElementMatchMax != null) {
|
|
||||||
schemaElementCount.setMaxSimilarity(schemaElementMatchMax.getSimilarity());
|
|
||||||
}
|
|
||||||
schemaElementCount.setCount(schemaElementTypes.size());
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return domainCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.parser;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
|
import com.tencent.supersonic.chat.application.knowledge.WordNatureService;
|
||||||
|
import com.tencent.supersonic.chat.application.query.LLMSemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.domain.config.LLMConfig;
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.chat.LLMReq;
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.chat.LLMResp;
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.chat.LLMSchema;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.DslToSemanticInfo;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.SemanticSatisfactionChecker;
|
||||||
|
import com.tencent.supersonic.common.nlp.ItemDO;
|
||||||
|
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||||
|
import com.tencent.supersonic.common.util.json.JsonUtil;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.springframework.core.io.support.SpringFactoriesLoader;
|
||||||
|
import org.springframework.http.HttpEntity;
|
||||||
|
import org.springframework.http.HttpHeaders;
|
||||||
|
import org.springframework.http.HttpMethod;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class LLMSemanticParser implements SemanticParser {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void parse(QueryContextReq queryContext, ChatContext chatCtx) {
|
||||||
|
if (SemanticSatisfactionChecker.check(queryContext)) {
|
||||||
|
log.info("There is no need parse by llm , queryText:{}", queryContext.getQueryText());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Integer domainId = getDomainId(queryContext, chatCtx);
|
||||||
|
LLMResp llmResp = requestLLM(queryContext, domainId);
|
||||||
|
if (Objects.isNull(llmResp)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LLMSemanticQuery semanticQuery = new LLMSemanticQuery();
|
||||||
|
SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
|
||||||
|
String sql = convertToSql(llmResp, parseInfo);
|
||||||
|
parseInfo.setDomainId(Long.valueOf(domainId));
|
||||||
|
parseInfo.setBonus(queryContext.getQueryText().length() * 1.0);
|
||||||
|
parseInfo.setQueryMode(LLMSemanticQuery.QUERY_MODE);
|
||||||
|
parseInfo.setInfo(sql);
|
||||||
|
queryContext.getCandidateQueries().add(semanticQuery);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String convertToSql(LLMResp llmResp, SemanticParseInfo parseInfo) {
|
||||||
|
return DslToSemanticInfo.convert(parseInfo, llmResp);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected LLMResp requestLLM(QueryContextReq queryContext, Integer domainId) {
|
||||||
|
try {
|
||||||
|
final LLMConfig llmConfig = ContextUtils.getBean(LLMConfig.class);
|
||||||
|
|
||||||
|
DomainInfos domainInfos = ContextUtils.getBean(WordNatureService.class).getCache().getUnchecked("");
|
||||||
|
|
||||||
|
Map<Integer, String> domainIdToName = domainInfos.getDomains().stream()
|
||||||
|
.collect(Collectors.toMap(ItemDO::getDomain, a -> a.getName(), (k1, k2) -> k1));
|
||||||
|
|
||||||
|
Map<Integer, String> itemIdToName = domainInfos.getDimensions().stream()
|
||||||
|
.filter(entry -> domainId.equals(entry.getDomain()))
|
||||||
|
.collect(Collectors.toMap(ItemDO::getItemId, ItemDO::getName, (value1, value2) -> value2));
|
||||||
|
|
||||||
|
String domainName = domainIdToName.get(domainId);
|
||||||
|
LLMReq llmReq = new LLMReq();
|
||||||
|
llmReq.setQueryText(queryContext.getQueryText());
|
||||||
|
|
||||||
|
List<SchemaElementMatch> matchedElements = queryContext.getMapInfo().getMatchedElements(domainId);
|
||||||
|
|
||||||
|
List<String> fieldNameList = matchedElements.stream()
|
||||||
|
.filter(schemaElementMatch ->
|
||||||
|
SchemaElementType.METRIC.equals(schemaElementMatch.getElementType()) ||
|
||||||
|
SchemaElementType.DIMENSION.equals(schemaElementMatch.getElementType()) ||
|
||||||
|
SchemaElementType.VALUE.equals(schemaElementMatch.getElementType()))
|
||||||
|
.map(schemaElementMatch -> {
|
||||||
|
if (!SchemaElementType.VALUE.equals(schemaElementMatch.getElementType())) {
|
||||||
|
return schemaElementMatch.getWord();
|
||||||
|
}
|
||||||
|
return itemIdToName.get(schemaElementMatch.getElementID());
|
||||||
|
})
|
||||||
|
.filter(name -> StringUtils.isNotEmpty(name) && !name.contains("%"))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
LLMSchema llmSchema = new LLMSchema();
|
||||||
|
llmSchema.setDomainName(domainName);
|
||||||
|
llmSchema.setFieldNameList(fieldNameList);
|
||||||
|
llmReq.setSchema(llmSchema);
|
||||||
|
|
||||||
|
log.info("domainId:{},llmReq:{}", domainId, llmReq);
|
||||||
|
String questUrl = llmConfig.getUrl() + llmConfig.getQueryToSqlPath();
|
||||||
|
|
||||||
|
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
|
||||||
|
|
||||||
|
HttpHeaders headers = new HttpHeaders();
|
||||||
|
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||||
|
HttpEntity<String> entity = new HttpEntity<>(JsonUtil.toString(llmReq), headers);
|
||||||
|
|
||||||
|
log.info("requestLLM request:{},entity:{}", questUrl, entity);
|
||||||
|
ResponseEntity<LLMResp> responseEntity = restTemplate.exchange(questUrl, HttpMethod.POST, entity,
|
||||||
|
LLMResp.class);
|
||||||
|
|
||||||
|
log.info("requestLLM result:{}", responseEntity);
|
||||||
|
return responseEntity.getBody();
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("requestLLM error", e);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Integer getDomainId(QueryContextReq queryContext, ChatContext chatCtx) {
|
||||||
|
SchemaMapInfo mapInfo = queryContext.getMapInfo();
|
||||||
|
Set<Integer> matchedDomains = mapInfo.getMatchedDomains();
|
||||||
|
Map<Integer, SemanticQuery> domainQueryModes = new HashMap<>();
|
||||||
|
for (Integer matchedDomain : matchedDomains) {
|
||||||
|
domainQueryModes.put(matchedDomain, new LLMSemanticQuery());
|
||||||
|
}
|
||||||
|
List<DomainResolver> domainResolverList = SpringFactoriesLoader.loadFactories(DomainResolver.class,
|
||||||
|
Thread.currentThread().getContextClassLoader());
|
||||||
|
Optional<Integer> domainId = domainResolverList.stream()
|
||||||
|
.map(domainResolver -> domainResolver.resolve(domainQueryModes, queryContext, chatCtx,
|
||||||
|
queryContext.getMapInfo())).filter(d -> d > 0).findFirst();
|
||||||
|
if (domainId.isPresent()) {
|
||||||
|
return domainId.get();
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser;
|
|
||||||
|
|
||||||
import static com.tencent.supersonic.common.constant.Constants.DAY;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticParser;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
|
||||||
import com.tencent.supersonic.chat.application.query.EntityListFilter;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
|
|
||||||
import com.tencent.supersonic.common.constant.Constants;
|
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
|
||||||
import com.tencent.supersonic.common.pojo.Order;
|
|
||||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
|
||||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.LinkedHashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
import org.springframework.beans.BeanUtils;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class ListFilterParser implements SemanticParser {
|
|
||||||
|
|
||||||
|
|
||||||
private DefaultSemanticInternalUtils defaultSemanticUtils;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean parse(QueryContextReq queryContext, ChatContext chatCtx) {
|
|
||||||
defaultSemanticUtils = ContextUtils.getBean(DefaultSemanticInternalUtils.class);
|
|
||||||
|
|
||||||
String queryMode = queryContext.getParseInfo().getQueryMode();
|
|
||||||
if (!EntityListFilter.QUERY_MODE.equals(queryMode)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
this.fillDateEntityFilter(queryContext.getParseInfo());
|
|
||||||
this.addEntityDetailAndOrderByMetric(queryContext, chatCtx);
|
|
||||||
this.dealNativeQuery(queryContext, true);
|
|
||||||
return true;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private void fillDateEntityFilter(SemanticParseInfo semanticParseInfo) {
|
|
||||||
DateConf dateInfo = new DateConf();
|
|
||||||
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
|
|
||||||
dateInfo.setUnit(1);
|
|
||||||
dateInfo.setPeriod(DAY);
|
|
||||||
dateInfo.setText(String.format("近1天"));
|
|
||||||
semanticParseInfo.setDateInfo(dateInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void addEntityDetailAndOrderByMetric(QueryContextReq queryContext, ChatContext chatCtx) {
|
|
||||||
if (queryContext.getParseInfo().getDomainId() > 0L) {
|
|
||||||
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(
|
|
||||||
queryContext.getParseInfo().getDomainId());
|
|
||||||
if (chaConfigRichDesc != null) {
|
|
||||||
SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
|
|
||||||
Set<SchemaItem> dimensions = new LinkedHashSet();
|
|
||||||
Set<String> primaryDimensions = this.addPrimaryDimension(chaConfigRichDesc.getEntity(), dimensions);
|
|
||||||
Set<SchemaItem> metrics = new LinkedHashSet();
|
|
||||||
if (chaConfigRichDesc.getEntity() != null
|
|
||||||
&& chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
|
|
||||||
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
|
|
||||||
.forEach((m) -> metrics.add(this.getMetric(m)));
|
|
||||||
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream()
|
|
||||||
.filter((m) -> !primaryDimensions.contains(m.getBizName()))
|
|
||||||
.forEach((m) -> dimensions.add(this.getDimension(m)));
|
|
||||||
}
|
|
||||||
|
|
||||||
semanticParseInfo.setDimensions(dimensions);
|
|
||||||
semanticParseInfo.setMetrics(metrics);
|
|
||||||
Set<Order> orders = new LinkedHashSet();
|
|
||||||
if (chaConfigRichDesc.getEntity() != null
|
|
||||||
&& chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
|
|
||||||
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
|
|
||||||
.forEach((metric) -> orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER)));
|
|
||||||
}
|
|
||||||
|
|
||||||
semanticParseInfo.setOrders(orders);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private Set<String> addPrimaryDimension(EntityRichInfo entity, Set<SchemaItem> dimensions) {
|
|
||||||
Set<String> primaryDimensions = new HashSet();
|
|
||||||
if (!Objects.isNull(entity) && !CollectionUtils.isEmpty(entity.getEntityIds())) {
|
|
||||||
entity.getEntityIds().stream().forEach((dimSchemaDesc) -> {
|
|
||||||
SchemaItem dimension = new SchemaItem();
|
|
||||||
BeanUtils.copyProperties(dimSchemaDesc, dimension);
|
|
||||||
dimensions.add(dimension);
|
|
||||||
primaryDimensions.add(dimSchemaDesc.getBizName());
|
|
||||||
});
|
|
||||||
return primaryDimensions;
|
|
||||||
} else {
|
|
||||||
return primaryDimensions;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private SchemaItem getMetric(MetricSchemaResp metricSchemaDesc) {
|
|
||||||
SchemaItem queryMeta = new SchemaItem();
|
|
||||||
queryMeta.setId(metricSchemaDesc.getId());
|
|
||||||
queryMeta.setBizName(metricSchemaDesc.getBizName());
|
|
||||||
queryMeta.setName(metricSchemaDesc.getName());
|
|
||||||
return queryMeta;
|
|
||||||
}
|
|
||||||
|
|
||||||
private SchemaItem getDimension(DimSchemaResp dimSchemaDesc) {
|
|
||||||
SchemaItem queryMeta = new SchemaItem();
|
|
||||||
queryMeta.setId(dimSchemaDesc.getId());
|
|
||||||
queryMeta.setBizName(dimSchemaDesc.getBizName());
|
|
||||||
queryMeta.setName(dimSchemaDesc.getName());
|
|
||||||
return queryMeta;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void dealNativeQuery(QueryContextReq queryContext, boolean isNativeQuery) {
|
|
||||||
if (Objects.nonNull(queryContext) && Objects.nonNull(queryContext.getParseInfo())) {
|
|
||||||
queryContext.getParseInfo().setNativeQuery(isNativeQuery);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,22 +1,30 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser;
|
package com.tencent.supersonic.chat.application.parser;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticParser;
|
import com.tencent.supersonic.chat.application.query.MetricSemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.application.query.RuleSemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.application.query.RuleSemanticQueryManager;
|
||||||
import com.tencent.supersonic.common.constant.Constants;
|
import com.tencent.supersonic.common.constant.Constants;
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.enums.TimeDimensionEnum;
|
||||||
import java.time.LocalDate;
|
import java.time.LocalDate;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.Stack;
|
import java.util.Stack;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class TimeSemanticParser implements SemanticParser {
|
public class TimeSemanticParser implements SemanticParser {
|
||||||
|
|
||||||
public TimeSemanticParser() {
|
private static final Pattern recentPeriodPattern = Pattern.compile(
|
||||||
}
|
".*(?<periodStr>(近|过去)((?<enNum>\\d+)|(?<zhNum>[一二三四五六七八九十百千万亿]+))个?(?<zhPeriod>[天周月年])).*");
|
||||||
|
|
||||||
private int zhNumParse(String zhNumStr) {
|
private int zhNumParse(String zhNumStr) {
|
||||||
Stack<Integer> stack = new Stack<>();
|
Stack<Integer> stack = new Stack<>();
|
||||||
@@ -42,11 +50,8 @@ public class TimeSemanticParser implements SemanticParser {
|
|||||||
return stack.stream().mapToInt(s -> s).sum();
|
return stack.stream().mapToInt(s -> s).sum();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final Pattern recentPeriodPattern = Pattern.compile(
|
|
||||||
".*(?<periodStr>(近|过去)((?<enNum>\\d+)|(?<zhNum>[一二三四五六七八九十百千万亿]+))个?(?<zhPeriod>[天周月年])).*");
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean parse(QueryContextReq queryContext, ChatContext chatCtx) {
|
public void parse(QueryContextReq queryContext, ChatContext chatContext) {
|
||||||
Matcher m = recentPeriodPattern.matcher(queryContext.getQueryText());
|
Matcher m = recentPeriodPattern.matcher(queryContext.getQueryText());
|
||||||
if (m.matches()) {
|
if (m.matches()) {
|
||||||
int num = 0;
|
int num = 0;
|
||||||
@@ -61,18 +66,19 @@ public class TimeSemanticParser implements SemanticParser {
|
|||||||
DateConf info = new DateConf();
|
DateConf info = new DateConf();
|
||||||
String zhPeriod = m.group("zhPeriod");
|
String zhPeriod = m.group("zhPeriod");
|
||||||
int days;
|
int days;
|
||||||
|
info.setPeriod(Constants.DAY);
|
||||||
switch (zhPeriod) {
|
switch (zhPeriod) {
|
||||||
case "周":
|
case "周":
|
||||||
days = 7;
|
days = 7;
|
||||||
info.setPeriod(Constants.WEEK);
|
//info.setPeriod(Constants.WEEK);
|
||||||
break;
|
break;
|
||||||
case "月":
|
case "月":
|
||||||
days = 30;
|
days = 30;
|
||||||
info.setPeriod(Constants.MONTH);
|
//info.setPeriod(Constants.MONTH);
|
||||||
break;
|
break;
|
||||||
case "年":
|
case "年":
|
||||||
days = 365;
|
days = 365;
|
||||||
info.setPeriod(Constants.YEAR);
|
//info.setPeriod(Constants.YEAR);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
days = 1;
|
days = 1;
|
||||||
@@ -87,9 +93,53 @@ public class TimeSemanticParser implements SemanticParser {
|
|||||||
info.setText(text);
|
info.setText(text);
|
||||||
info.setStartDate(LocalDate.now().minusDays(days).toString());
|
info.setStartDate(LocalDate.now().minusDays(days).toString());
|
||||||
info.setUnit(days);
|
info.setUnit(days);
|
||||||
queryContext.getParseInfo().setDateInfo(info);
|
//queryContext.getParseInfo().setDateInfo(info);
|
||||||
|
for (SemanticQuery query : queryContext.getCandidateQueries()) {
|
||||||
|
if (query instanceof MetricSemanticQuery) {
|
||||||
|
query.getParseInfo().setDateInfo(info);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
doParseOnlyTime(queryContext, chatContext, info);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void doParseOnlyTime(QueryContextReq queryContext, ChatContext chatContext, DateConf info) {
|
||||||
|
if (!queryContext.getCandidateQueries().isEmpty() || chatContext.getParseInfo() == null || Objects.isNull(
|
||||||
|
info.getText())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (info.getText().equals(queryContext.getQueryText()) && queryContext.getMapInfo().getDomainElementMatches()
|
||||||
|
.isEmpty()
|
||||||
|
) {
|
||||||
|
if (Objects.nonNull(chatContext.getParseInfo().getQueryMode()) && Objects.nonNull(
|
||||||
|
chatContext.getParseInfo().getDomainId()) && chatContext.getParseInfo().getDomainId() > 0) {
|
||||||
|
if (Objects.nonNull(chatContext.getParseInfo().getDateInfo()) && !chatContext.getParseInfo()
|
||||||
|
.getDateInfo().getPeriod().equals(info.getPeriod())) {
|
||||||
|
if (!CollectionUtils.isEmpty(chatContext.getParseInfo().getDimensions())) {
|
||||||
|
String dateField = TimeDimensionEnum.DAY.getName();
|
||||||
|
if (Constants.MONTH.equals(chatContext.getParseInfo().getDateInfo().getPeriod())) {
|
||||||
|
dateField = TimeDimensionEnum.MONTH.getName();
|
||||||
|
}
|
||||||
|
if (Constants.WEEK.equals(chatContext.getParseInfo().getDateInfo().getPeriod())) {
|
||||||
|
dateField = TimeDimensionEnum.WEEK.getName();
|
||||||
|
}
|
||||||
|
Set<SchemaItem> dimensions = new HashSet<>();
|
||||||
|
for (SchemaItem schemaItem : chatContext.getParseInfo().getDimensions()) {
|
||||||
|
if (schemaItem.getBizName().equals(dateField)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
dimensions.add(schemaItem);
|
||||||
|
}
|
||||||
|
chatContext.getParseInfo().setDimensions(dimensions);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chatContext.getParseInfo().setDateInfo(info);
|
||||||
|
RuleSemanticQuery semanticQuery = RuleSemanticQueryManager.create(
|
||||||
|
chatContext.getParseInfo().getQueryMode());
|
||||||
|
semanticQuery.setParseInfo(chatContext.getParseInfo());
|
||||||
|
queryContext.getCandidateQueries().add(semanticQuery);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser.resolver;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
|
||||||
|
|
||||||
/***
|
|
||||||
* aggregate parser
|
|
||||||
*/
|
|
||||||
public interface AggregateTypeResolver {
|
|
||||||
|
|
||||||
AggregateTypeEnum resolve(String queryText);
|
|
||||||
|
|
||||||
boolean hasCompareIntentionalWords(String queryText);
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser.resolver;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementCount;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.chat.application.query.SemanticQueryFactory;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class DomainSemanticQueryResolver implements SemanticQueryResolver {
|
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(DomainSemanticQueryResolver.class);
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticQuery resolve(List<SchemaElementMatch> elementMatches, QueryContextReq queryCtx) {
|
|
||||||
Map<SemanticQuery, SchemaElementCount> matchMap = new HashMap<>();
|
|
||||||
|
|
||||||
for (SemanticQuery semanticQuery : SemanticQueryFactory.getSemanticQueries()) {
|
|
||||||
|
|
||||||
SchemaElementCount match = semanticQuery.match(elementMatches, queryCtx);
|
|
||||||
|
|
||||||
if (match != null && match.getCount() > 0 && match.getMaxSimilarity() > 0) {
|
|
||||||
LOGGER.info("resolve match [{}:{}] ", semanticQuery.getQueryMode(), match);
|
|
||||||
matchMap.put(semanticQuery, match);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
// get the similarity max
|
|
||||||
Map.Entry<SemanticQuery, SchemaElementCount> matchMax = matchMap.entrySet().stream()
|
|
||||||
.sorted(ContextHelper.SemanticQueryStatComparator).findFirst().orElse(null);
|
|
||||||
if (matchMax != null) {
|
|
||||||
LOGGER.info("resolve max [{}:{}] ", matchMax.getKey().getQueryMode(), matchMax.getValue());
|
|
||||||
return matchMax.getKey();
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser.resolver;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementCount;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.context.annotation.Primary;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
|
|
||||||
@Service("DomainResolver")
|
|
||||||
@Slf4j
|
|
||||||
public class HeuristicDomainResolver extends BaseDomainResolver {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Integer selectDomain(Map<Integer, SemanticQuery> domainQueryModes, QueryContextReq searchCtx,
|
|
||||||
ChatContext chatCtx,
|
|
||||||
SchemaMapInfo schemaMap) {
|
|
||||||
// if QueryContext has domainId and in domainQueryModes
|
|
||||||
if (domainQueryModes.containsKey(searchCtx.getDomainId())) {
|
|
||||||
log.info("selectDomain from QueryContext [{}]", searchCtx.getDomainId());
|
|
||||||
return searchCtx.getDomainId();
|
|
||||||
}
|
|
||||||
// if ChatContext has domainId and in domainQueryModes
|
|
||||||
if (chatCtx.getParseInfo().getDomainId() > 0) {
|
|
||||||
Integer domainId = Integer.valueOf(chatCtx.getParseInfo().getDomainId().intValue());
|
|
||||||
if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, searchCtx, domainId)) {
|
|
||||||
log.info("selectDomain from ChatContext [{}]", domainId);
|
|
||||||
return domainId;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// default 0
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser.resolver;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Map.Entry;
|
|
||||||
import java.util.regex.Matcher;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.context.annotation.Primary;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
@Slf4j
|
|
||||||
@Primary
|
|
||||||
public class RegexAggregateTypeResolver implements AggregateTypeResolver {
|
|
||||||
|
|
||||||
private static Map<AggregateTypeEnum, Pattern> aggregateRegexMap = new HashMap<>();
|
|
||||||
private static Pattern compareIntentionalWord = Pattern.compile("(?i)(比较|对比)");
|
|
||||||
|
|
||||||
static {
|
|
||||||
aggregateRegexMap.put(AggregateTypeEnum.MAX, Pattern.compile("(?i)(最大值|最大|max|峰值|最高)"));
|
|
||||||
aggregateRegexMap.put(AggregateTypeEnum.MIN, Pattern.compile("(?i)(最小值|最小|min|最低)"));
|
|
||||||
aggregateRegexMap.put(AggregateTypeEnum.SUM, Pattern.compile("(?i)(汇总|总和|sum)"));
|
|
||||||
aggregateRegexMap.put(AggregateTypeEnum.AVG, Pattern.compile("(?i)(平均值|日均|平均|avg)"));
|
|
||||||
aggregateRegexMap.put(AggregateTypeEnum.TOPN, Pattern.compile("(?i)(top)"));
|
|
||||||
aggregateRegexMap.put(AggregateTypeEnum.DISTINCT, Pattern.compile("(?i)(uv)"));
|
|
||||||
aggregateRegexMap.put(AggregateTypeEnum.COUNT, Pattern.compile("(?i)(总数|pv)"));
|
|
||||||
aggregateRegexMap.put(AggregateTypeEnum.NONE, Pattern.compile("(?i)(明细)"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public AggregateTypeEnum resolve(String text) {
|
|
||||||
|
|
||||||
Map<AggregateTypeEnum, Integer> aggregateCount = new HashMap<>(aggregateRegexMap.size());
|
|
||||||
for (Entry<AggregateTypeEnum, Pattern> entry : aggregateRegexMap.entrySet()) {
|
|
||||||
Matcher matcher = entry.getValue().matcher(text);
|
|
||||||
int count = 0;
|
|
||||||
while (matcher.find()) {
|
|
||||||
count++;
|
|
||||||
}
|
|
||||||
if (count > 0) {
|
|
||||||
aggregateCount.put(entry.getKey(), count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return aggregateCount.entrySet().stream().max(Map.Entry.comparingByValue()).map(entry -> entry.getKey())
|
|
||||||
.orElse(null);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean hasCompareIntentionalWords(String queryText) {
|
|
||||||
return compareIntentionalWord.matcher(queryText).find();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser.resolver;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Base interface for resolving query mode.
|
|
||||||
*/
|
|
||||||
public interface SemanticQueryResolver {
|
|
||||||
|
|
||||||
SemanticQuery resolve(List<SchemaElementMatch> elementMatches, QueryContextReq queryCtx);
|
|
||||||
}
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementCount;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
|
||||||
import com.tencent.supersonic.chat.application.DomainEntityService;
|
|
||||||
import com.tencent.supersonic.chat.application.parser.resolver.DomainResolver;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.search.QueryState;
|
|
||||||
import com.tencent.supersonic.chat.domain.service.ChatService;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
|
||||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
|
|
||||||
public abstract class BaseSemanticQuery implements SemanticQuery {
|
|
||||||
|
|
||||||
protected QueryModeOption queryModeOption = QueryModeOption.build();
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(BaseSemanticQuery.class);
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public QueryResultResp execute(QueryContextReq queryCtx, ChatContext chatCtx) {
|
|
||||||
|
|
||||||
DomainResolver domainResolver = ContextUtils.getBean(DomainResolver.class);
|
|
||||||
ChatService chatService = ContextUtils.getBean(ChatService.class);
|
|
||||||
SemanticLayer semanticLayer = ContextUtils.getBean(SemanticLayer.class);
|
|
||||||
|
|
||||||
SemanticParseInfo semanticParse = queryCtx.getParseInfo();
|
|
||||||
|
|
||||||
String queryMode = semanticParse.getQueryMode();
|
|
||||||
|
|
||||||
if (semanticParse.getDomainId() < 0 || StringUtils.isEmpty(queryMode)) {
|
|
||||||
// reach here some error may happen
|
|
||||||
LOGGER.error("not find QueryMode");
|
|
||||||
throw new RuntimeException("not find QueryMode");
|
|
||||||
}
|
|
||||||
|
|
||||||
supplyMetadata(semanticLayer, queryCtx);
|
|
||||||
|
|
||||||
// is domain switch
|
|
||||||
if (domainResolver.isDomainSwitch(chatCtx, queryCtx)) {
|
|
||||||
chatService.switchContext(chatCtx);
|
|
||||||
}
|
|
||||||
// submit semantic query based on the result of semantic parsing
|
|
||||||
SemanticParseInfo semanticParseInfo = getContext(chatCtx, queryCtx);
|
|
||||||
QueryResultResp queryResponse = new QueryResultResp();
|
|
||||||
QueryResultWithSchemaResp queryResult = semanticLayer.queryByStruct(
|
|
||||||
SchemaInfoConverter.convertTo(semanticParseInfo), queryCtx.getUser());
|
|
||||||
if (queryResult != null) {
|
|
||||||
queryResponse.setQueryAuthorization(queryResult.getQueryAuthorization());
|
|
||||||
}
|
|
||||||
String sql = queryResult == null ? null : queryResult.getSql();
|
|
||||||
List<Map<String, Object>> resultList = queryResult == null ? new ArrayList<>()
|
|
||||||
: queryResult.getResultList();
|
|
||||||
List<QueryColumn> columns = queryResult == null ? new ArrayList<>() : queryResult.getColumns();
|
|
||||||
queryResponse.setQuerySql(sql);
|
|
||||||
queryResponse.setQueryResults(resultList);
|
|
||||||
queryResponse.setQueryColumns(columns);
|
|
||||||
queryResponse.setQueryMode(queryMode);
|
|
||||||
|
|
||||||
// add domain info
|
|
||||||
EntityInfo entityInfo = ContextUtils.getBean(DomainEntityService.class)
|
|
||||||
.getEntityInfo(queryCtx, chatCtx, queryCtx.getUser());
|
|
||||||
queryResponse.setEntityInfo(entityInfo);
|
|
||||||
return queryResponse;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private void supplyMetadata(SemanticLayer semanticLayer, QueryContextReq queryCtx) {
|
|
||||||
DefaultSemanticInternalUtils defaultSemanticUtils = ContextUtils.getBean(DefaultSemanticInternalUtils.class);
|
|
||||||
|
|
||||||
SchemaMapInfo mapInfo = queryCtx.getMapInfo();
|
|
||||||
SemanticParseInfo semanticParse = queryCtx.getParseInfo();
|
|
||||||
Long domain = semanticParse.getDomainId();
|
|
||||||
List<SchemaElementMatch> schemaElementMatches = mapInfo.getMatchedElements(domain.intValue());
|
|
||||||
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domain);
|
|
||||||
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(domain);
|
|
||||||
|
|
||||||
// supply metadata
|
|
||||||
if (!CollectionUtils.isEmpty(schemaElementMatches)) {
|
|
||||||
this.queryModeOption.addQuerySemanticParseInfo(schemaElementMatches, domainSchemaDesc,
|
|
||||||
chaConfigRichDesc, semanticParse);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void updateContext(QueryResultResp queryResponse, ChatContext chatCtx, QueryContextReq queryCtx) {
|
|
||||||
if (queryCtx.isSaveAnswer() && queryResponse != null
|
|
||||||
&& queryResponse.getQueryState() == QueryState.NORMAL.getState()) {
|
|
||||||
chatCtx.setParseInfo(getParseInfo(queryCtx, chatCtx));
|
|
||||||
chatCtx.setQueryText(queryCtx.getQueryText());
|
|
||||||
ContextUtils.getBean(ChatService.class).updateContext(chatCtx);
|
|
||||||
}
|
|
||||||
queryResponse.setChatContext(queryCtx.getParseInfo());
|
|
||||||
}
|
|
||||||
|
|
||||||
public abstract SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx);
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SchemaElementCount match(List<SchemaElementMatch> elementMatches, QueryContextReq queryCtx) {
|
|
||||||
return queryModeOption.match(elementMatches, queryCtx);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,26 +1,22 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
@Service
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
public class EntityDetail extends BaseSemanticQuery {
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class EntityDetail extends EntitySemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "ENTITY_DETAIL";
|
public static String QUERY_MODE = "ENTITY_DETAIL";
|
||||||
|
|
||||||
public EntityDetail() {
|
public EntityDetail() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.unused());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.unused());
|
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1)
|
||||||
queryModeOption.setDimension(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST,
|
.addOption(VALUE, REQUIRED, AT_LEAST, 1);
|
||||||
1);
|
|
||||||
queryModeOption.setFilter(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setMetric(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setEntity(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_MOST, 1);
|
|
||||||
queryModeOption.setDomain(QueryModeElementOption.optional());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -29,21 +25,9 @@ public class EntityDetail extends BaseSemanticQuery {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
SemanticParseInfo semanticParseInfo = chatCtx.getParseInfo();
|
ContextHelper.addIfEmpty(chatContext.getParseInfo().getDimensionFilters(),
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
parseInfo.getDimensionFilters());
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getDimensionFilters(),
|
|
||||||
semanticParseInfo.getDimensionFilters());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters());
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,26 +1,42 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
|
||||||
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||||
|
import static com.tencent.supersonic.common.constant.Constants.DAY;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import org.springframework.stereotype.Service;
|
import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
|
||||||
|
import com.tencent.supersonic.common.constant.Constants;
|
||||||
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
|
import com.tencent.supersonic.common.pojo.Order;
|
||||||
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
|
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.BeanUtils;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@Service
|
@Slf4j
|
||||||
public class EntityListFilter extends BaseSemanticQuery {
|
@Component
|
||||||
|
public class EntityListFilter extends EntitySemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "ENTITY_LIST_FILTER";
|
public static String QUERY_MODE = "ENTITY_LIST_FILTER";
|
||||||
private static Long entityListLimit = 200L;
|
private static Long entityListLimit = 200L;
|
||||||
|
|
||||||
public EntityListFilter() {
|
public EntityListFilter() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.unused());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.unused());
|
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1);
|
||||||
queryModeOption.setDimension(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setFilter(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setMetric(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setEntity(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setDomain(QueryModeElementOption.optional());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -28,23 +44,98 @@ public class EntityListFilter extends BaseSemanticQuery {
|
|||||||
return QUERY_MODE;
|
return QUERY_MODE;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx) {
|
|
||||||
SemanticParseInfo semanticParseInfo = chatCtx.getParseInfo();
|
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getDimensionFilters(),
|
|
||||||
semanticParseInfo.getDimensionFilters());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters());
|
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||||
semanticParseInfo.setLimit(entityListLimit);
|
parseInfo.setLimit(entityListLimit);
|
||||||
return semanticParseInfo;
|
this.fillDateEntityFilter(parseInfo);
|
||||||
|
this.addEntityDetailAndOrderByMetric(parseInfo);
|
||||||
|
this.dealNativeQuery(parseInfo, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void fillDateEntityFilter(SemanticParseInfo semanticParseInfo) {
|
||||||
|
DateConf dateInfo = new DateConf();
|
||||||
|
dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS);
|
||||||
|
dateInfo.setUnit(1);
|
||||||
|
dateInfo.setPeriod(DAY);
|
||||||
|
dateInfo.setText(String.format("近1天"));
|
||||||
|
semanticParseInfo.setDateInfo(dateInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addEntityDetailAndOrderByMetric(SemanticParseInfo semanticParseInfo) {
|
||||||
|
if (semanticParseInfo.getDomainId() > 0L) {
|
||||||
|
DefaultSemanticInternalUtils defaultSemanticUtils = ContextUtils.getBean(
|
||||||
|
DefaultSemanticInternalUtils.class);
|
||||||
|
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(
|
||||||
|
semanticParseInfo.getDomainId());
|
||||||
|
if (chaConfigRichDesc != null) {
|
||||||
|
//SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
|
||||||
|
Set<SchemaItem> dimensions = new LinkedHashSet();
|
||||||
|
Set<String> primaryDimensions = this.addPrimaryDimension(chaConfigRichDesc.getEntity(), dimensions);
|
||||||
|
Set<SchemaItem> metrics = new LinkedHashSet();
|
||||||
|
if (chaConfigRichDesc.getEntity() != null
|
||||||
|
&& chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
|
||||||
|
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
|
||||||
|
.forEach((m) -> metrics.add(this.getMetric(m)));
|
||||||
|
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream()
|
||||||
|
.filter((m) -> !primaryDimensions.contains(m.getBizName()))
|
||||||
|
.forEach((m) -> dimensions.add(this.getDimension(m)));
|
||||||
|
}
|
||||||
|
|
||||||
|
semanticParseInfo.setDimensions(dimensions);
|
||||||
|
semanticParseInfo.setMetrics(metrics);
|
||||||
|
Set<Order> orders = new LinkedHashSet();
|
||||||
|
if (chaConfigRichDesc.getEntity() != null
|
||||||
|
&& chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
|
||||||
|
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
|
||||||
|
.forEach((metric) -> orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER)));
|
||||||
|
}
|
||||||
|
|
||||||
|
semanticParseInfo.setOrders(orders);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private Set<String> addPrimaryDimension(EntityRichInfo entity, Set<SchemaItem> dimensions) {
|
||||||
|
Set<String> primaryDimensions = new HashSet();
|
||||||
|
if (!Objects.isNull(entity) && !CollectionUtils.isEmpty(entity.getEntityIds())) {
|
||||||
|
entity.getEntityIds().stream().forEach((dimSchemaDesc) -> {
|
||||||
|
SchemaItem dimension = new SchemaItem();
|
||||||
|
BeanUtils.copyProperties(dimSchemaDesc, dimension);
|
||||||
|
dimensions.add(dimension);
|
||||||
|
primaryDimensions.add(dimSchemaDesc.getBizName());
|
||||||
|
});
|
||||||
|
return primaryDimensions;
|
||||||
|
} else {
|
||||||
|
return primaryDimensions;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private SchemaItem getMetric(MetricSchemaResp metricSchemaDesc) {
|
||||||
|
SchemaItem queryMeta = new SchemaItem();
|
||||||
|
queryMeta.setId(metricSchemaDesc.getId());
|
||||||
|
queryMeta.setBizName(metricSchemaDesc.getBizName());
|
||||||
|
queryMeta.setName(metricSchemaDesc.getName());
|
||||||
|
return queryMeta;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SchemaItem getDimension(DimSchemaResp dimSchemaDesc) {
|
||||||
|
SchemaItem queryMeta = new SchemaItem();
|
||||||
|
queryMeta.setId(dimSchemaDesc.getId());
|
||||||
|
queryMeta.setBizName(dimSchemaDesc.getBizName());
|
||||||
|
queryMeta.setName(dimSchemaDesc.getName());
|
||||||
|
return queryMeta;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void dealNativeQuery(SemanticParseInfo semanticParseInfo, boolean isNativeQuery) {
|
||||||
|
if (Objects.nonNull(semanticParseInfo)) {
|
||||||
|
semanticParseInfo.setNativeQuery(isNativeQuery);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,27 +1,23 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
|
||||||
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Service
|
@Component
|
||||||
public class EntityListTopN extends BaseSemanticQuery {
|
public class EntityListTopN extends EntitySemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "ENTITY_LIST_TOPN";
|
public static String QUERY_MODE = "ENTITY_LIST_TOPN";
|
||||||
|
|
||||||
|
|
||||||
public EntityListTopN() {
|
public EntityListTopN() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.optional());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.optional());
|
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1)
|
||||||
queryModeOption.setDimension(QueryModeElementOption.unused());
|
.setSupportOrderBy(true);
|
||||||
queryModeOption.setFilter(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setMetric(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setEntity(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setDomain(QueryModeElementOption.optional());
|
|
||||||
queryModeOption.setSupportOrderBy(true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -30,22 +26,10 @@ public class EntityListTopN extends BaseSemanticQuery {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
SemanticParseInfo semanticParseInfo = chatCtx.getParseInfo();
|
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||||
ContextHelper.updateTime(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
|
||||||
ContextHelper.updateTimeIfEmpty(chatCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,26 +1,24 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
|
||||||
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
|
||||||
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Service
|
@Component
|
||||||
public class EntityMetricFilter extends BaseSemanticQuery {
|
public class EntityMetricFilter extends EntitySemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "ENTITY_METRIC_FILTER";
|
public static String QUERY_MODE = "ENTITY_METRIC_FILTER";
|
||||||
|
|
||||||
|
|
||||||
public EntityMetricFilter() {
|
public EntityMetricFilter() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.optional());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.optional());
|
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1)
|
||||||
queryModeOption.setDimension(QueryModeElementOption.unused());
|
.addOption(VALUE, REQUIRED, AT_LEAST, 1);
|
||||||
queryModeOption.setFilter(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setMetric(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setEntity(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setDomain(QueryModeElementOption.optional());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -29,21 +27,9 @@ public class EntityMetricFilter extends BaseSemanticQuery {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
SemanticParseInfo semanticParseInfo = chatCtx.getParseInfo();
|
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getDimensionFilters(),
|
|
||||||
semanticParseInfo.getDimensionFilters());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters());
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
|
||||||
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||||
|
|
||||||
|
public abstract class EntitySemanticQuery extends RuleSemanticQuery {
|
||||||
|
|
||||||
|
public EntitySemanticQuery() {
|
||||||
|
super();
|
||||||
|
queryMatcher.addOption(ENTITY, REQUIRED, AT_LEAST, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
|
import com.tencent.supersonic.common.constant.Constants;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class HeuristicQuerySelector implements QuerySelector {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SemanticQuery select(List<SemanticQuery> candidateQueries) {
|
||||||
|
double maxScore = 0;
|
||||||
|
SemanticQuery pickedQuery = null;
|
||||||
|
|
||||||
|
for (SemanticQuery query : candidateQueries) {
|
||||||
|
if (query instanceof LLMSemanticQuery) {
|
||||||
|
log.info("force to use LLM if existed");
|
||||||
|
return query;
|
||||||
|
}
|
||||||
|
|
||||||
|
SemanticParseInfo semanticParse = query.getParseInfo();
|
||||||
|
double score = computeScore(semanticParse);
|
||||||
|
if (score > maxScore) {
|
||||||
|
maxScore = score;
|
||||||
|
pickedQuery = query;
|
||||||
|
}
|
||||||
|
log.info("candidate query (domain={}, queryMode={}) with score={}",
|
||||||
|
semanticParse.getDomainName(), semanticParse.getQueryMode(), score);
|
||||||
|
}
|
||||||
|
|
||||||
|
return pickedQuery;
|
||||||
|
}
|
||||||
|
|
||||||
|
private double computeScore(SemanticParseInfo semanticParse) {
|
||||||
|
double score = 0;
|
||||||
|
|
||||||
|
Map<SchemaElementType, SchemaElementMatch> maxSimilarityMatch = new HashMap<>();
|
||||||
|
for (SchemaElementMatch match : semanticParse.getElementMatches()) {
|
||||||
|
SchemaElementType type = match.getElementType();
|
||||||
|
if (!maxSimilarityMatch.containsKey(type) ||
|
||||||
|
match.getSimilarity() > maxSimilarityMatch.get(type).getSimilarity()) {
|
||||||
|
maxSimilarityMatch.put(type, match);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (SchemaElementMatch match : maxSimilarityMatch.values()) {
|
||||||
|
score +=
|
||||||
|
Optional.ofNullable(match.getDetectWord()).orElse(Constants.EMPTY).length() * match.getSimilarity();
|
||||||
|
}
|
||||||
|
|
||||||
|
// bonus is a special construct to control the final score
|
||||||
|
score += semanticParse.getBonus();
|
||||||
|
|
||||||
|
return score;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||||
|
import com.tencent.supersonic.chat.application.DomainEntityService;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||||
|
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class LLMSemanticQuery implements SemanticQuery {
|
||||||
|
|
||||||
|
public static String QUERY_MODE = "DSL";
|
||||||
|
|
||||||
|
private SemanticParseInfo semanticParse = new SemanticParseInfo();
|
||||||
|
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getQueryMode() {
|
||||||
|
return QUERY_MODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryResultResp execute(User user) {
|
||||||
|
String queryMode = semanticParse.getQueryMode();
|
||||||
|
|
||||||
|
if (semanticParse.getDomainId() < 0 || StringUtils.isEmpty(queryMode)) {
|
||||||
|
// reach here some error may happen
|
||||||
|
log.error("not find QueryMode");
|
||||||
|
throw new RuntimeException("not find QueryMode");
|
||||||
|
}
|
||||||
|
QueryResultResp queryResponse = new QueryResultResp();
|
||||||
|
QueryResultWithSchemaResp queryResult = semanticLayer.queryBySql(
|
||||||
|
SchemaInfoConverter.convertToQuerySqlReq(semanticParse), user);
|
||||||
|
|
||||||
|
if (queryResult != null) {
|
||||||
|
queryResponse.setQueryAuthorization(queryResult.getQueryAuthorization());
|
||||||
|
}
|
||||||
|
String sql = queryResult == null ? null : queryResult.getSql();
|
||||||
|
List<Map<String, Object>> resultList = queryResult == null ? new ArrayList<>()
|
||||||
|
: queryResult.getResultList();
|
||||||
|
List<QueryColumn> columns = queryResult == null ? new ArrayList<>() : queryResult.getColumns();
|
||||||
|
queryResponse.setQuerySql(sql);
|
||||||
|
queryResponse.setQueryResults(resultList);
|
||||||
|
queryResponse.setQueryColumns(columns);
|
||||||
|
queryResponse.setQueryMode(queryMode);
|
||||||
|
|
||||||
|
// add domain info
|
||||||
|
EntityInfo entityInfo = ContextUtils.getBean(DomainEntityService.class)
|
||||||
|
.getEntityInfo(semanticParse, user);
|
||||||
|
queryResponse.setEntityInfo(entityInfo);
|
||||||
|
return queryResponse;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SemanticParseInfo getParseInfo() {
|
||||||
|
return semanticParse;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,39 +1,43 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
|
||||||
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
|
||||||
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||||
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_MOST;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.OPTIONAL;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
import com.tencent.supersonic.chat.api.pojo.Filter;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Service
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class MetricCompare extends BaseSemanticQuery {
|
@Component
|
||||||
|
public class MetricCompare extends MetricSemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "METRIC_COMPARE";
|
public static String QUERY_MODE = "METRIC_COMPARE";
|
||||||
|
public static Pattern intentWordPattern = Pattern.compile("(?i)(比较|对比)");
|
||||||
|
|
||||||
public MetricCompare() {
|
public MetricCompare() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.optional());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.optional());
|
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 2)
|
||||||
queryModeOption.setDimension(QueryModeElementOption.unused());
|
.addOption(ENTITY, OPTIONAL, AT_MOST, 1);
|
||||||
queryModeOption.setFilter(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setMetric(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
queryMatcher.setSupportCompare(true);
|
||||||
queryModeOption.setEntity(QueryModeElementOption.unused());
|
queryMatcher.setSupportOrderBy(true);
|
||||||
queryModeOption.setDomain(QueryModeElementOption.optional());
|
|
||||||
queryModeOption.setSupportCompare(true);
|
|
||||||
queryModeOption.setSupportOrderBy(true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -42,26 +46,22 @@ public class MetricCompare extends BaseSemanticQuery {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCt) {
|
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches, QueryContextReq queryCtx) {
|
||||||
SemanticParseInfo semanticParseInfo = chatCt.getParseInfo();
|
if (intentWordPattern.matcher(queryCtx.getQueryText()).find()) {
|
||||||
ContextHelper.updateTime(queryCtx.getParseInfo(), semanticParseInfo);
|
return super.match(candidateElementMatches, queryCtx);
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
} else {
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
return new ArrayList<>();
|
||||||
ContextHelper.addIfEmpty(queryCtx.getParseInfo().getDimensionFilters(),
|
}
|
||||||
semanticParseInfo.getDimensionFilters());
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||||
ContextHelper.updateTimeIfEmpty(chatCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||||
mergeAppend(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters());
|
mergeAppend(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||||
addCompareDimension(semanticParseInfo);
|
addCompareDimension(parseInfo);
|
||||||
return semanticParseInfo;
|
parseInfo.setBonus(2 * 1.0);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addCompareDimension(SemanticParseInfo semanticParseInfo) {
|
private void addCompareDimension(SemanticParseInfo semanticParseInfo) {
|
||||||
|
|||||||
@@ -1,26 +1,23 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Service
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||||
public class MetricDomain extends BaseSemanticQuery {
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class MetricDomain extends MetricSemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "METRIC_DOMAIN";
|
public static String QUERY_MODE = "METRIC_DOMAIN";
|
||||||
|
|
||||||
public MetricDomain() {
|
public MetricDomain() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.optional());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.optional());
|
queryMatcher.addOption(DOMAIN, REQUIRED, AT_LEAST, 1);
|
||||||
queryModeOption.setDimension(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setFilter(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setMetric(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setEntity(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setDomain(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -29,22 +26,10 @@ public class MetricDomain extends BaseSemanticQuery {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
SemanticParseInfo semanticParseInfo = chatCtx.getParseInfo();
|
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||||
ContextHelper.updateTime(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
|
||||||
ContextHelper.updateTimeIfEmpty(chatCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,25 +1,25 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Service
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||||
public class MetricFilter extends BaseSemanticQuery {
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class MetricFilter extends MetricSemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "METRIC_FILTER";
|
public static String QUERY_MODE = "METRIC_FILTER";
|
||||||
|
|
||||||
public MetricFilter() {
|
public MetricFilter() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.optional());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.optional());
|
queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1)
|
||||||
queryModeOption.setDimension(QueryModeElementOption.unused());
|
.addOption(ENTITY, OPTIONAL, AT_MOST, 1);
|
||||||
queryModeOption.setFilter(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setMetric(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setEntity(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setDomain(QueryModeElementOption.optional());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -28,25 +28,10 @@ public class MetricFilter extends BaseSemanticQuery {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
|
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||||
SemanticParseInfo semanticParseInfo = chatCtx.getParseInfo();
|
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||||
ContextHelper.updateTime(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getDimensionFilters(),
|
|
||||||
semanticParseInfo.getDimensionFilters());
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
|
||||||
ContextHelper.updateTimeIfEmpty(chatCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getDimensionFilters(), semanticParseInfo.getDimensionFilters());
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,26 +2,21 @@ package com.tencent.supersonic.chat.application.query;
|
|||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Service
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
|
||||||
public class MetricGroupBy extends BaseSemanticQuery {
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class MetricGroupBy extends MetricSemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "METRIC_GROUPBY";
|
public static String QUERY_MODE = "METRIC_GROUPBY";
|
||||||
|
|
||||||
|
|
||||||
public MetricGroupBy() {
|
public MetricGroupBy() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.optional());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.optional());
|
queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1);
|
||||||
queryModeOption.setDimension(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST,
|
|
||||||
1);
|
|
||||||
queryModeOption.setFilter(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setMetric(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setEntity(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setDomain(QueryModeElementOption.optional());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -30,24 +25,11 @@ public class MetricGroupBy extends BaseSemanticQuery {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
SemanticParseInfo semanticParseInfo = chatCtx.getParseInfo();
|
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||||
ContextHelper.updateTime(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getDimensions(), parseInfo.getDimensions());
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getDimensions(), semanticParseInfo.getDimensions());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
|
||||||
ContextHelper.updateTimeIfEmpty(chatCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getDimensions(), semanticParseInfo.getDimensions());
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,27 +1,18 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@Service
|
@Component
|
||||||
public class MetricOrderBy extends BaseSemanticQuery {
|
public class MetricOrderBy extends RuleSemanticQuery {
|
||||||
|
|
||||||
public static String QUERY_MODE = "METRIC_ORDERBY";
|
public static String QUERY_MODE = "METRIC_ORDERBY";
|
||||||
|
|
||||||
public MetricOrderBy() {
|
public MetricOrderBy() {
|
||||||
queryModeOption.setAggregation(QueryModeElementOption.optional());
|
super();
|
||||||
queryModeOption.setDate(QueryModeElementOption.optional());
|
|
||||||
queryModeOption.setDimension(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST,
|
|
||||||
1);
|
|
||||||
queryModeOption.setFilter(QueryModeElementOption.optional());
|
|
||||||
queryModeOption.setMetric(SchemaElementOption.REQUIRED, QueryModeElementOption.RequireNumberType.AT_LEAST, 1);
|
|
||||||
queryModeOption.setEntity(QueryModeElementOption.unused());
|
|
||||||
queryModeOption.setDomain(QueryModeElementOption.optional());
|
|
||||||
queryModeOption.setSupportOrderBy(true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -30,24 +21,12 @@ public class MetricOrderBy extends BaseSemanticQuery {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SemanticParseInfo getParseInfo(QueryContextReq queryCtx, ChatContext chatCtx) {
|
public void inheritContext(ChatContext chatContext) {
|
||||||
SemanticParseInfo semanticParseInfo = chatCtx.getParseInfo();
|
SemanticParseInfo chatParseInfo = chatContext.getParseInfo();
|
||||||
ContextHelper.updateTime(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.updateTimeIfEmpty(chatParseInfo, parseInfo);
|
||||||
ContextHelper.updateDomain(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getMetrics(), parseInfo.getMetrics());
|
||||||
ContextHelper.updateSemanticQuery(queryCtx.getParseInfo(), semanticParseInfo);
|
ContextHelper.addIfEmpty(chatParseInfo.getDimensions(), parseInfo.getDimensions());
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
ContextHelper.addIfEmpty(chatParseInfo.getDimensionFilters(), parseInfo.getDimensionFilters());
|
||||||
ContextHelper.updateList(queryCtx.getParseInfo().getDimensions(), semanticParseInfo.getDimensions());
|
|
||||||
ContextHelper.updateEntity(queryCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public SemanticParseInfo getContext(ChatContext chatCtx, QueryContextReq queryCtx) {
|
|
||||||
SemanticParseInfo semanticParseInfo = queryCtx.getParseInfo();
|
|
||||||
ContextHelper.updateTimeIfEmpty(chatCtx.getParseInfo(), semanticParseInfo);
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getMetrics(), semanticParseInfo.getMetrics());
|
|
||||||
ContextHelper.addIfEmpty(chatCtx.getParseInfo().getDimensions(), semanticParseInfo.getDimensions());
|
|
||||||
return semanticParseInfo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
|
||||||
|
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||||
|
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
|
||||||
|
|
||||||
|
public abstract class MetricSemanticQuery extends RuleSemanticQuery {
|
||||||
|
|
||||||
|
public MetricSemanticQuery() {
|
||||||
|
super();
|
||||||
|
queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class QueryMatchOption {
|
||||||
|
|
||||||
|
private SchemaElementOption schemaElementOption;
|
||||||
|
private RequireNumberType requireNumberType;
|
||||||
|
private Integer requireNumber;
|
||||||
|
|
||||||
|
public static QueryMatchOption build(SchemaElementOption schemaElementOption,
|
||||||
|
RequireNumberType requireNumberType, Integer requireNumber) {
|
||||||
|
QueryMatchOption queryMatchOption = new QueryMatchOption();
|
||||||
|
queryMatchOption.requireNumber = requireNumber;
|
||||||
|
queryMatchOption.requireNumberType = requireNumberType;
|
||||||
|
queryMatchOption.schemaElementOption = schemaElementOption;
|
||||||
|
return queryMatchOption;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static QueryMatchOption optional() {
|
||||||
|
QueryMatchOption queryMatchOption = new QueryMatchOption();
|
||||||
|
queryMatchOption.setSchemaElementOption(SchemaElementOption.OPTIONAL);
|
||||||
|
queryMatchOption.setRequireNumber(0);
|
||||||
|
queryMatchOption.setRequireNumberType(RequireNumberType.AT_LEAST);
|
||||||
|
return queryMatchOption;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static QueryMatchOption unused() {
|
||||||
|
QueryMatchOption queryMatchOption = new QueryMatchOption();
|
||||||
|
queryMatchOption.setSchemaElementOption(SchemaElementOption.UNUSED);
|
||||||
|
queryMatchOption.setRequireNumber(0);
|
||||||
|
queryMatchOption.setRequireNumberType(RequireNumberType.EQUAL);
|
||||||
|
return queryMatchOption;
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum RequireNumberType {
|
||||||
|
AT_MOST, AT_LEAST, EQUAL
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
|
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
||||||
|
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.ToString;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@ToString
|
||||||
|
public class QueryMatcher {
|
||||||
|
|
||||||
|
private HashMap<SchemaElementType, QueryMatchOption> elementOptionMap = new HashMap<>();
|
||||||
|
private boolean supportCompare;
|
||||||
|
private boolean supportOrderBy;
|
||||||
|
private List<AggregateTypeEnum> orderByTypes = Arrays.asList(AggregateTypeEnum.MAX, AggregateTypeEnum.MIN,
|
||||||
|
AggregateTypeEnum.TOPN);
|
||||||
|
|
||||||
|
public QueryMatcher() {
|
||||||
|
for (SchemaElementType type : SchemaElementType.values()) {
|
||||||
|
if (type.equals(SchemaElementType.DOMAIN)) {
|
||||||
|
elementOptionMap.put(type, QueryMatchOption.optional());
|
||||||
|
} else {
|
||||||
|
elementOptionMap.put(type, QueryMatchOption.unused());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueryMatcher addOption(SchemaElementType type, SchemaElementOption option,
|
||||||
|
QueryMatchOption.RequireNumberType requireNumberType, Integer requireNumber) {
|
||||||
|
elementOptionMap.put(type, QueryMatchOption.build(option, requireNumberType, requireNumber));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Match schema element with current query according to the options.
|
||||||
|
*
|
||||||
|
* @param candidateElementMatches
|
||||||
|
* @return a list of all matched schema elements,
|
||||||
|
* empty list if no matches can be found
|
||||||
|
*/
|
||||||
|
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches) {
|
||||||
|
List<SchemaElementMatch> elementMatches = new ArrayList<>();
|
||||||
|
|
||||||
|
HashMap<SchemaElementType, Integer> schemaElementTypeCount = new HashMap<>();
|
||||||
|
for (SchemaElementMatch schemaElementMatch : candidateElementMatches) {
|
||||||
|
SchemaElementType schemaElementType = schemaElementMatch.getElementType();
|
||||||
|
if (schemaElementTypeCount.containsKey(schemaElementType)) {
|
||||||
|
schemaElementTypeCount.put(schemaElementType, schemaElementTypeCount.get(schemaElementType) + 1);
|
||||||
|
} else {
|
||||||
|
schemaElementTypeCount.put(schemaElementType, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// check if current query options are satisfied, return immediately if not
|
||||||
|
for (Map.Entry<SchemaElementType, QueryMatchOption> e : elementOptionMap.entrySet()) {
|
||||||
|
SchemaElementType elementType = e.getKey();
|
||||||
|
QueryMatchOption elementOption = e.getValue();
|
||||||
|
if (!isMatch(elementOption, getCount(schemaElementTypeCount, elementType))) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// add element match if its element type is not declared as unused
|
||||||
|
for (SchemaElementMatch elementMatch : candidateElementMatches) {
|
||||||
|
QueryMatchOption elementOption = elementOptionMap.get(elementMatch.getElementType());
|
||||||
|
if (Objects.nonNull(elementOption) && !elementOption.getSchemaElementOption()
|
||||||
|
.equals(SchemaElementOption.UNUSED)) {
|
||||||
|
elementMatches.add(elementMatch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return elementMatches;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int getCount(HashMap<SchemaElementType, Integer> schemaElementTypeCount,
|
||||||
|
SchemaElementType schemaElementType) {
|
||||||
|
if (schemaElementTypeCount.containsKey(schemaElementType)) {
|
||||||
|
return schemaElementTypeCount.get(schemaElementType);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isMatch(QueryMatchOption queryMatchOption, int count) {
|
||||||
|
// check if required but empty
|
||||||
|
if (queryMatchOption.getSchemaElementOption().equals(SchemaElementOption.REQUIRED) && count <= 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (queryMatchOption.getRequireNumberType().equals(QueryMatchOption.RequireNumberType.AT_LEAST)
|
||||||
|
&& count < queryMatchOption.getRequireNumber()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (queryMatchOption.getRequireNumberType().equals(QueryMatchOption.RequireNumberType.AT_MOST)
|
||||||
|
&& count > queryMatchOption.getRequireNumber()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import lombok.Data;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
public class QueryModeElementOption {
|
|
||||||
|
|
||||||
public enum RequireNumberType {
|
|
||||||
AT_MOST, AT_LEAST, EQUAL
|
|
||||||
}
|
|
||||||
|
|
||||||
private SchemaElementOption schemaElementOption;
|
|
||||||
private RequireNumberType requireNumberType;
|
|
||||||
private Integer requireNumber;
|
|
||||||
|
|
||||||
public static QueryModeElementOption build(SchemaElementOption schemaElementOption,
|
|
||||||
RequireNumberType requireNumberType, Integer requireNumber) {
|
|
||||||
QueryModeElementOption queryModeElementOption = new QueryModeElementOption();
|
|
||||||
queryModeElementOption.requireNumber = requireNumber;
|
|
||||||
queryModeElementOption.requireNumberType = requireNumberType;
|
|
||||||
queryModeElementOption.schemaElementOption = schemaElementOption;
|
|
||||||
return queryModeElementOption;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static QueryModeElementOption optional() {
|
|
||||||
QueryModeElementOption queryModeElementOption = new QueryModeElementOption();
|
|
||||||
queryModeElementOption.setSchemaElementOption(SchemaElementOption.OPTIONAL);
|
|
||||||
queryModeElementOption.setRequireNumber(0);
|
|
||||||
queryModeElementOption.setRequireNumberType(RequireNumberType.AT_LEAST);
|
|
||||||
return queryModeElementOption;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static QueryModeElementOption unused() {
|
|
||||||
QueryModeElementOption queryModeElementOption = new QueryModeElementOption();
|
|
||||||
queryModeElementOption.setSchemaElementOption(SchemaElementOption.UNUSED);
|
|
||||||
queryModeElementOption.setRequireNumber(0);
|
|
||||||
queryModeElementOption.setRequireNumberType(RequireNumberType.EQUAL);
|
|
||||||
return queryModeElementOption;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,307 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementCount;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
|
||||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
|
||||||
import com.tencent.supersonic.chat.application.parser.resolver.AggregateTypeResolver;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption;
|
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
|
|
||||||
import com.tencent.supersonic.common.enums.AggregateTypeEnum;
|
|
||||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
|
||||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.ToString;
|
|
||||||
|
|
||||||
@Data
|
|
||||||
@ToString
|
|
||||||
public class QueryModeOption {
|
|
||||||
|
|
||||||
private QueryModeElementOption domain;
|
|
||||||
private QueryModeElementOption entity;
|
|
||||||
private QueryModeElementOption metric;
|
|
||||||
private QueryModeElementOption dimension;
|
|
||||||
private QueryModeElementOption filter;
|
|
||||||
private QueryModeElementOption date;
|
|
||||||
|
|
||||||
private QueryModeElementOption aggregation;
|
|
||||||
private boolean supportCompare = false;
|
|
||||||
private boolean supportOrderBy = false;
|
|
||||||
List<AggregateTypeEnum> orderByTypes = Arrays.asList(AggregateTypeEnum.MAX, AggregateTypeEnum.MIN,
|
|
||||||
AggregateTypeEnum.TOPN);
|
|
||||||
|
|
||||||
public static QueryModeOption build() {
|
|
||||||
return new QueryModeOption();
|
|
||||||
}
|
|
||||||
|
|
||||||
public QueryModeOption setDimension(SchemaElementOption dimension,
|
|
||||||
QueryModeElementOption.RequireNumberType requireNumberType, Integer requireNumber) {
|
|
||||||
this.dimension = QueryModeElementOption.build(dimension, requireNumberType, requireNumber);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public QueryModeOption setDomain(SchemaElementOption domain,
|
|
||||||
QueryModeElementOption.RequireNumberType requireNumberType, Integer requireNumber) {
|
|
||||||
this.domain = QueryModeElementOption.build(domain, requireNumberType, requireNumber);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public QueryModeOption setEntity(SchemaElementOption entity,
|
|
||||||
QueryModeElementOption.RequireNumberType requireNumberType, Integer requireNumber) {
|
|
||||||
this.entity = QueryModeElementOption.build(entity, requireNumberType, requireNumber);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public QueryModeOption setFilter(SchemaElementOption filter,
|
|
||||||
QueryModeElementOption.RequireNumberType requireNumberType, Integer requireNumber) {
|
|
||||||
this.filter = QueryModeElementOption.build(filter, requireNumberType, requireNumber);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public QueryModeOption setMetric(SchemaElementOption metric,
|
|
||||||
QueryModeElementOption.RequireNumberType requireNumberType, Integer requireNumber) {
|
|
||||||
this.metric = QueryModeElementOption.build(metric, requireNumberType, requireNumber);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* add query semantic parse info
|
|
||||||
* @param schemaElementMatches
|
|
||||||
* @param domainSchemaDesc
|
|
||||||
* @param chaConfigRichDesc
|
|
||||||
* @param semanticParseInfo
|
|
||||||
*/
|
|
||||||
public void addQuerySemanticParseInfo(List<SchemaElementMatch> schemaElementMatches,
|
|
||||||
DomainSchemaResp domainSchemaDesc, ChatConfigRichInfo chaConfigRichDesc,
|
|
||||||
SemanticParseInfo semanticParseInfo) {
|
|
||||||
Map<Long, DimSchemaResp> dimensionDescMap = domainSchemaDesc.getDimensions().stream()
|
|
||||||
.collect(Collectors.toMap(DimSchemaResp::getId, Function.identity()));
|
|
||||||
Map<Long, MetricSchemaResp> metricDescMap = domainSchemaDesc.getMetrics().stream()
|
|
||||||
.collect(Collectors.toMap(MetricSchemaResp::getId, Function.identity()));
|
|
||||||
|
|
||||||
Map<Long, List<SchemaElementMatch>> values = getLinkSchemaElementMatch(schemaElementMatches, dimensionDescMap,
|
|
||||||
semanticParseInfo, metricDescMap);
|
|
||||||
if (!values.isEmpty()) {
|
|
||||||
for (Map.Entry<Long, List<SchemaElementMatch>> entry : values.entrySet()) {
|
|
||||||
DimSchemaResp dimensionDesc = dimensionDescMap.get(entry.getKey());
|
|
||||||
if (entry.getValue().size() == 1) {
|
|
||||||
SchemaElementMatch schemaElementMatch = entry.getValue().get(0);
|
|
||||||
Filter chatFilter = new Filter();
|
|
||||||
chatFilter.setValue(schemaElementMatch.getWord());
|
|
||||||
chatFilter.setBizName(dimensionDesc.getBizName());
|
|
||||||
chatFilter.setName(dimensionDesc.getName());
|
|
||||||
chatFilter.setOperator(FilterOperatorEnum.EQUALS);
|
|
||||||
chatFilter.setElementID(Long.valueOf(schemaElementMatch.getElementID()));
|
|
||||||
semanticParseInfo.getDimensionFilters().add(chatFilter);
|
|
||||||
ContextHelper.setEntityId(entry.getKey(), schemaElementMatch.getWord(), chaConfigRichDesc,
|
|
||||||
semanticParseInfo);
|
|
||||||
} else {
|
|
||||||
Filter chatFilter = new Filter();
|
|
||||||
List<String> vals = new ArrayList<>();
|
|
||||||
entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
|
|
||||||
chatFilter.setValue(vals);
|
|
||||||
chatFilter.setBizName(dimensionDesc.getBizName());
|
|
||||||
chatFilter.setName(dimensionDesc.getName());
|
|
||||||
chatFilter.setOperator(FilterOperatorEnum.IN);
|
|
||||||
chatFilter.setElementID(entry.getKey());
|
|
||||||
semanticParseInfo.getDimensionFilters().add(chatFilter);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<Long, List<SchemaElementMatch>> getLinkSchemaElementMatch(List<SchemaElementMatch> schemaElementMatches,
|
|
||||||
Map<Long, DimSchemaResp> dimensionDescMap, SemanticParseInfo semanticParseInfo,
|
|
||||||
Map<Long, MetricSchemaResp> metricDescMap) {
|
|
||||||
Map<Long, List<SchemaElementMatch>> values = new HashMap<>();
|
|
||||||
for (SchemaElementMatch schemaElementMatch : schemaElementMatches) {
|
|
||||||
Long elementID = Long.valueOf(schemaElementMatch.getElementID());
|
|
||||||
switch (schemaElementMatch.getElementType()) {
|
|
||||||
case DATE:
|
|
||||||
case DOMAIN:
|
|
||||||
case ENTITY:
|
|
||||||
break;
|
|
||||||
case ID:
|
|
||||||
case VALUE:
|
|
||||||
case DIMENSION:
|
|
||||||
if (dimensionDescMap.containsKey(elementID)) {
|
|
||||||
DimSchemaResp dimensionDesc = dimensionDescMap.get(elementID);
|
|
||||||
SchemaItem dimensionParseInfo = new SchemaItem();
|
|
||||||
dimensionParseInfo.setBizName(dimensionDesc.getBizName());
|
|
||||||
dimensionParseInfo.setName(dimensionDesc.getName());
|
|
||||||
dimensionParseInfo.setId(dimensionDesc.getId());
|
|
||||||
if (!dimension.getSchemaElementOption().equals(SchemaElementOption.UNUSED)
|
|
||||||
&& schemaElementMatch.getElementType().equals(SchemaElementType.DIMENSION)) {
|
|
||||||
semanticParseInfo.getDimensions().add(dimensionParseInfo);
|
|
||||||
}
|
|
||||||
if (!filter.getSchemaElementOption().equals(SchemaElementOption.UNUSED) && (
|
|
||||||
schemaElementMatch.getElementType().equals(SchemaElementType.VALUE)
|
|
||||||
|| schemaElementMatch.getElementType().equals(SchemaElementType.ID))) {
|
|
||||||
if (values.containsKey(elementID)) {
|
|
||||||
values.get(elementID).add(schemaElementMatch);
|
|
||||||
} else {
|
|
||||||
values.put(elementID, new ArrayList<>(Arrays.asList(schemaElementMatch)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case METRIC:
|
|
||||||
if (!metric.getSchemaElementOption().equals(SchemaElementOption.UNUSED)) {
|
|
||||||
if (metricDescMap.containsKey(elementID)) {
|
|
||||||
MetricSchemaResp metricDesc = metricDescMap.get(elementID);
|
|
||||||
SchemaItem metric = new SchemaItem();
|
|
||||||
metric.setBizName(metricDesc.getBizName());
|
|
||||||
metric.setName(metricDesc.getName());
|
|
||||||
metric.setId(metricDesc.getId());
|
|
||||||
semanticParseInfo.getMetrics().add(metric);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return values;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* math
|
|
||||||
* @param elementMatches
|
|
||||||
* @param queryCtx
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
public SchemaElementCount match(List<SchemaElementMatch> elementMatches, QueryContextReq queryCtx) {
|
|
||||||
AggregateTypeResolver aggregateTypeResolver = ContextUtils.getBean(AggregateTypeResolver.class);
|
|
||||||
|
|
||||||
boolean isCompareType = aggregateTypeResolver.hasCompareIntentionalWords(queryCtx.getQueryText());
|
|
||||||
boolean isOrderByType = orderByTypes.contains(aggregateTypeResolver.resolve(queryCtx.getQueryText()));
|
|
||||||
|
|
||||||
if ((isOrderByType && !supportOrderBy) || (isCompareType && !supportCompare)) {
|
|
||||||
return new SchemaElementCount();
|
|
||||||
}
|
|
||||||
|
|
||||||
SchemaElementCount schemaElementCount = new SchemaElementCount();
|
|
||||||
schemaElementCount.setCount(0);
|
|
||||||
schemaElementCount.setMaxSimilarity(0);
|
|
||||||
HashMap<SchemaElementType, Integer> schemaElementTypeCount = new HashMap<>();
|
|
||||||
for (SchemaElementMatch schemaElementMatch : elementMatches) {
|
|
||||||
SchemaElementType schemaElementType = schemaElementMatch.getElementType();
|
|
||||||
if (schemaElementTypeCount.containsKey(schemaElementType)) {
|
|
||||||
schemaElementTypeCount.put(schemaElementType, schemaElementTypeCount.get(schemaElementType) + 1);
|
|
||||||
} else {
|
|
||||||
schemaElementTypeCount.put(schemaElementType, 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// test each element
|
|
||||||
if (!isMatch(domain, getCount(schemaElementTypeCount, SchemaElementType.DOMAIN))) {
|
|
||||||
return schemaElementCount;
|
|
||||||
}
|
|
||||||
if (!isMatch(dimension, getCount(schemaElementTypeCount, SchemaElementType.DIMENSION))) {
|
|
||||||
return schemaElementCount;
|
|
||||||
}
|
|
||||||
if (!isMatch(metric, getCount(schemaElementTypeCount, SchemaElementType.METRIC))) {
|
|
||||||
return schemaElementCount;
|
|
||||||
}
|
|
||||||
int filterCount = getCount(schemaElementTypeCount, SchemaElementType.VALUE) + getCount(schemaElementTypeCount,
|
|
||||||
SchemaElementType.ID);
|
|
||||||
if (!isMatch(filter, filterCount)) {
|
|
||||||
return schemaElementCount;
|
|
||||||
}
|
|
||||||
if (!isMatch(entity, getCount(schemaElementTypeCount, SchemaElementType.ENTITY))) {
|
|
||||||
return schemaElementCount;
|
|
||||||
}
|
|
||||||
if (!isMatch(date, getCount(schemaElementTypeCount, SchemaElementType.DATE))) {
|
|
||||||
return schemaElementCount;
|
|
||||||
}
|
|
||||||
// count the max similarity
|
|
||||||
double similarity = 0;
|
|
||||||
Set<SchemaElementType> schemaElementTypeSet = new HashSet<>();
|
|
||||||
for (SchemaElementMatch schemaElementMatch : elementMatches) {
|
|
||||||
double schemaElementMatchSimilarity = getSimilarity(schemaElementMatch);
|
|
||||||
if (schemaElementMatchSimilarity > similarity) {
|
|
||||||
similarity = schemaElementMatchSimilarity;
|
|
||||||
}
|
|
||||||
schemaElementTypeSet.add(schemaElementMatch.getElementType());
|
|
||||||
}
|
|
||||||
schemaElementCount.setCount(schemaElementTypeSet.size());
|
|
||||||
schemaElementCount.setMaxSimilarity(similarity);
|
|
||||||
return schemaElementCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
private int getCount(HashMap<SchemaElementType, Integer> schemaElementTypeCount,
|
|
||||||
SchemaElementType schemaElementType) {
|
|
||||||
if (schemaElementTypeCount.containsKey(schemaElementType)) {
|
|
||||||
return schemaElementTypeCount.get(schemaElementType);
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private double getSimilarity(SchemaElementMatch schemaElementMatch) {
|
|
||||||
switch (schemaElementMatch.getElementType()) {
|
|
||||||
case DATE:
|
|
||||||
return getSimilarity(date, schemaElementMatch.getSimilarity());
|
|
||||||
case DOMAIN:
|
|
||||||
return getSimilarity(domain, schemaElementMatch.getSimilarity());
|
|
||||||
case ENTITY:
|
|
||||||
return getSimilarity(entity, schemaElementMatch.getSimilarity());
|
|
||||||
case DIMENSION:
|
|
||||||
return getSimilarity(dimension, schemaElementMatch.getSimilarity());
|
|
||||||
case METRIC:
|
|
||||||
return getSimilarity(metric, schemaElementMatch.getSimilarity());
|
|
||||||
case ID:
|
|
||||||
case VALUE:
|
|
||||||
return getSimilarity(filter, schemaElementMatch.getSimilarity());
|
|
||||||
default:
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private double getSimilarity(QueryModeElementOption queryModeElementOption, double similarity) {
|
|
||||||
if (queryModeElementOption.getSchemaElementOption().equals(SchemaElementOption.REQUIRED)) {
|
|
||||||
return similarity;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isMatch(QueryModeElementOption queryModeElementOption, int count) {
|
|
||||||
// first find if unused but not empty
|
|
||||||
if (queryModeElementOption.getSchemaElementOption().equals(SchemaElementOption.UNUSED) && count > 0) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// find if required but empty
|
|
||||||
if (queryModeElementOption.getSchemaElementOption().equals(SchemaElementOption.REQUIRED) && count <= 0) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// find if count no satisfy
|
|
||||||
if (queryModeElementOption.getRequireNumberType().equals(QueryModeElementOption.RequireNumberType.EQUAL)
|
|
||||||
&& queryModeElementOption.getRequireNumber() != count) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (queryModeElementOption.getRequireNumberType().equals(QueryModeElementOption.RequireNumberType.AT_LEAST)
|
|
||||||
&& count < queryModeElementOption.getRequireNumber()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (queryModeElementOption.getRequireNumberType().equals(QueryModeElementOption.RequireNumberType.AT_MOST)
|
|
||||||
&& count > queryModeElementOption.getRequireNumber()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// here default satisfy
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This interface defines the contract for a selector that picks the most suitable semantic query.
|
||||||
|
**/
|
||||||
|
public interface QuerySelector {
|
||||||
|
|
||||||
|
SemanticQuery select(List<SemanticQuery> candidateQueries);
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.*;
|
||||||
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
|
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||||
|
import com.tencent.supersonic.chat.application.DomainEntityService;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
|
||||||
|
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
|
||||||
|
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
|
||||||
|
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||||
|
import lombok.ToString;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@ToString
|
||||||
|
public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
|
||||||
|
|
||||||
|
protected SemanticParseInfo parseInfo = new SemanticParseInfo();
|
||||||
|
protected List<SchemaElementMatch> schemaElementMatches = new ArrayList<>();
|
||||||
|
protected QueryMatcher queryMatcher = new QueryMatcher();
|
||||||
|
protected SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
|
|
||||||
|
public RuleSemanticQuery() {
|
||||||
|
RuleSemanticQueryManager.register(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<SchemaElementMatch> match(List<SchemaElementMatch> candidateElementMatches, QueryContextReq queryCtx) {
|
||||||
|
return queryMatcher.match(candidateElementMatches);
|
||||||
|
}
|
||||||
|
|
||||||
|
public abstract void inheritContext(ChatContext chatContext);
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryResultResp execute(User user) {
|
||||||
|
String queryMode = parseInfo.getQueryMode();
|
||||||
|
|
||||||
|
if (parseInfo.getDomainId() < 0 || StringUtils.isEmpty(queryMode)) {
|
||||||
|
// reach here some error may happen
|
||||||
|
log.error("not find QueryMode");
|
||||||
|
throw new RuntimeException("not find QueryMode");
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> semanticQueryModes = RuleSemanticQueryManager.getSemanticQueryModes();
|
||||||
|
if (!semanticQueryModes.contains(parseInfo.getQueryMode())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
QueryResultResp queryResponse = new QueryResultResp();
|
||||||
|
QueryResultWithSchemaResp queryResult = semanticLayer.queryByStruct(
|
||||||
|
SchemaInfoConverter.convertTo(parseInfo), user);
|
||||||
|
|
||||||
|
if (queryResult != null) {
|
||||||
|
queryResponse.setQueryAuthorization(queryResult.getQueryAuthorization());
|
||||||
|
}
|
||||||
|
String sql = queryResult == null ? null : queryResult.getSql();
|
||||||
|
List<Map<String, Object>> resultList = queryResult == null ? new ArrayList<>()
|
||||||
|
: queryResult.getResultList();
|
||||||
|
List<QueryColumn> columns = queryResult == null ? new ArrayList<>() : queryResult.getColumns();
|
||||||
|
queryResponse.setQuerySql(sql);
|
||||||
|
queryResponse.setQueryResults(resultList);
|
||||||
|
queryResponse.setQueryColumns(columns);
|
||||||
|
queryResponse.setQueryMode(queryMode);
|
||||||
|
|
||||||
|
// add domain info
|
||||||
|
EntityInfo entityInfo = ContextUtils.getBean(DomainEntityService.class)
|
||||||
|
.getEntityInfo(parseInfo, user);
|
||||||
|
queryResponse.setEntityInfo(entityInfo);
|
||||||
|
return queryResponse;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SemanticParseInfo getParseInfo() {
|
||||||
|
return parseInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setParseInfo(SemanticParseInfo parseInfo) {
|
||||||
|
this.parseInfo = parseInfo;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package com.tencent.supersonic.chat.application.query;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* RuleSemanticQueryManager
|
||||||
|
*/
|
||||||
|
public class RuleSemanticQueryManager {
|
||||||
|
|
||||||
|
private static Map<String, RuleSemanticQuery> semanticQueryMap = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
public static RuleSemanticQuery create(String queryMode) {
|
||||||
|
RuleSemanticQuery semanticQuery = semanticQueryMap.get(queryMode);
|
||||||
|
if (Objects.isNull(semanticQuery)) {
|
||||||
|
throw new RuntimeException("no supported queryMode :" + queryMode);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return semanticQuery.getClass().getDeclaredConstructor().newInstance();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException("no supported queryMode :" + queryMode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void register(RuleSemanticQuery query) {
|
||||||
|
semanticQueryMap.put(query.getQueryMode(), query);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<RuleSemanticQuery> getSemanticQueries() {
|
||||||
|
return new ArrayList<>(semanticQueryMap.values());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<String> getSemanticQueryModes() {
|
||||||
|
return new ArrayList<>(semanticQueryMap.keySet());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
package com.tencent.supersonic.chat.application.query;
|
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import org.springframework.core.io.support.SpringFactoriesLoader;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* SemanticQueryFactory
|
|
||||||
*/
|
|
||||||
public class SemanticQueryFactory {
|
|
||||||
|
|
||||||
private static Map<String, SemanticQuery> strategyFactory = new ConcurrentHashMap<>();
|
|
||||||
|
|
||||||
private static List<SemanticQuery> semanticQueries;
|
|
||||||
|
|
||||||
|
|
||||||
public static SemanticQuery get(String queryMode) {
|
|
||||||
if (CollectionUtils.isEmpty(strategyFactory)) {
|
|
||||||
init();
|
|
||||||
}
|
|
||||||
|
|
||||||
SemanticQuery semanticQuery = strategyFactory.get(queryMode);
|
|
||||||
if (Objects.isNull(semanticQuery)) {
|
|
||||||
throw new RuntimeException("not support queryMode :" + queryMode);
|
|
||||||
}
|
|
||||||
return semanticQuery;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void init() {
|
|
||||||
for (SemanticQuery semanticQuery : getSemanticQueries()) {
|
|
||||||
strategyFactory.put(semanticQuery.getQueryMode(), semanticQuery);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<SemanticQuery> getSemanticQueries() {
|
|
||||||
if (CollectionUtils.isEmpty(semanticQueries)) {
|
|
||||||
semanticQueries = SpringFactoriesLoader.loadFactories(SemanticQuery.class,
|
|
||||||
Thread.currentThread().getContextClassLoader());
|
|
||||||
}
|
|
||||||
return semanticQueries;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package com.tencent.supersonic.chat.domain.config;
|
||||||
|
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
@Data
|
||||||
|
public class LLMConfig {
|
||||||
|
|
||||||
|
|
||||||
|
@Value("${llm.url:}")
|
||||||
|
private String url;
|
||||||
|
|
||||||
|
@Value("${query2sql.path:query2sql}")
|
||||||
|
private String queryToSqlPath;
|
||||||
|
|
||||||
|
|
||||||
|
@Value("${query2sql.endpoint:}")
|
||||||
|
private String endpoint;
|
||||||
|
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import java.util.Date;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public class ChatQueryDOExample {
|
public class ChatQueryDOExample {
|
||||||
|
|
||||||
protected String orderByClause;
|
protected String orderByClause;
|
||||||
protected boolean distinct;
|
protected boolean distinct;
|
||||||
protected List<Criteria> oredCriteria;
|
protected List<Criteria> oredCriteria;
|
||||||
@@ -15,23 +16,22 @@ public class ChatQueryDOExample {
|
|||||||
oredCriteria = new ArrayList<Criteria>();
|
oredCriteria = new ArrayList<Criteria>();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setOrderByClause(String orderByClause) {
|
|
||||||
this.orderByClause = orderByClause;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public String getOrderByClause() {
|
public String getOrderByClause() {
|
||||||
return orderByClause;
|
return orderByClause;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setDistinct(boolean distinct) {
|
public void setOrderByClause(String orderByClause) {
|
||||||
this.distinct = distinct;
|
this.orderByClause = orderByClause;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isDistinct() {
|
public boolean isDistinct() {
|
||||||
return distinct;
|
return distinct;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setDistinct(boolean distinct) {
|
||||||
|
this.distinct = distinct;
|
||||||
|
}
|
||||||
|
|
||||||
public List<Criteria> getOredCriteria() {
|
public List<Criteria> getOredCriteria() {
|
||||||
return oredCriteria;
|
return oredCriteria;
|
||||||
}
|
}
|
||||||
@@ -65,22 +65,22 @@ public class ChatQueryDOExample {
|
|||||||
distinct = false;
|
distinct = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setLimitStart(Integer limitStart) {
|
|
||||||
this.limitStart = limitStart;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Integer getLimitStart() {
|
public Integer getLimitStart() {
|
||||||
return limitStart;
|
return limitStart;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setLimitEnd(Integer limitEnd) {
|
public void setLimitStart(Integer limitStart) {
|
||||||
this.limitEnd = limitEnd;
|
this.limitStart = limitStart;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Integer getLimitEnd() {
|
public Integer getLimitEnd() {
|
||||||
return limitEnd;
|
return limitEnd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setLimitEnd(Integer limitEnd) {
|
||||||
|
this.limitEnd = limitEnd;
|
||||||
|
}
|
||||||
|
|
||||||
protected abstract static class GeneratedCriteria {
|
protected abstract static class GeneratedCriteria {
|
||||||
|
|
||||||
protected List<Criterion> criteria;
|
protected List<Criterion> criteria;
|
||||||
@@ -589,38 +589,6 @@ public class ChatQueryDOExample {
|
|||||||
|
|
||||||
private String typeHandler;
|
private String typeHandler;
|
||||||
|
|
||||||
public String getCondition() {
|
|
||||||
return condition;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object getValue() {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object getSecondValue() {
|
|
||||||
return secondValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isNoValue() {
|
|
||||||
return noValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isSingleValue() {
|
|
||||||
return singleValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isBetweenValue() {
|
|
||||||
return betweenValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isListValue() {
|
|
||||||
return listValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getTypeHandler() {
|
|
||||||
return typeHandler;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected Criterion(String condition) {
|
protected Criterion(String condition) {
|
||||||
super();
|
super();
|
||||||
this.condition = condition;
|
this.condition = condition;
|
||||||
@@ -656,5 +624,37 @@ public class ChatQueryDOExample {
|
|||||||
protected Criterion(String condition, Object value, Object secondValue) {
|
protected Criterion(String condition, Object value, Object secondValue) {
|
||||||
this(condition, value, secondValue, null);
|
this(condition, value, secondValue, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getCondition() {
|
||||||
|
return condition;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Object getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Object getSecondValue() {
|
||||||
|
return secondValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isNoValue() {
|
||||||
|
return noValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isSingleValue() {
|
||||||
|
return singleValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isBetweenValue() {
|
||||||
|
return betweenValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isListValue() {
|
||||||
|
return listValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTypeHandler() {
|
||||||
|
return typeHandler;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,6 +5,9 @@ import lombok.Data;
|
|||||||
@Data
|
@Data
|
||||||
public class QueryDO {
|
public class QueryDO {
|
||||||
|
|
||||||
|
public String aggregator = "trend";
|
||||||
|
public String startTime;
|
||||||
|
public String endTime;
|
||||||
private long id;
|
private long id;
|
||||||
private long questionId;
|
private long questionId;
|
||||||
private String createTime;
|
private String createTime;
|
||||||
@@ -19,10 +22,7 @@ public class QueryDO {
|
|||||||
private int isDeleted;
|
private int isDeleted;
|
||||||
private String module;
|
private String module;
|
||||||
private long chatId;
|
private long chatId;
|
||||||
public String aggregator = "trend";
|
|
||||||
private int topNum;
|
private int topNum;
|
||||||
public String startTime;
|
|
||||||
public String endTime;
|
|
||||||
private String querySql;
|
private String querySql;
|
||||||
private Object queryColumn;
|
private Object queryColumn;
|
||||||
private Object entityInfo;
|
private Object entityInfo;
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class LLMReq {
|
||||||
|
|
||||||
|
private String queryText;
|
||||||
|
private LLMSchema schema;
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class LLMResp {
|
||||||
|
|
||||||
|
private String query;
|
||||||
|
|
||||||
|
private String domainName;
|
||||||
|
|
||||||
|
private String sqlOutput;
|
||||||
|
|
||||||
|
private List<String> fields;
|
||||||
|
|
||||||
|
private String schemaLinkingOutput;
|
||||||
|
|
||||||
|
private String schemaLinkStr;
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package com.tencent.supersonic.chat.domain.pojo.chat;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class LLMSchema {
|
||||||
|
|
||||||
|
private String domainName;
|
||||||
|
|
||||||
|
private List<String> fieldNameList;
|
||||||
|
|
||||||
|
}
|
||||||
@@ -15,20 +15,20 @@ public class PageQueryInfoReq {
|
|||||||
return pageSize;
|
return pageSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
public int getCurrent() {
|
public void setPageSize(int pageSize) {
|
||||||
return current;
|
this.pageSize = pageSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getUserName() {
|
public int getCurrent() {
|
||||||
return userName;
|
return current;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setCurrent(int current) {
|
public void setCurrent(int current) {
|
||||||
this.current = current;
|
this.current = current;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setPageSize(int pageSize) {
|
public String getUserName() {
|
||||||
this.pageSize = pageSize;
|
return userName;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setUserName(String userName) {
|
public void setUserName(String userName) {
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
package com.tencent.supersonic.chat.domain.pojo.config;
|
package com.tencent.supersonic.chat.domain.pojo.config;
|
||||||
|
|
||||||
import com.tencent.supersonic.common.enums.TypeEnums;
|
import com.tencent.supersonic.common.enums.TypeEnums;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import javax.validation.constraints.NotNull;
|
import javax.validation.constraints.NotNull;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -17,6 +19,7 @@ public class KnowledgeInfo {
|
|||||||
*/
|
*/
|
||||||
private Long itemId;
|
private Long itemId;
|
||||||
|
|
||||||
|
private String bizName;
|
||||||
/**
|
/**
|
||||||
* type: IntentionTypeEnum
|
* type: IntentionTypeEnum
|
||||||
* temporarily only supports dimension-related information
|
* temporarily only supports dimension-related information
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ public interface ChatRepository {
|
|||||||
|
|
||||||
Boolean updateChatName(Long chatId, String chatName, String lastTime, String creator);
|
Boolean updateChatName(Long chatId, String chatName, String lastTime, String creator);
|
||||||
|
|
||||||
|
Boolean updateLastQuestion(Long chatId, String lastQuestion, String lastTime);
|
||||||
|
|
||||||
Boolean updateConversionIsTop(Long chatId, int isTop);
|
Boolean updateConversionIsTop(Long chatId, int isTop);
|
||||||
|
|
||||||
boolean updateFeedback(QueryDO queryDO);
|
boolean updateFeedback(QueryDO queryDO);
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.domain.service;
|
|||||||
import com.github.pagehelper.PageInfo;
|
import com.github.pagehelper.PageInfo;
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
import com.tencent.supersonic.chat.api.response.QueryResultResp;
|
||||||
import com.tencent.supersonic.chat.domain.dataobject.ChatDO;
|
import com.tencent.supersonic.chat.domain.dataobject.ChatDO;
|
||||||
@@ -24,6 +25,8 @@ public interface ChatService {
|
|||||||
|
|
||||||
public void updateContext(ChatContext chatCtx);
|
public void updateContext(ChatContext chatCtx);
|
||||||
|
|
||||||
|
public void updateContext(ChatContext chatCtx, QueryContextReq queryCtx, SemanticParseInfo semanticParseInfo);
|
||||||
|
|
||||||
public void switchContext(ChatContext chatCtx);
|
public void switchContext(ChatContext chatCtx);
|
||||||
|
|
||||||
public Boolean addChat(User user, String chatName);
|
public Boolean addChat(User user, String chatName);
|
||||||
|
|||||||
@@ -18,4 +18,6 @@ public interface ConfigService {
|
|||||||
List<ChatConfigInfo> search(ChatConfigFilter filter, User user);
|
List<ChatConfigInfo> search(ChatConfigFilter filter, User user);
|
||||||
|
|
||||||
ChatConfigRichInfo getConfigRichInfo(Long domainId);
|
ChatConfigRichInfo getConfigRichInfo(Long domainId);
|
||||||
|
|
||||||
|
List<ChatConfigRichInfo> getAllChatRichConfig();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,9 +11,9 @@ import com.tencent.supersonic.chat.domain.pojo.chat.QueryData;
|
|||||||
*/
|
*/
|
||||||
public interface QueryService {
|
public interface QueryService {
|
||||||
|
|
||||||
public QueryResultResp executeQuery(QueryContextReq queryCtx) throws Exception;
|
QueryResultResp executeQuery(QueryContextReq queryCtx) throws Exception;
|
||||||
|
|
||||||
public SemanticParseInfo queryContext(QueryContextReq queryCtx);
|
SemanticParseInfo queryContext(QueryContextReq queryCtx);
|
||||||
|
|
||||||
QueryResultResp queryData(QueryData queryData, User user) throws Exception;
|
QueryResultResp executeDirectQuery(QueryData queryData, User user) throws Exception;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ public class ChatConfigUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public EntityInternalDetail generateEntityDetailData(EntityDetailData detailData,
|
public EntityInternalDetail generateEntityDetailData(EntityDetailData detailData,
|
||||||
DomainSchemaResp domainSchemaDesc) {
|
DomainSchemaResp domainSchemaDesc) {
|
||||||
EntityInternalDetail entityInternalDetailDesc = new EntityInternalDetail();
|
EntityInternalDetail entityInternalDetailDesc = new EntityInternalDetail();
|
||||||
if (Objects.isNull(detailData)) {
|
if (Objects.isNull(detailData)) {
|
||||||
return entityInternalDetailDesc;
|
return entityInternalDetailDesc;
|
||||||
@@ -100,7 +100,7 @@ public class ChatConfigUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public Map<Long, MetricSchemaResp> generateMetricIdAndDescPair(List<Long> metricIds,
|
public Map<Long, MetricSchemaResp> generateMetricIdAndDescPair(List<Long> metricIds,
|
||||||
DomainSchemaResp domainSchemaDesc) {
|
DomainSchemaResp domainSchemaDesc) {
|
||||||
Map<Long, MetricSchemaResp> metricIdAndDescPair = new HashMap<>();
|
Map<Long, MetricSchemaResp> metricIdAndDescPair = new HashMap<>();
|
||||||
List<MetricSchemaResp> metricDescList = generateMetricDesc(metricIds, domainSchemaDesc);
|
List<MetricSchemaResp> metricDescList = generateMetricDesc(metricIds, domainSchemaDesc);
|
||||||
|
|
||||||
@@ -118,6 +118,9 @@ public class ChatConfigUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public List<Long> generateAllMetricIdList(DomainSchemaResp domainSchemaDesc) {
|
public List<Long> generateAllMetricIdList(DomainSchemaResp domainSchemaDesc) {
|
||||||
|
if (Objects.isNull(domainSchemaDesc) || CollectionUtils.isEmpty(domainSchemaDesc.getMetrics())) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
Map<Long, List<MetricSchemaResp>> metricIdAndDescPair = domainSchemaDesc.getMetrics()
|
Map<Long, List<MetricSchemaResp>> metricIdAndDescPair = domainSchemaDesc.getMetrics()
|
||||||
.stream().collect(Collectors.groupingBy(MetricSchemaResp::getId));
|
.stream().collect(Collectors.groupingBy(MetricSchemaResp::getId));
|
||||||
return new ArrayList<>(metricIdAndDescPair.keySet());
|
return new ArrayList<>(metricIdAndDescPair.keySet());
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package com.tencent.supersonic.chat.domain.utils;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.api.component.SchemaMapper;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
|
import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.chat.application.parser.DomainResolver;
|
||||||
|
import com.tencent.supersonic.chat.application.query.QuerySelector;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
import org.springframework.core.io.support.SpringFactoriesLoader;
|
||||||
|
|
||||||
|
public class ComponentFactory {
|
||||||
|
|
||||||
|
private static List<SchemaMapper> schemaMappers = new ArrayList<>();
|
||||||
|
private static List<SemanticParser> semanticParsers = new ArrayList<>();
|
||||||
|
private static SemanticLayer semanticLayer;
|
||||||
|
private static QuerySelector querySelector;
|
||||||
|
private static DomainResolver domainResolver;
|
||||||
|
|
||||||
|
public static List<SchemaMapper> getSchemaMappers() {
|
||||||
|
return CollectionUtils.isEmpty(schemaMappers) ? init(SchemaMapper.class, schemaMappers) : schemaMappers;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<SemanticParser> getSemanticParsers() {
|
||||||
|
return CollectionUtils.isEmpty(semanticParsers) ? init(SemanticParser.class, semanticParsers) : semanticParsers;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static SemanticLayer getSemanticLayer() {
|
||||||
|
if (Objects.isNull(semanticLayer)) {
|
||||||
|
semanticLayer = init(SemanticLayer.class);
|
||||||
|
}
|
||||||
|
return semanticLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void setSemanticLayer(SemanticLayer layer) {
|
||||||
|
semanticLayer = layer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static QuerySelector getQuerySelector() {
|
||||||
|
if (Objects.isNull(querySelector)) {
|
||||||
|
querySelector = init(QuerySelector.class);
|
||||||
|
}
|
||||||
|
return querySelector;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DomainResolver getDomainResolver() {
|
||||||
|
if (Objects.isNull(domainResolver)) {
|
||||||
|
domainResolver = init(DomainResolver.class);
|
||||||
|
}
|
||||||
|
return domainResolver;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <T> List<T> init(Class<T> factoryType, List list) {
|
||||||
|
list.addAll(SpringFactoriesLoader.loadFactories(factoryType,
|
||||||
|
Thread.currentThread().getContextClassLoader()));
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <T> T init(Class<T> factoryType) {
|
||||||
|
return SpringFactoriesLoader.loadFactories(factoryType,
|
||||||
|
Thread.currentThread().getContextClassLoader()).get(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,11 +2,11 @@ package com.tencent.supersonic.chat.domain.utils;
|
|||||||
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||||
import com.tencent.supersonic.chat.api.pojo.Filter;
|
import com.tencent.supersonic.chat.api.pojo.Filter;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementCount;
|
import com.tencent.supersonic.chat.api.pojo.QueryMatchInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticQuery;
|
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
||||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
@@ -22,6 +22,21 @@ import java.util.Optional;
|
|||||||
|
|
||||||
public class ContextHelper {
|
public class ContextHelper {
|
||||||
|
|
||||||
|
public static Comparator<Map.Entry<Integer, QueryMatchInfo>> DomainStatComparator
|
||||||
|
= (o1, o2) -> domainSchemaElementCountComparator(o1.getValue(), o2.getValue());
|
||||||
|
public static Comparator<Map.Entry<SemanticQuery, QueryMatchInfo>> SemanticQueryStatComparator
|
||||||
|
= (o1, o2) -> domainSchemaElementCountComparator(o1.getValue(), o2.getValue());
|
||||||
|
/**
|
||||||
|
* similarity desc
|
||||||
|
*/
|
||||||
|
public static Comparator<SchemaElementMatch> schemaElementMatchComparatorBySimilarity
|
||||||
|
= new Comparator<SchemaElementMatch>() {
|
||||||
|
@Override
|
||||||
|
public int compare(SchemaElementMatch o1, SchemaElementMatch o2) {
|
||||||
|
return (int) ((o2.getSimilarity() - o1.getSimilarity()) * 100);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
public static void updateDomain(SemanticParseInfo from, SemanticParseInfo to) {
|
public static void updateDomain(SemanticParseInfo from, SemanticParseInfo to) {
|
||||||
if (from != null && from.getDomainId() != null) {
|
if (from != null && from.getDomainId() != null) {
|
||||||
to.setDomainId(from.getDomainId());
|
to.setDomainId(from.getDomainId());
|
||||||
@@ -65,6 +80,15 @@ public class ContextHelper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void updateDomainIfEmpty(SemanticParseInfo from, SemanticParseInfo to) {
|
||||||
|
if (from != null && from.getDomainId() != null && to.getDomainId() == null) {
|
||||||
|
to.setDomainId(from.getDomainId());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (from != null && from.getDomainName() != null && to.getDomainName() == null) {
|
||||||
|
to.setDomainName(from.getDomainName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* add from to list if list is empty and from is not empty
|
* add from to list if list is empty and from is not empty
|
||||||
@@ -105,7 +129,7 @@ public class ContextHelper {
|
|||||||
/**
|
/**
|
||||||
* count desc > similarity desc
|
* count desc > similarity desc
|
||||||
*/
|
*/
|
||||||
public static int domainSchemaElementCountComparator(SchemaElementCount o1, SchemaElementCount o2) {
|
public static int domainSchemaElementCountComparator(QueryMatchInfo o1, QueryMatchInfo o2) {
|
||||||
int difference = o1.getCount() - o2.getCount();
|
int difference = o1.getCount() - o2.getCount();
|
||||||
if (difference == 0) {
|
if (difference == 0) {
|
||||||
return (int) ((o1.getMaxSimilarity() - o2.getMaxSimilarity()) * 100);
|
return (int) ((o1.getMaxSimilarity() - o2.getMaxSimilarity()) * 100);
|
||||||
@@ -113,22 +137,6 @@ public class ContextHelper {
|
|||||||
return difference;
|
return difference;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Comparator<Map.Entry<Integer, SchemaElementCount>> DomainStatComparator
|
|
||||||
= (o1, o2) -> domainSchemaElementCountComparator(o1.getValue(), o2.getValue());
|
|
||||||
|
|
||||||
public static Comparator<Map.Entry<SemanticQuery, SchemaElementCount>> SemanticQueryStatComparator
|
|
||||||
= (o1, o2) -> domainSchemaElementCountComparator(o1.getValue(), o2.getValue());
|
|
||||||
/**
|
|
||||||
* similarity desc
|
|
||||||
*/
|
|
||||||
public static Comparator<SchemaElementMatch> schemaElementMatchComparatorBySimilarity
|
|
||||||
= new Comparator<SchemaElementMatch>() {
|
|
||||||
@Override
|
|
||||||
public int compare(SchemaElementMatch o1, SchemaElementMatch o2) {
|
|
||||||
return (int) ((o2.getSimilarity() - o1.getSimilarity()) * 100);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
public static void setEntityId(Long dimensionId, String value, ChatConfigRichInfo chaConfigRichDesc,
|
public static void setEntityId(Long dimensionId, String value, ChatConfigRichInfo chaConfigRichDesc,
|
||||||
SemanticParseInfo semanticParseInfo) {
|
SemanticParseInfo semanticParseInfo) {
|
||||||
if (chaConfigRichDesc != null && chaConfigRichDesc.getEntity() != null) {
|
if (chaConfigRichDesc != null && chaConfigRichDesc.getEntity() != null) {
|
||||||
@@ -152,35 +160,42 @@ public class ContextHelper {
|
|||||||
* @param toSchemaElementMatch
|
* @param toSchemaElementMatch
|
||||||
* @param elementMatches
|
* @param elementMatches
|
||||||
* @param schemaElementTypes
|
* @param schemaElementTypes
|
||||||
* @param contextSemanticParseInfo
|
* @param contextSemanticParse
|
||||||
*/
|
*/
|
||||||
public static void mergeContextSchemaElementMatch(List<SchemaElementMatch> toSchemaElementMatch,
|
public static void mergeContextSchemaElementMatch(List<SchemaElementMatch> toSchemaElementMatch,
|
||||||
List<SchemaElementMatch> elementMatches, List<SchemaElementType> schemaElementTypes,
|
List<SchemaElementMatch> elementMatches, List<SchemaElementType> schemaElementTypes,
|
||||||
SemanticParseInfo contextSemanticParseInfo) {
|
SemanticParseInfo contextSemanticParse) {
|
||||||
|
|
||||||
|
SchemaElementMatch domainMatch = SchemaElementMatch.builder()
|
||||||
|
.elementType(SchemaElementType.DOMAIN)
|
||||||
|
.elementID(contextSemanticParse.getDomainId().intValue())
|
||||||
|
.similarity(1.0)
|
||||||
|
.word(contextSemanticParse.getDomainName())
|
||||||
|
.detectWord(contextSemanticParse.getDomainName())
|
||||||
|
.build();
|
||||||
|
toSchemaElementMatch.add(domainMatch);
|
||||||
|
|
||||||
for (SchemaElementType schemaElementType : schemaElementTypes) {
|
for (SchemaElementType schemaElementType : schemaElementTypes) {
|
||||||
switch (schemaElementType) {
|
switch (schemaElementType) {
|
||||||
case DIMENSION:
|
case DIMENSION:
|
||||||
if (contextSemanticParseInfo.getDimensions() != null
|
if (contextSemanticParse.getDimensions().size() > 0) {
|
||||||
&& contextSemanticParseInfo.getDimensions().size() > 0) {
|
for (SchemaItem dimension : contextSemanticParse.getDimensions()) {
|
||||||
for (SchemaItem dimension : contextSemanticParseInfo.getDimensions()) {
|
|
||||||
addSchemaElementMatch(toSchemaElementMatch, elementMatches, SchemaElementType.DIMENSION,
|
addSchemaElementMatch(toSchemaElementMatch, elementMatches, SchemaElementType.DIMENSION,
|
||||||
dimension);
|
dimension);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case METRIC:
|
case METRIC:
|
||||||
if (contextSemanticParseInfo.getMetrics() != null
|
if (contextSemanticParse.getMetrics().size() > 0) {
|
||||||
&& contextSemanticParseInfo.getMetrics().size() > 0) {
|
for (SchemaItem metric : contextSemanticParse.getMetrics()) {
|
||||||
for (SchemaItem metric : contextSemanticParseInfo.getMetrics()) {
|
|
||||||
addSchemaElementMatch(toSchemaElementMatch, elementMatches, SchemaElementType.METRIC,
|
addSchemaElementMatch(toSchemaElementMatch, elementMatches, SchemaElementType.METRIC,
|
||||||
metric);
|
metric);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case VALUE:
|
case VALUE:
|
||||||
if (contextSemanticParseInfo.getDimensionFilters() != null
|
if (contextSemanticParse.getDimensionFilters().size() > 0) {
|
||||||
&& contextSemanticParseInfo.getDimensionFilters().size() > 0) {
|
for (Filter chatFilter : contextSemanticParse.getDimensionFilters()) {
|
||||||
for (Filter chatFilter : contextSemanticParseInfo.getDimensionFilters()) {
|
|
||||||
if (!isInSchemaElementMatchList(elementMatches, SchemaElementType.VALUE,
|
if (!isInSchemaElementMatchList(elementMatches, SchemaElementType.VALUE,
|
||||||
chatFilter.getValue().toString())) {
|
chatFilter.getValue().toString())) {
|
||||||
toSchemaElementMatch.add(
|
toSchemaElementMatch.add(
|
||||||
@@ -228,13 +243,12 @@ public class ContextHelper {
|
|||||||
|
|
||||||
private static SchemaElementMatch getSchemaElementMatchByContext(int id, String word,
|
private static SchemaElementMatch getSchemaElementMatchByContext(int id, String word,
|
||||||
SchemaElementType schemaElementType) {
|
SchemaElementType schemaElementType) {
|
||||||
SchemaElementMatch schemaElementMatch = new SchemaElementMatch();
|
return SchemaElementMatch.builder()
|
||||||
schemaElementMatch.setElementID(id);
|
.elementID(id)
|
||||||
schemaElementMatch.setElementType(schemaElementType);
|
.elementType(schemaElementType)
|
||||||
schemaElementMatch.setWord(word);
|
.word(word)
|
||||||
// todo default similarity
|
.similarity(0.5)
|
||||||
schemaElementMatch.setSimilarity(0.5);
|
.build();
|
||||||
return schemaElementMatch;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package com.tencent.supersonic.chat.application.parser;
|
package com.tencent.supersonic.chat.domain.utils;
|
||||||
|
|
||||||
import static java.time.LocalDate.now;
|
import static java.time.LocalDate.now;
|
||||||
|
|
||||||
@@ -8,22 +8,21 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
|||||||
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
import com.tencent.supersonic.chat.api.request.QueryContextReq;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticParser;
|
import com.tencent.supersonic.chat.application.parser.DomainResolver;
|
||||||
import com.tencent.supersonic.chat.application.parser.resolver.DomainResolver;
|
|
||||||
import com.tencent.supersonic.chat.application.query.EntityDetail;
|
import com.tencent.supersonic.chat.application.query.EntityDetail;
|
||||||
import com.tencent.supersonic.chat.application.query.EntityListFilter;
|
import com.tencent.supersonic.chat.application.query.EntityListFilter;
|
||||||
|
import com.tencent.supersonic.chat.application.query.EntityListTopN;
|
||||||
import com.tencent.supersonic.chat.application.query.EntityMetricFilter;
|
import com.tencent.supersonic.chat.application.query.EntityMetricFilter;
|
||||||
import com.tencent.supersonic.chat.application.query.MetricDomain;
|
import com.tencent.supersonic.chat.application.query.MetricDomain;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.DefaultMetric;
|
import com.tencent.supersonic.chat.domain.pojo.config.DefaultMetric;
|
||||||
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
|
||||||
import com.tencent.supersonic.chat.domain.service.ChatService;
|
|
||||||
import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
|
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
import com.tencent.supersonic.common.pojo.SchemaItem;
|
import com.tencent.supersonic.common.pojo.SchemaItem;
|
||||||
import com.tencent.supersonic.common.util.context.ContextUtils;
|
import com.tencent.supersonic.common.util.context.ContextUtils;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -31,99 +30,129 @@ import java.util.Objects;
|
|||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.BeanUtils;
|
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.util.CollectionUtils;
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
public class DefaultMetricSemanticParser implements SemanticParser {
|
public class DefaultMetricUtils {
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(DefaultMetricSemanticParser.class);
|
/**
|
||||||
private DomainResolver selectStrategy;
|
* supplementary default metric date dimension
|
||||||
private ChatService chatService;
|
*/
|
||||||
private DefaultSemanticInternalUtils defaultSemanticUtils;
|
public void injectDefaultMetric(SemanticParseInfo semanticParseInfo, QueryContextReq queryContext,
|
||||||
|
ChatContext chatContext) {
|
||||||
@Override
|
String queryMode = semanticParseInfo.getQueryMode();
|
||||||
public boolean parse(QueryContextReq queryContext, ChatContext chatCtx) {
|
ChatConfigRichInfo chaConfigRichDesc = null;
|
||||||
selectStrategy = ContextUtils.getBean(DomainResolver.class);
|
|
||||||
chatService = ContextUtils.getBean(ChatService.class);
|
|
||||||
defaultSemanticUtils = ContextUtils.getBean(DefaultSemanticInternalUtils.class);
|
|
||||||
String queryMode = queryContext.getParseInfo().getQueryMode();
|
|
||||||
if (StringUtils.isNotEmpty(queryMode)) {
|
if (StringUtils.isNotEmpty(queryMode)) {
|
||||||
// QueryMode Selected
|
if (semanticParseInfo == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (!EntityListFilter.QUERY_MODE.equals(queryMode)) {
|
if (!EntityListFilter.QUERY_MODE.equals(queryMode)) {
|
||||||
|
boolean isFillThemeDefaultMetricLogic = false;
|
||||||
|
boolean isAddEntityDetailDimensionMetric = false;
|
||||||
Integer domainId = queryContext.getDomainId().intValue();
|
Integer domainId = queryContext.getDomainId().intValue();
|
||||||
|
|
||||||
List<SchemaElementMatch> matchedElements = queryContext.getMapInfo().getMatchedElements(domainId);
|
List<SchemaElementMatch> matchedElements = queryContext.getMapInfo().getMatchedElements(domainId);
|
||||||
if (!CollectionUtils.isEmpty(matchedElements)) {
|
if (!CollectionUtils.isEmpty(matchedElements)) {
|
||||||
long metricCount = matchedElements.stream()
|
long metricCount = matchedElements.stream()
|
||||||
.filter(schemaElementMatch -> schemaElementMatch.getElementType()
|
.filter(schemaElementMatch -> schemaElementMatch.getElementType()
|
||||||
.equals(SchemaElementType.METRIC)).count();
|
.equals(SchemaElementType.METRIC)).count();
|
||||||
if (metricCount <= 0) {
|
if (metricCount <= 0) {
|
||||||
if (chatCtx.getParseInfo() == null
|
if (chatContext.getParseInfo() == null
|
||||||
|| chatCtx.getParseInfo().getMetrics() == null
|
|| chatContext.getParseInfo().getMetrics() == null
|
||||||
|| chatCtx.getParseInfo().getMetrics().size() <= 0) {
|
|| chatContext.getParseInfo().getMetrics().size() <= 0) {
|
||||||
logger.info("fillThemeDefaultMetricLogic");
|
log.info("fillThemeDefaultMetricLogic");
|
||||||
fillThemeDefaultMetricLogic(queryContext.getParseInfo());
|
isFillThemeDefaultMetricLogic = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
log.info("fillThemeDefaultMetricLogic for empty matchedElements ");
|
||||||
|
isFillThemeDefaultMetricLogic = true;
|
||||||
}
|
}
|
||||||
fillDateDomain(chatCtx, queryContext);
|
if (EntityDetail.QUERY_MODE.equals(queryMode) || EntityMetricFilter.QUERY_MODE.equals(queryMode)) {
|
||||||
|
isAddEntityDetailDimensionMetric = true;
|
||||||
|
dealNativeQuery(semanticParseInfo, queryContext, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isFillThemeDefaultMetricLogic) {
|
||||||
|
if (chaConfigRichDesc == null) {
|
||||||
|
chaConfigRichDesc = getChatConfigRichInfo(semanticParseInfo.getDomainId());
|
||||||
|
}
|
||||||
|
fillThemeDefaultMetricLogic(semanticParseInfo, chaConfigRichDesc, queryContext);
|
||||||
|
}
|
||||||
|
if (isAddEntityDetailDimensionMetric) {
|
||||||
|
if (chaConfigRichDesc == null) {
|
||||||
|
chaConfigRichDesc = getChatConfigRichInfo(semanticParseInfo.getDomainId());
|
||||||
|
}
|
||||||
|
addEntityDetailDimensionMetric(semanticParseInfo, chaConfigRichDesc, queryContext, chatContext);
|
||||||
|
}
|
||||||
|
fillDateDomain(semanticParseInfo, chatContext, chaConfigRichDesc, queryContext);
|
||||||
}
|
}
|
||||||
}
|
defaultQueryMode(semanticParseInfo, queryContext, chatContext);
|
||||||
defaultQueryMode(queryContext, chatCtx);
|
addEntityTopDimension(semanticParseInfo, chaConfigRichDesc);
|
||||||
|
|
||||||
if (EntityDetail.QUERY_MODE.equals(queryMode) || EntityMetricFilter.QUERY_MODE.equals(queryMode)) {
|
|
||||||
addEntityDetailDimensionMetric(queryContext, chatCtx);
|
|
||||||
dealNativeQuery(queryContext, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void dealNativeQuery(QueryContextReq queryContext, boolean isNativeQuery) {
|
|
||||||
if (Objects.nonNull(queryContext) && Objects.nonNull(queryContext.getParseInfo())) {
|
|
||||||
queryContext.getParseInfo().setNativeQuery(isNativeQuery);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Set<String> addPrimaryDimension(EntityRichInfo entity, List<SchemaItem> dimensions) {
|
public void dealNativeQuery(SemanticParseInfo semanticParseInfo, QueryContextReq queryContext,
|
||||||
|
boolean isNativeQuery) {
|
||||||
|
if (Objects.nonNull(queryContext) && Objects.nonNull(semanticParseInfo)) {
|
||||||
|
semanticParseInfo.setNativeQuery(isNativeQuery);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Set<String> addPrimaryDimension(EntityRichInfo entity, List<SchemaItem> dimensions) {
|
||||||
Set<String> primaryDimensions = new HashSet<>();
|
Set<String> primaryDimensions = new HashSet<>();
|
||||||
if (Objects.isNull(entity) || CollectionUtils.isEmpty(entity.getEntityIds())) {
|
if (Objects.isNull(entity) || CollectionUtils.isEmpty(entity.getEntityIds())) {
|
||||||
return primaryDimensions;
|
return primaryDimensions;
|
||||||
}
|
}
|
||||||
entity.getEntityIds().stream().forEach(dimSchemaDesc -> {
|
entity.getEntityIds().stream().forEach(dimSchemaDesc -> {
|
||||||
SchemaItem dimension = new SchemaItem();
|
SchemaItem dimension = new SchemaItem();
|
||||||
BeanUtils.copyProperties(dimSchemaDesc, dimension);
|
//BeanUtils.copyProperties(dimSchemaDesc, dimension);
|
||||||
|
dimension.setName(dimSchemaDesc.getName());
|
||||||
|
dimension.setBizName(dimSchemaDesc.getBizName());
|
||||||
|
dimension.setId(dimSchemaDesc.getId());
|
||||||
dimensions.add(dimension);
|
dimensions.add(dimension);
|
||||||
primaryDimensions.add(dimSchemaDesc.getBizName());
|
primaryDimensions.add(dimSchemaDesc.getBizName());
|
||||||
});
|
});
|
||||||
return primaryDimensions;
|
return primaryDimensions;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void addEntityDetailDimensionMetric(QueryContextReq queryContext, ChatContext chatCtx) {
|
public void addEntityTopDimension(SemanticParseInfo semanticParseInfo, ChatConfigRichInfo chaConfigRichDesc) {
|
||||||
if (queryContext.getParseInfo().getDomainId() > 0) {
|
if (!semanticParseInfo.getQueryMode().equals(EntityListTopN.QUERY_MODE) || !semanticParseInfo.getDimensions()
|
||||||
Long domainId = queryContext.getParseInfo().getDomainId();
|
.isEmpty()) {
|
||||||
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(domainId);
|
return;
|
||||||
|
}
|
||||||
|
if (semanticParseInfo.getDomainId() > 0) {
|
||||||
|
Long domainId = semanticParseInfo.getDomainId();
|
||||||
|
if (chaConfigRichDesc == null) {
|
||||||
|
chaConfigRichDesc = getChatConfigRichInfo(domainId);
|
||||||
|
}
|
||||||
|
if (chaConfigRichDesc != null && chaConfigRichDesc.getEntity() != null) {
|
||||||
|
List<SchemaItem> dimensions = new ArrayList<>();
|
||||||
|
addPrimaryDimension(chaConfigRichDesc.getEntity(), dimensions);
|
||||||
|
semanticParseInfo.setDimensions(new HashSet<>(dimensions));
|
||||||
|
semanticParseInfo.setLimit(1L);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addEntityDetailDimensionMetric(SemanticParseInfo semanticParseInfo,
|
||||||
|
ChatConfigRichInfo chaConfigRichDesc, QueryContextReq queryContext,
|
||||||
|
ChatContext chatCtx) {
|
||||||
|
if (semanticParseInfo.getDomainId() > 0) {
|
||||||
|
Long domainId = semanticParseInfo.getDomainId();
|
||||||
|
|
||||||
if (chaConfigRichDesc != null) {
|
if (chaConfigRichDesc != null) {
|
||||||
if (chaConfigRichDesc.getEntity() == null
|
if (chaConfigRichDesc.getEntity() == null
|
||||||
|| chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() == null) {
|
|| chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
|
|
||||||
Set<SchemaItem> metrics = new LinkedHashSet();
|
|
||||||
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
|
|
||||||
.forEach(m -> metrics.add(getMetric(m)));
|
|
||||||
semanticParseInfo.setMetrics(metrics);
|
|
||||||
|
|
||||||
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo()
|
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo()
|
||||||
.getMatchedElements(domainId.intValue());
|
.getMatchedElements(domainId.intValue());
|
||||||
if (CollectionUtils.isEmpty(schemaElementMatches) || schemaElementMatches.stream()
|
if (CollectionUtils.isEmpty(schemaElementMatches) || schemaElementMatches.stream()
|
||||||
.filter(s -> SchemaElementType.DIMENSION.equals(s.getElementType())).count() <= 0) {
|
.filter(s -> SchemaElementType.DIMENSION.equals(s.getElementType())).count() <= 0) {
|
||||||
logger.info("addEntityDetailDimensionMetric catch");
|
log.info("addEntityDetailDimensionMetric catch");
|
||||||
Set<SchemaItem> dimensions = new LinkedHashSet();
|
Set<SchemaItem> dimensions = new LinkedHashSet();
|
||||||
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream()
|
chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream()
|
||||||
.forEach(m -> dimensions.add(getDimension(m)));
|
.forEach(m -> dimensions.add(getDimension(m)));
|
||||||
@@ -134,10 +163,10 @@ public class DefaultMetricSemanticParser implements SemanticParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void defaultQueryMode(QueryContextReq queryContext, ChatContext chatCtx) {
|
public void defaultQueryMode(SemanticParseInfo semanticParseInfo, QueryContextReq queryContext,
|
||||||
|
ChatContext chatCtx) {
|
||||||
SchemaMapInfo schemaMap = queryContext.getMapInfo();
|
SchemaMapInfo schemaMap = queryContext.getMapInfo();
|
||||||
SemanticParseInfo parseInfo = queryContext.getParseInfo();
|
if (StringUtils.isEmpty(semanticParseInfo.getQueryMode())) {
|
||||||
if (StringUtils.isEmpty(parseInfo.getQueryMode())) {
|
|
||||||
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getDomainId() > 0) {
|
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getDomainId() > 0) {
|
||||||
//
|
//
|
||||||
Long domain = chatCtx.getParseInfo().getDomainId();
|
Long domain = chatCtx.getParseInfo().getDomainId();
|
||||||
@@ -154,9 +183,9 @@ public class DefaultMetricSemanticParser implements SemanticParser {
|
|||||||
.filter(e -> e.getElementType().equals(SchemaElementType.METRIC)).count();
|
.filter(e -> e.getElementType().equals(SchemaElementType.METRIC)).count();
|
||||||
if (filterNUm > 0 && dimensionNUm > 0 && metricrNUm > 0) {
|
if (filterNUm > 0 && dimensionNUm > 0 && metricrNUm > 0) {
|
||||||
// default as entity detail queryMode
|
// default as entity detail queryMode
|
||||||
logger.info("defaultQueryMode [{}]", EntityDetail.QUERY_MODE);
|
log.info("defaultQueryMode [{}]", EntityDetail.QUERY_MODE);
|
||||||
parseInfo.setQueryMode(EntityDetail.QUERY_MODE);
|
semanticParseInfo.setQueryMode(EntityDetail.QUERY_MODE);
|
||||||
parseInfo.setDomainId(domain);
|
semanticParseInfo.setDomainId(domain);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Long entityNUm = elementMatches.stream()
|
Long entityNUm = elementMatches.stream()
|
||||||
@@ -165,21 +194,21 @@ public class DefaultMetricSemanticParser implements SemanticParser {
|
|||||||
// default as metric domain
|
// default as metric domain
|
||||||
if (metricrNUm > 0 || MetricDomain.QUERY_MODE.equals(queryMode)) {
|
if (metricrNUm > 0 || MetricDomain.QUERY_MODE.equals(queryMode)) {
|
||||||
// default as entity detail queryMode
|
// default as entity detail queryMode
|
||||||
logger.info("defaultQueryMode [{}]", MetricDomain.QUERY_MODE);
|
log.info("defaultQueryMode [{}]", MetricDomain.QUERY_MODE);
|
||||||
parseInfo.setQueryMode(MetricDomain.QUERY_MODE);
|
semanticParseInfo.setQueryMode(MetricDomain.QUERY_MODE);
|
||||||
parseInfo.setDomainId(domain);
|
semanticParseInfo.setDomainId(domain);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (CollectionUtils.isEmpty(schemaMap.getMatchedDomains()) && parseInfo != null
|
if (CollectionUtils.isEmpty(schemaMap.getMatchedDomains()) && semanticParseInfo != null
|
||||||
&& parseInfo.getDateInfo() != null) {
|
&& semanticParseInfo.getDateInfo() != null) {
|
||||||
// only query time
|
// only query time
|
||||||
if (MetricDomain.QUERY_MODE.equals(queryMode)) {
|
if (MetricDomain.QUERY_MODE.equals(queryMode)) {
|
||||||
// METRIC_DOMAIN context
|
// METRIC_DOMAIN context
|
||||||
logger.info("defaultQueryMode [{}]", MetricDomain.QUERY_MODE);
|
log.info("defaultQueryMode [{}]", MetricDomain.QUERY_MODE);
|
||||||
parseInfo.setQueryMode(MetricDomain.QUERY_MODE);
|
semanticParseInfo.setQueryMode(MetricDomain.QUERY_MODE);
|
||||||
parseInfo.setDomainId(domain);
|
semanticParseInfo.setDomainId(domain);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -188,12 +217,14 @@ public class DefaultMetricSemanticParser implements SemanticParser {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void fillDateDomain(ChatContext chatCtx, QueryContextReq queryContext) {
|
public void fillDateDomain(SemanticParseInfo parseInfo, ChatContext chatCtx, ChatConfigRichInfo chaConfigRichDesc,
|
||||||
SemanticParseInfo parseInfo = queryContext.getParseInfo();
|
QueryContextReq queryContext) {
|
||||||
|
//SemanticParseInfo parseInfo = queryContext.getParseInfo();
|
||||||
|
|
||||||
if (parseInfo == null || parseInfo.getDateInfo() == null) {
|
if (parseInfo == null || parseInfo.getDateInfo() == null) {
|
||||||
|
DomainResolver selectStrategy = ComponentFactory.getDomainResolver();
|
||||||
boolean isUpdateTime = false;
|
boolean isUpdateTime = false;
|
||||||
if (selectStrategy.isDomainSwitch(chatCtx, queryContext)) {
|
if (selectStrategy.isDomainSwitch(chatCtx, parseInfo)) {
|
||||||
isUpdateTime = true;
|
isUpdateTime = true;
|
||||||
}
|
}
|
||||||
if (chatCtx.getParseInfo() == null
|
if (chatCtx.getParseInfo() == null
|
||||||
@@ -201,23 +232,31 @@ public class DefaultMetricSemanticParser implements SemanticParser {
|
|||||||
isUpdateTime = true;
|
isUpdateTime = true;
|
||||||
}
|
}
|
||||||
if (isUpdateTime && parseInfo != null && parseInfo.getDomainId() > 0) {
|
if (isUpdateTime && parseInfo != null && parseInfo.getDomainId() > 0) {
|
||||||
logger.info("fillThemeDefaultTime");
|
fillThemeDefaultTime(parseInfo.getDomainId(), chaConfigRichDesc, parseInfo);
|
||||||
fillThemeDefaultTime(parseInfo.getDomainId(), parseInfo);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void fillThemeDefaultMetricLogic(SemanticParseInfo semanticParseInfo) {
|
public void fillThemeDefaultMetricLogic(SemanticParseInfo semanticParseInfo, ChatConfigRichInfo chaConfigRichDesc,
|
||||||
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(
|
QueryContextReq queryContext) {
|
||||||
semanticParseInfo.getDomainId());
|
//SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
|
||||||
|
|
||||||
if (Objects.isNull(chaConfigRichDesc) || CollectionUtils.isEmpty(chaConfigRichDesc.getDefaultMetrics())) {
|
if (Objects.isNull(chaConfigRichDesc) || CollectionUtils.isEmpty(chaConfigRichDesc.getDefaultMetrics())) {
|
||||||
log.info("there is no defaultMetricIds info");
|
log.info("there is no defaultMetricIds info");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) && CollectionUtils.isEmpty(
|
if (queryContext.getMapInfo() == null || !queryContext.getMapInfo().getMatchedDomains()
|
||||||
semanticParseInfo.getDimensions())) {
|
.contains(chaConfigRichDesc.getDomainId().intValue())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo()
|
||||||
|
.getMatchedElements(chaConfigRichDesc.getDomainId().intValue());
|
||||||
|
long metricNum = schemaElementMatches.stream().filter(e -> e.getElementType().equals(SchemaElementType.METRIC))
|
||||||
|
.count();
|
||||||
|
long dimensionNum = schemaElementMatches.stream()
|
||||||
|
.filter(e -> e.getElementType().equals(SchemaElementType.DIMENSION)).count();
|
||||||
|
if (metricNum <= 0 && dimensionNum <= 0) {
|
||||||
Set<SchemaItem> metrics = new LinkedHashSet();
|
Set<SchemaItem> metrics = new LinkedHashSet();
|
||||||
chaConfigRichDesc.getDefaultMetrics().stream().forEach(metric -> {
|
chaConfigRichDesc.getDefaultMetrics().stream().forEach(metric -> {
|
||||||
SchemaItem metricTmp = new SchemaItem();
|
SchemaItem metricTmp = new SchemaItem();
|
||||||
@@ -240,9 +279,15 @@ public class DefaultMetricSemanticParser implements SemanticParser {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void fillThemeDefaultTime(Long domain, SemanticParseInfo semanticParseInfo) {
|
public void fillThemeDefaultTime(Long domain, ChatConfigRichInfo chaConfigRichDesc,
|
||||||
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(
|
SemanticParseInfo semanticParseInfo) {
|
||||||
semanticParseInfo.getDomainId());
|
if (!Objects.isNull(semanticParseInfo.getDateInfo()) && !Objects.isNull(
|
||||||
|
semanticParseInfo.getDateInfo().getDateMode())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (chaConfigRichDesc == null) {
|
||||||
|
chaConfigRichDesc = getChatConfigRichInfo(semanticParseInfo.getDomainId());
|
||||||
|
}
|
||||||
if (!Objects.isNull(chaConfigRichDesc) && !CollectionUtils.isEmpty(chaConfigRichDesc.getDefaultMetrics())) {
|
if (!Objects.isNull(chaConfigRichDesc) && !CollectionUtils.isEmpty(chaConfigRichDesc.getDefaultMetrics())) {
|
||||||
DefaultMetric defaultMetricInfo = chaConfigRichDesc.getDefaultMetrics().get(0);
|
DefaultMetric defaultMetricInfo = chaConfigRichDesc.getDefaultMetrics().get(0);
|
||||||
DateConf dateInfo = new DateConf();
|
DateConf dateInfo = new DateConf();
|
||||||
@@ -252,9 +297,16 @@ public class DefaultMetricSemanticParser implements SemanticParser {
|
|||||||
dateInfo.setEndDate(now().minusDays(1).toString());
|
dateInfo.setEndDate(now().minusDays(1).toString());
|
||||||
dateInfo.setPeriod(defaultMetricInfo.getPeriod());
|
dateInfo.setPeriod(defaultMetricInfo.getPeriod());
|
||||||
semanticParseInfo.setDateInfo(dateInfo);
|
semanticParseInfo.setDateInfo(dateInfo);
|
||||||
|
log.info("fillThemeDefaultTime");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ChatConfigRichInfo getChatConfigRichInfo(Long domain) {
|
||||||
|
DefaultSemanticInternalUtils defaultSemanticUtils = ContextUtils.getBean(DefaultSemanticInternalUtils.class);
|
||||||
|
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(domain);
|
||||||
|
return chaConfigRichDesc;
|
||||||
|
}
|
||||||
|
|
||||||
private SchemaItem getMetric(MetricSchemaResp metricSchemaDesc) {
|
private SchemaItem getMetric(MetricSchemaResp metricSchemaDesc) {
|
||||||
SchemaItem queryMeta = new SchemaItem();
|
SchemaItem queryMeta = new SchemaItem();
|
||||||
queryMeta.setId(metricSchemaDesc.getId());
|
queryMeta.setId(metricSchemaDesc.getId());
|
||||||
@@ -19,10 +19,12 @@ import com.tencent.supersonic.common.result.ReturnCode;
|
|||||||
import com.tencent.supersonic.common.util.context.S2ThreadContext;
|
import com.tencent.supersonic.common.util.context.S2ThreadContext;
|
||||||
import com.tencent.supersonic.common.util.context.ThreadContext;
|
import com.tencent.supersonic.common.util.context.ThreadContext;
|
||||||
import com.tencent.supersonic.common.util.json.JsonUtil;
|
import com.tencent.supersonic.common.util.json.JsonUtil;
|
||||||
|
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.springframework.beans.BeanUtils;
|
import org.springframework.beans.BeanUtils;
|
||||||
@@ -139,13 +141,13 @@ public class DefaultSemanticInternalUtils {
|
|||||||
headers.set(authenticationConfig.getTokenHttpHeaderKey(), threadContext.getToken());
|
headers.set(authenticationConfig.getTokenHttpHeaderKey(), threadContext.getToken());
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.info("threadContext is null:{}", Objects.isNull(threadContext));
|
log.debug("threadContext is null:{}", Objects.isNull(threadContext));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<DomainResp> getDomainListForUser(User user) {
|
public List<DomainResp> getDomainListForAdmin() {
|
||||||
Object domainDescListObject = fetchHttpResult(semanticUrl + fetchDomainListPath, null, HttpMethod.GET);
|
Object domainDescListObject = fetchHttpResult(semanticUrl + fetchDomainListPath, null, HttpMethod.GET);
|
||||||
List<DomainResp> domainDescList = (List<DomainResp>) domainDescListObject;
|
List<DomainResp> domainDescList = JsonUtil.toList(JsonUtil.toString(domainDescListObject), DomainResp.class);
|
||||||
return domainDescList;
|
return domainDescList;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.domain.utils;
|
|||||||
import static com.tencent.supersonic.common.constant.Constants.DAY;
|
import static com.tencent.supersonic.common.constant.Constants.DAY;
|
||||||
import static com.tencent.supersonic.common.constant.Constants.UNDERLINE;
|
import static com.tencent.supersonic.common.constant.Constants.UNDERLINE;
|
||||||
|
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
|
||||||
import com.tencent.supersonic.chat.domain.dataobject.DimValueDO;
|
import com.tencent.supersonic.chat.domain.dataobject.DimValueDO;
|
||||||
@@ -32,15 +32,12 @@ import org.springframework.util.CollectionUtils;
|
|||||||
@Component
|
@Component
|
||||||
public class DictMetaUtils {
|
public class DictMetaUtils {
|
||||||
|
|
||||||
|
private final DefaultSemanticInternalUtils defaultSemanticUtils;
|
||||||
@Value("${model.internal.metric.suffix:internal_cnt}")
|
@Value("${model.internal.metric.suffix:internal_cnt}")
|
||||||
private String internalMetricNameSuffix;
|
private String internalMetricNameSuffix;
|
||||||
|
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
|
|
||||||
private final SemanticLayer semanticLayer;
|
public DictMetaUtils(DefaultSemanticInternalUtils defaultSemanticUtils) {
|
||||||
private final DefaultSemanticInternalUtils defaultSemanticUtils;
|
|
||||||
|
|
||||||
public DictMetaUtils(SemanticLayer semanticLayer,
|
|
||||||
DefaultSemanticInternalUtils defaultSemanticUtils) {
|
|
||||||
this.semanticLayer = semanticLayer;
|
|
||||||
this.defaultSemanticUtils = defaultSemanticUtils;
|
this.defaultSemanticUtils = defaultSemanticUtils;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -131,7 +128,7 @@ public class DictMetaUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void fillDimValueDOList(List<DimValueDO> dimValueDOList, Long domainId,
|
private void fillDimValueDOList(List<DimValueDO> dimValueDOList, Long domainId,
|
||||||
Map<Long, DimSchemaResp> dimIdAndDescPair) {
|
Map<Long, DimSchemaResp> dimIdAndDescPair) {
|
||||||
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(domainId);
|
ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(domainId);
|
||||||
if (Objects.nonNull(chaConfigRichDesc)) {
|
if (Objects.nonNull(chaConfigRichDesc)) {
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import static com.tencent.supersonic.common.constant.Constants.COMMA;
|
|||||||
import static com.tencent.supersonic.common.constant.Constants.UNDERLINE_DOUBLE;
|
import static com.tencent.supersonic.common.constant.Constants.UNDERLINE_DOUBLE;
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.chat.api.service.SemanticLayer;
|
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||||
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
|
import com.tencent.supersonic.semantic.api.core.pojo.QueryColumn;
|
||||||
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
|
||||||
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
|
||||||
@@ -19,6 +19,7 @@ import com.tencent.supersonic.common.enums.AggOperatorEnum;
|
|||||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
import com.tencent.supersonic.common.pojo.Order;
|
import com.tencent.supersonic.common.pojo.Order;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -26,6 +27,7 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.StringJoiner;
|
import java.util.StringJoiner;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.logging.log4j.util.Strings;
|
import org.apache.logging.log4j.util.Strings;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -36,9 +38,8 @@ import org.springframework.util.CollectionUtils;
|
|||||||
@Component
|
@Component
|
||||||
public class DictQueryUtils {
|
public class DictQueryUtils {
|
||||||
|
|
||||||
private final SemanticLayer semanticLayer;
|
|
||||||
Long frequencyMax = 99999999L;
|
Long frequencyMax = 99999999L;
|
||||||
|
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||||
@Value("${dimension.multi.value.split:#}")
|
@Value("${dimension.multi.value.split:#}")
|
||||||
private String dimMultiValueSplit;
|
private String dimMultiValueSplit;
|
||||||
|
|
||||||
@@ -48,11 +49,6 @@ public class DictQueryUtils {
|
|||||||
@Value("${dimension.max.limit:3000000}")
|
@Value("${dimension.max.limit:3000000}")
|
||||||
private Long dimMaxLimit;
|
private Long dimMaxLimit;
|
||||||
|
|
||||||
public DictQueryUtils(SemanticLayer semanticLayer) {
|
|
||||||
this.semanticLayer = semanticLayer;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public List<String> fetchDimValueSingle(Long domainId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict,
|
public List<String> fetchDimValueSingle(Long domainId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict,
|
||||||
User user) {
|
User user) {
|
||||||
List<String> data = new ArrayList<>();
|
List<String> data = new ArrayList<>();
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user