[improvement][project] supersonic 0.7.0 version backend update (#24)

* [improvement][project] supersonic 0.7.0 version backend update

* [improvement][project] supersonic 0.7.0 version backend update

* [improvement][project] supersonic 0.7.0 version readme update

---------

Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
SunDean
2023-08-05 22:17:56 +08:00
committed by GitHub
parent 6951eada9d
commit aa0a100a85
184 changed files with 2609 additions and 1238 deletions

View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.auth.api.authentication.adaptor;
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
import java.util.List;
import java.util.Set;
public interface UserAdaptor {
List<String> getUserNames();
List<User> getUserList();
List<Organization> getOrganizationTree();
void register(UserReq userReq);
String login(UserReq userReq);
List<User> getUserByOrg(String key);
Set<String> getUserAllOrgId(String userName);
}

View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.auth.api.authentication.pojo;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class Organization {
private String id;
private String parentId;
private String name;
private String fullName;
private List<Organization> subOrganizations = Lists.newArrayList();
private boolean isRoot;
}

View File

@@ -1,9 +1,11 @@
package com.tencent.supersonic.auth.api.authentication.service; package com.tencent.supersonic.auth.api.authentication.service;
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.request.UserReq; import com.tencent.supersonic.auth.api.authentication.request.UserReq;
import java.util.List; import java.util.List;
import java.util.Set;
public interface UserService { public interface UserService {
@@ -14,4 +16,10 @@ public interface UserService {
void register(UserReq userCmd); void register(UserReq userCmd);
String login(UserReq userCmd); String login(UserReq userCmd);
Set<String> getUserAllOrgId(String userName);
List<User> getUserByOrg(String key);
List<Organization> getOrganizationTree();
} }

View File

@@ -1,35 +1,30 @@
package com.tencent.supersonic.auth.authentication.application; package com.tencent.supersonic.auth.authentication.adaptor;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor;
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword; import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
import com.tencent.supersonic.auth.api.authentication.request.UserReq; import com.tencent.supersonic.auth.api.authentication.request.UserReq;
import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO;
import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO; import com.tencent.supersonic.auth.authentication.persistence.repository.UserRepository;
import com.tencent.supersonic.auth.authentication.domain.repository.UserRepository; import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils;
import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service; import java.util.List;
import java.util.Set;
@Service import java.util.stream.Collectors;
public class UserServiceImpl implements UserService {
private UserRepository userRepository;
private UserTokenUtils userTokenUtils;
public UserServiceImpl(UserRepository userRepository, UserTokenUtils userTokenUtils) {
this.userRepository = userRepository;
this.userTokenUtils = userTokenUtils;
}
public class DefaultUserAdaptor implements UserAdaptor {
private List<UserDO> getUserDOList() { private List<UserDO> getUserDOList() {
UserRepository userRepository = ContextUtils.getBean(UserRepository.class);
return userRepository.getUserList(); return userRepository.getUserList();
} }
private UserDO getUser(String name) { private UserDO getUser(String name) {
UserRepository userRepository = ContextUtils.getBean(UserRepository.class);
return userRepository.getUser(name); return userRepository.getUser(name);
} }
@@ -38,22 +33,26 @@ public class UserServiceImpl implements UserService {
return getUserDOList().stream().map(UserDO::getName).collect(Collectors.toList()); return getUserDOList().stream().map(UserDO::getName).collect(Collectors.toList());
} }
@Override @Override
public List<User> getUserList() { public List<User> getUserList() {
List<UserDO> userDOS = getUserDOList(); List<UserDO> userDOS = getUserDOList();
return userDOS.stream().map(this::convert).collect(Collectors.toList()); return userDOS.stream().map(this::convert).collect(Collectors.toList());
} }
@Override
public List<Organization> getOrganizationTree() {
return Lists.newArrayList();
}
private User convert(UserDO userDO) { private User convert(UserDO userDO) {
User user = new User(); User user = new User();
BeanUtils.copyProperties(userDO, user); BeanUtils.copyProperties(userDO, user);
return user; return user;
} }
@Override @Override
public void register(UserReq userReq) { public void register(UserReq userReq) {
UserRepository userRepository = ContextUtils.getBean(UserRepository.class);
List<String> userDOS = getUserNames(); List<String> userDOS = getUserNames();
if (userDOS.contains(userReq.getName())) { if (userDOS.contains(userReq.getName())) {
throw new RuntimeException(String.format("user %s exist", userReq.getName())); throw new RuntimeException(String.format("user %s exist", userReq.getName()));
@@ -65,6 +64,7 @@ public class UserServiceImpl implements UserService {
@Override @Override
public String login(UserReq userReq) { public String login(UserReq userReq) {
UserTokenUtils userTokenUtils = ContextUtils.getBean(UserTokenUtils.class);
UserDO userDO = getUser(userReq.getName()); UserDO userDO = getUser(userReq.getName());
if (userDO == null) { if (userDO == null) {
throw new RuntimeException("user not exist,please register"); throw new RuntimeException("user not exist,please register");
@@ -77,5 +77,14 @@ public class UserServiceImpl implements UserService {
throw new RuntimeException("password not correct, please try again"); throw new RuntimeException("password not correct, please try again");
} }
@Override
public List<User> getUserByOrg(String key) {
return Lists.newArrayList();
}
} @Override
public Set<String> getUserAllOrgId(String userName) {
return Sets.newHashSet();
}
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.auth.authentication.domain.interceptor; package com.tencent.supersonic.auth.authentication.interceptor;
import java.lang.annotation.ElementType; import java.lang.annotation.ElementType;

View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.auth.authentication.domain.interceptor; package com.tencent.supersonic.auth.authentication.interceptor;
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
import com.tencent.supersonic.auth.api.authentication.constant.UserConstants; import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
import com.tencent.supersonic.auth.authentication.application.UserServiceImpl; import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils; import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils;
import com.tencent.supersonic.common.util.S2ThreadContext; import com.tencent.supersonic.common.util.S2ThreadContext;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.util.Arrays; import java.util.Arrays;

View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.auth.authentication.domain.interceptor; package com.tencent.supersonic.auth.authentication.interceptor;
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword; import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
import com.tencent.supersonic.auth.authentication.application.UserServiceImpl; import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils; import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils;
import com.tencent.supersonic.common.pojo.exception.AccessException; import com.tencent.supersonic.common.pojo.exception.AccessException;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.S2ThreadContext; import com.tencent.supersonic.common.util.S2ThreadContext;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.auth.authentication.domain.interceptor; package com.tencent.supersonic.auth.authentication.interceptor;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.SpringFactoriesLoader; import org.springframework.core.io.support.SpringFactoriesLoader;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.auth.authentication.domain.dataobject; package com.tencent.supersonic.auth.authentication.persistence.dataobject;
public class UserDO { public class UserDO {

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.auth.authentication.domain.dataobject; package com.tencent.supersonic.auth.authentication.persistence.dataobject;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.auth.authentication.infrastructure.mapper; package com.tencent.supersonic.auth.authentication.persistence.mapper;
import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO;
import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDOExample; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample;
import java.util.List; import java.util.List;
import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Mapper;

View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.auth.authentication.infrastructure.repository; package com.tencent.supersonic.auth.authentication.persistence.repository.Impl;
import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO;
import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDOExample; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample;
import com.tencent.supersonic.auth.authentication.domain.repository.UserRepository; import com.tencent.supersonic.auth.authentication.persistence.repository.UserRepository;
import com.tencent.supersonic.auth.authentication.infrastructure.mapper.UserDOMapper; import com.tencent.supersonic.auth.authentication.persistence.mapper.UserDOMapper;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.auth.authentication.domain.repository; package com.tencent.supersonic.auth.authentication.persistence.repository;
import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO; import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO;
import java.util.List; import java.util.List;
public interface UserRepository { public interface UserRepository {

View File

@@ -1,19 +1,17 @@
package com.tencent.supersonic.auth.authentication.rest; package com.tencent.supersonic.auth.authentication.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.request.UserReq; import com.tencent.supersonic.auth.api.authentication.request.UserReq;
import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import java.util.List; import java.util.List;
import java.util.Set;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController @RestController
@RequestMapping("/api/auth/user") @RequestMapping("/api/auth/user")
@@ -32,7 +30,6 @@ public class UserController {
return UserHolder.findUser(httpServletRequest, httpServletResponse); return UserHolder.findUser(httpServletRequest, httpServletResponse);
} }
@GetMapping("/getUserNames") @GetMapping("/getUserNames")
public List<String> getUserNames() { public List<String> getUserNames() {
return userService.getUserNames(); return userService.getUserNames();
@@ -43,6 +40,21 @@ public class UserController {
return userService.getUserList(); return userService.getUserList();
} }
@GetMapping("/getOrganizationTree")
public List<Organization> getOrganizationTree() {
return userService.getOrganizationTree();
}
@GetMapping("/getUserAllOrgId/{userName}")
public Set<String> getUserAllOrgId(@PathVariable("userName") String userName) {
return userService.getUserAllOrgId(userName);
}
@GetMapping("/getUserByOrg/{org}")
public List<User> getUserByOrg(@PathVariable("org") String org) {
return userService.getUserByOrg(org);
}
@PostMapping("/register") @PostMapping("/register")
public void register(@RequestBody UserReq userCmd) { public void register(@RequestBody UserReq userCmd) {
userService.register(userCmd); userService.register(userCmd);

View File

@@ -0,0 +1,52 @@
package com.tencent.supersonic.auth.authentication.service;
import com.tencent.supersonic.auth.api.authentication.pojo.Organization;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.request.UserReq;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.auth.authentication.utils.ComponentFactory;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
@Service
public class UserServiceImpl implements UserService {
@Override
public List<String> getUserNames() {
return ComponentFactory.getUserAdaptor().getUserNames();
}
@Override
public List<User> getUserList() {
return ComponentFactory.getUserAdaptor().getUserList();
}
@Override
public Set<String> getUserAllOrgId(String userName) {
return ComponentFactory.getUserAdaptor().getUserAllOrgId(userName);
}
@Override
public List<User> getUserByOrg(String key) {
return ComponentFactory.getUserAdaptor().getUserByOrg(key);
}
@Override
public List<Organization> getOrganizationTree() {
return ComponentFactory.getUserAdaptor().getOrganizationTree();
}
@Override
public void register(UserReq userReq) {
ComponentFactory.getUserAdaptor().register(userReq);
}
@Override
public String login(UserReq userReq) {
return ComponentFactory.getUserAdaptor().login(userReq);
}
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.auth.authentication.domain.strategy; package com.tencent.supersonic.auth.authentication.strategy;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;

View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.auth.authentication.domain.strategy; package com.tencent.supersonic.auth.authentication.strategy;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.service.UserStrategy; import com.tencent.supersonic.auth.api.authentication.service.UserStrategy;
import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils; import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.auth.authentication.domain.strategy; package com.tencent.supersonic.auth.authentication.strategy;
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;

View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.auth.authentication.utils;
import com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor;
import org.springframework.core.io.support.SpringFactoriesLoader;
import java.util.Objects;
public class ComponentFactory {
private static UserAdaptor userAdaptor;
public static UserAdaptor getUserAdaptor() {
if (Objects.isNull(userAdaptor)) {
userAdaptor = init(UserAdaptor.class);
}
return userAdaptor;
}
private static <T> T init(Class<T> factoryType) {
return SpringFactoriesLoader.loadFactories(factoryType,
Thread.currentThread().getContextClassLoader()).get(0);
}
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.auth.authentication.domain.utils; package com.tencent.supersonic.auth.authentication.utils;
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_ALGORITHM; import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_ALGORITHM;
import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_CREATE_TIME; import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_CREATE_TIME;

View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.tencent.supersonic.auth.authentication.infrastructure.mapper.UserDOMapper"> <mapper namespace="com.tencent.supersonic.auth.authentication.persistence.mapper.UserDOMapper">
<resultMap id="BaseResultMap" type="com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO"> <resultMap id="BaseResultMap" type="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO">
<id column="id" jdbcType="BIGINT" property="id" /> <id column="id" jdbcType="BIGINT" property="id" />
<result column="name" jdbcType="VARCHAR" property="name" /> <result column="name" jdbcType="VARCHAR" property="name" />
<result column="password" jdbcType="VARCHAR" property="password" /> <result column="password" jdbcType="VARCHAR" property="password" />
@@ -40,7 +40,7 @@
<sql id="Base_Column_List"> <sql id="Base_Column_List">
id, name, password, display_name, email id, name, password, display_name, email
</sql> </sql>
<select id="selectByExample" parameterType="com.tencent.supersonic.auth.authentication.domain.dataobject.UserDOExample" resultMap="BaseResultMap"> <select id="selectByExample" parameterType="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample" resultMap="BaseResultMap">
select select
<if test="distinct"> <if test="distinct">
distinct distinct
@@ -67,13 +67,13 @@
delete from s2_user delete from s2_user
where id = #{id,jdbcType=BIGINT} where id = #{id,jdbcType=BIGINT}
</delete> </delete>
<insert id="insert" parameterType="com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO"> <insert id="insert" parameterType="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO">
insert into s2_user (id, name, password, insert into s2_user (id, name, password,
display_name, email) display_name, email)
values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{password,jdbcType=VARCHAR}, values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{password,jdbcType=VARCHAR},
#{displayName,jdbcType=VARCHAR}, #{email,jdbcType=VARCHAR}) #{displayName,jdbcType=VARCHAR}, #{email,jdbcType=VARCHAR})
</insert> </insert>
<insert id="insertSelective" parameterType="com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO"> <insert id="insertSelective" parameterType="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO">
insert into s2_user insert into s2_user
<trim prefix="(" suffix=")" suffixOverrides=","> <trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null"> <if test="id != null">
@@ -110,13 +110,13 @@
</if> </if>
</trim> </trim>
</insert> </insert>
<select id="countByExample" parameterType="com.tencent.supersonic.auth.authentication.domain.dataobject.UserDOExample" resultType="java.lang.Long"> <select id="countByExample" parameterType="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample" resultType="java.lang.Long">
select count(*) from s2_user select count(*) from s2_user
<if test="_parameter != null"> <if test="_parameter != null">
<include refid="Example_Where_Clause" /> <include refid="Example_Where_Clause" />
</if> </if>
</select> </select>
<update id="updateByPrimaryKeySelective" parameterType="com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO"> <update id="updateByPrimaryKeySelective" parameterType="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO">
update s2_user update s2_user
<set> <set>
<if test="name != null"> <if test="name != null">
@@ -134,7 +134,7 @@
</set> </set>
where id = #{id,jdbcType=BIGINT} where id = #{id,jdbcType=BIGINT}
</update> </update>
<update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO"> <update id="updateByPrimaryKey" parameterType="com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO">
update s2_user update s2_user
set name = #{name,jdbcType=VARCHAR}, set name = #{name,jdbcType=VARCHAR},
password = #{password,jdbcType=VARCHAR}, password = #{password,jdbcType=VARCHAR},

View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.auth.authorization.application;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp; import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter; import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
@@ -18,10 +19,7 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Service @Service
@@ -30,8 +28,12 @@ public class AuthServiceImpl implements AuthService {
private JdbcTemplate jdbcTemplate; private JdbcTemplate jdbcTemplate;
public AuthServiceImpl(JdbcTemplate jdbcTemplate) { private UserService userService;
public AuthServiceImpl(JdbcTemplate jdbcTemplate,
UserService userService) {
this.jdbcTemplate = jdbcTemplate; this.jdbcTemplate = jdbcTemplate;
this.userService = userService;
} }
private List<AuthGroup> load() { private List<AuthGroup> load() {
@@ -75,6 +77,10 @@ public class AuthServiceImpl implements AuthService {
@Override @Override
public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, HttpServletRequest request) { public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, HttpServletRequest request) {
Set<String> userOrgIds = userService.getUserAllOrgId(req.getUser());
if (!CollectionUtils.isEmpty(userOrgIds)) {
req.setDepartmentIds(new ArrayList<>(userOrgIds));
}
List<AuthGroup> groups = getAuthGroups(req); List<AuthGroup> groups = getAuthGroups(req);
AuthorizedResourceResp resource = new AuthorizedResourceResp(); AuthorizedResourceResp resource = new AuthorizedResourceResp();
Map<String, List<AuthGroup>> authGroupsByDomainId = groups.stream() Map<String, List<AuthGroup>> authGroupsByDomainId = groups.stream()
@@ -119,7 +125,6 @@ public class AuthServiceImpl implements AuthService {
} }
} }
} }
return resource; return resource;
} }
@@ -133,9 +138,9 @@ public class AuthServiceImpl implements AuthService {
.contains(req.getUser())) { .contains(req.getUser())) {
return true; return true;
} }
for (String deparmentId : req.getDepartmentIds()) { for (String departmentId : req.getDepartmentIds()) {
if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds()) if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds())
&& group.getAuthorizedDepartmentIds().contains(deparmentId)) { && group.getAuthorizedDepartmentIds().contains(departmentId)) {
return true; return true;
} }
} }

View File

@@ -16,4 +16,6 @@ public interface SemanticQuery {
QueryResult execute(User user) throws SqlParseException; QueryResult execute(User user) throws SqlParseException;
SemanticParseInfo getParseInfo(); SemanticParseInfo getParseInfo();
void setParseInfo(SemanticParseInfo parseInfo);
} }

View File

@@ -13,11 +13,15 @@ public class DomainSchema {
private Set<SchemaElement> metrics = new HashSet<>(); private Set<SchemaElement> metrics = new HashSet<>();
private Set<SchemaElement> dimensions = new HashSet<>(); private Set<SchemaElement> dimensions = new HashSet<>();
private Set<SchemaElement> dimensionValues = new HashSet<>(); private Set<SchemaElement> dimensionValues = new HashSet<>();
private Set<SchemaElement> entities = new HashSet<>(); private SchemaElement entity = new SchemaElement();
public SchemaElement getElement(SchemaElementType elementType, long elementID) { public SchemaElement getElement(SchemaElementType elementType, long elementID) {
Optional<SchemaElement> element = Optional.empty(); Optional<SchemaElement> element = Optional.empty();
switch (elementType) { switch (elementType) {
case ENTITY:
element = Optional.ofNullable(entity);
break;
case DOMAIN: case DOMAIN:
element = Optional.of(domain); element = Optional.of(domain);
break; break;
@@ -27,9 +31,6 @@ public class DomainSchema {
case DIMENSION: case DIMENSION:
element = dimensions.stream().filter(e -> e.getId() == elementID).findFirst(); element = dimensions.stream().filter(e -> e.getId() == elementID).findFirst();
break; break;
case ENTITY:
element = entities.stream().filter(e -> e.getId() == elementID).findFirst();
break;
case VALUE: case VALUE:
element = dimensionValues.stream().filter(e -> e.getId() == elementID).findFirst(); element = dimensionValues.stream().filter(e -> e.getId() == elementID).findFirst();
default: default:

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat.api.pojo; package com.tencent.supersonic.chat.api.pojo;
import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import lombok.Data; import lombok.Data;
import java.util.ArrayList; import java.util.ArrayList;
@@ -10,11 +10,11 @@ import java.util.List;
@Data @Data
public class QueryContext { public class QueryContext {
private QueryRequest request; private QueryReq request;
private List<SemanticQuery> candidateQueries = new ArrayList<>(); private List<SemanticQuery> candidateQueries = new ArrayList<>();
private SchemaMapInfo mapInfo = new SchemaMapInfo(); private SchemaMapInfo mapInfo = new SchemaMapInfo();
public QueryContext(QueryRequest request) { public QueryContext(QueryReq request) {
this.request = request; this.request = request;
} }
} }

View File

@@ -5,11 +5,13 @@ import com.google.common.base.Objects;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;
import lombok.Builder; import lombok.*;
import lombok.Data;
@Data @Data
@Getter
@Builder @Builder
@NoArgsConstructor
//@AllArgsConstructor
public class SchemaElement implements Serializable { public class SchemaElement implements Serializable {
private Long domain; private Long domain;
@@ -20,8 +22,8 @@ public class SchemaElement implements Serializable {
private SchemaElementType type; private SchemaElementType type;
private List<String> alias; private List<String> alias;
public SchemaElement() { // public SchemaElement() {
} // }
public SchemaElement(Long domain, Long id, String name, String bizName, public SchemaElement(Long domain, Long id, String name, String bizName,
Long useCnt, SchemaElementType type, List<String> alias) { Long useCnt, SchemaElementType type, List<String> alias) {

View File

@@ -18,4 +18,10 @@ public class SchemaElementMatch {
String detectWord; String detectWord;
String word; String word;
Long frequency; Long frequency;
MatchMode mode = MatchMode.CURRENT;
public enum MatchMode {
CURRENT,
INHERIT
}
} }

View File

@@ -1,36 +1,32 @@
package com.tencent.supersonic.chat.api.pojo; package com.tencent.supersonic.chat.api.pojo;
import java.util.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.Data; import lombok.Data;
@Data @Data
public class SemanticParseInfo { public class SemanticParseInfo {
String queryMode; private String queryMode;
SchemaElement domain; private SchemaElement domain;
Set<SchemaElement> metrics = new LinkedHashSet(); private Set<SchemaElement> metrics = new TreeSet<>(new SchemaNameLengthComparator());
Set<SchemaElement> dimensions = new LinkedHashSet(); private Set<SchemaElement> dimensions = new LinkedHashSet();
Long entity = 0L; private SchemaElement entity;
AggregateTypeEnum aggType = AggregateTypeEnum.NONE; private AggregateTypeEnum aggType = AggregateTypeEnum.NONE;
Set<QueryFilter> dimensionFilters = new LinkedHashSet(); private Set<QueryFilter> dimensionFilters = new LinkedHashSet();
Set<QueryFilter> metricFilters = new LinkedHashSet(); private Set<QueryFilter> metricFilters = new LinkedHashSet();
private Set<Order> orders = new LinkedHashSet(); private Set<Order> orders = new LinkedHashSet();
private DateConf dateInfo; private DateConf dateInfo;
private Long limit; private Long limit;
private Boolean nativeQuery = false; private Boolean nativeQuery = false;
private Double bonus = 0d; private double score;
private List<SchemaElementMatch> elementMatches = new ArrayList<>(); private List<SchemaElementMatch> elementMatches = new ArrayList<>();
private Map<String, Object> properties; private Map<String, Object> properties = new HashMap<>();
public Long getDomainId() { public Long getDomainId() {
return domain != null ? domain.getId() : 0L; return domain != null ? domain.getId() : 0L;
@@ -40,8 +36,9 @@ public class SemanticParseInfo {
return domain != null ? domain.getName() : "null"; return domain != null ? domain.getName() : "null";
} }
public Set<SchemaElement> getMetrics() { private static class SchemaNameLengthComparator implements Comparator<SchemaElement> {
this.metrics = this.metrics.stream().sorted((o1, o2) -> { @Override
public int compare(SchemaElement o1, SchemaElement o2) {
int len1 = o1.getName().length(); int len1 = o1.getName().length();
int len2 = o2.getName().length(); int len2 = o2.getName().length();
if (len1 != len2) { if (len1 != len2) {
@@ -49,7 +46,7 @@ public class SemanticParseInfo {
} else { } else {
return o1.getName().compareTo(o2.getName()); return o1.getName().compareTo(o2.getName());
} }
}).collect(Collectors.toCollection(LinkedHashSet::new)); }
return this.metrics;
} }
} }

View File

@@ -48,7 +48,7 @@ public class SemanticSchema implements Serializable {
public List<SchemaElement> getEntities() { public List<SchemaElement> getEntities() {
List<SchemaElement> entities = new ArrayList<>(); List<SchemaElement> entities = new ArrayList<>();
domainSchemaList.stream().forEach(d -> entities.addAll(d.getEntities())); domainSchemaList.stream().forEach(d -> entities.add(d.getEntity()));
return entities; return entities;
} }
} }

View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
@Data @Data
public class ChatAggConfig { public class ChatAggConfigReq {
/** /**
* invisible dimensions/metrics * invisible dimensions/metrics
@@ -15,10 +15,10 @@ public class ChatAggConfig {
/** /**
* information about dictionary about the domain * information about dictionary about the domain
*/ */
private List<KnowledgeInfo> knowledgeInfos; private List<KnowledgeInfoReq> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig; private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultConfig chatDefaultConfig; private ChatDefaultConfigReq chatDefaultConfig;
} }

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion;
import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data; import lombok.Data;
@@ -20,18 +19,18 @@ public class ChatConfigBaseReq {
/** /**
* the chatDetailConfig about the domain * the chatDetailConfig about the domain
*/ */
private ChatDetailConfig chatDetailConfig; private ChatDetailConfigReq chatDetailConfig;
/** /**
* the chatAggConfig about the domain * the chatAggConfig about the domain
*/ */
private ChatAggConfig chatAggConfig; private ChatAggConfigReq chatAggConfig;
/** /**
* the recommended questions about the domain * the recommended questions about the domain
*/ */
private List<RecommendedQuestion> recommendedQuestions; private List<RecommendedQuestionReq> recommendedQuestions;
/** /**
* available status * available status

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
@@ -8,7 +8,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
@Data @Data
public class ChatDefaultConfig { public class ChatDefaultConfigReq {
private List<Long> dimensionIds = new ArrayList<>(); private List<Long> dimensionIds = new ArrayList<>();
private List<Long> metricIds = new ArrayList<>(); private List<Long> metricIds = new ArrayList<>();
@@ -24,4 +24,15 @@ public class ChatDefaultConfig {
*/ */
private String period = Constants.DAY; private String period = Constants.DAY;
private TimeMode timeMode = TimeMode.LAST;
public enum TimeMode {
/**
* date mode
* LAST - a certain time
* RECENT - a period time
*/
LAST, RECENT
}
} }

View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
@Data @Data
public class ChatDetailConfig { public class ChatDetailConfigReq {
/** /**
* invisible dimensions/metrics * invisible dimensions/metrics
@@ -15,15 +15,10 @@ public class ChatDetailConfig {
/** /**
* information about dictionary about the domain * information about dictionary about the domain
*/ */
private List<KnowledgeInfo> knowledgeInfos; private List<KnowledgeInfoReq> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig; private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultConfig chatDefaultConfig; private ChatDefaultConfigReq chatDefaultConfig;
/**
* the entity info about the domain
*/
private Entity entity;
} }

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import java.util.List; import java.util.List;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

View File

@@ -0,0 +1,15 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import lombok.Data;
@Data
public class ExecuteQueryReq {
private User user;
private Integer chatId;
private String queryText;
private SemanticParseInfo parseInfo;
private boolean saveAnswer = true;
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data; import lombok.Data;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.pojo.enums.TypeEnums;
@@ -11,7 +11,7 @@ import lombok.Data;
*/ */
@Data @Data
public class KnowledgeInfo { public class KnowledgeInfoReq {
/** /**
* metricIdDimensionIddomainId * metricIdDimensionIddomainId

View File

@@ -7,10 +7,9 @@ import lombok.Data;
public class PluginQueryReq { public class PluginQueryReq {
private String showElementId; private String name;
//DASHBOARD WIDGET private String parseMode;
private String showType;
private String type; private String type;
@@ -18,5 +17,5 @@ public class PluginQueryReq {
private String pattern; private String pattern;
private String createdBy;
} }

View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.common.pojo.Order;
import lombok.Data; import lombok.Data;
@Data @Data
public class QueryDataRequest { public class QueryDataReq {
String queryMode; String queryMode;
SchemaElement domain; SchemaElement domain;
Set<SchemaElement> metrics = new HashSet<>(); Set<SchemaElement> metrics = new HashSet<>();

View File

@@ -4,8 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import lombok.Data; import lombok.Data;
@Data @Data
public class QueryRequest { public class QueryReq {
private String queryText; private String queryText;
private Integer chatId; private Integer chatId;
private Long domainId = 0L; private Long domainId = 0L;

View File

@@ -1,11 +1,15 @@
package com.tencent.supersonic.chat.api.pojo.request; package com.tencent.supersonic.chat.api.pojo.request;
import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString; import lombok.ToString;
@Data @Data
@ToString @ToString
public class RecommendedQuestion { @AllArgsConstructor
@NoArgsConstructor
public class RecommendedQuestionReq {
private String question; private String question;

View File

@@ -0,0 +1,26 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import lombok.Data;
import java.util.List;
@Data
public class ChatAggRichConfigResp {
/**
* invisible dimensions/metrics
*/
private ItemVisibilityInfo visibility;
/**
* information about dictionary about the domain
*/
private List<KnowledgeInfoReq> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultRichConfigResp chatDefaultConfig;
}

View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date; import java.util.Date;
@@ -15,11 +17,11 @@ public class ChatConfigResp {
private Long domainId; private Long domainId;
private ChatDetailConfig chatDetailConfig; private ChatDetailConfigReq chatDetailConfig;
private ChatAggConfig chatAggConfig; private ChatAggConfigReq chatAggConfig;
private List<RecommendedQuestion> recommendedQuestions; private List<RecommendedQuestionReq> recommendedQuestions;
/** /**
* available status * available status

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
@@ -8,7 +8,7 @@ import java.util.List;
import lombok.Data; import lombok.Data;
@Data @Data
public class ChatConfigRich { public class ChatConfigRichResp {
private Long id; private Long id;
@@ -17,11 +17,11 @@ public class ChatConfigRich {
private String domainName; private String domainName;
private String bizName; private String bizName;
private ChatAggRichConfig chatAggRichConfig; private ChatAggRichConfigResp chatAggRichConfig;
private ChatDetailRichConfig chatDetailRichConfig; private ChatDetailRichConfigResp chatDetailRichConfig;
private List<RecommendedQuestion> recommendedQuestions; private List<RecommendedQuestionReq> recommendedQuestions;
/** /**
* available status * available status

View File

@@ -1,14 +1,15 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
@Data @Data
public class ChatDefaultRichConfig { public class ChatDefaultRichConfigResp {
private List<SchemaElement> dimensions; private List<SchemaElement> dimensions;
private List<SchemaElement> metrics; private List<SchemaElement> metrics;
@@ -25,4 +26,6 @@ public class ChatDefaultRichConfig {
*/ */
private String period = Constants.DAY; private String period = Constants.DAY;
private ChatDefaultConfigReq.TimeMode timeMode;
} }

View File

@@ -0,0 +1,27 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import lombok.Data;
import java.util.List;
@Data
public class ChatDetailRichConfigResp {
/**
* invisible dimensions/metrics
*/
private ItemVisibilityInfo visibility;
/**
* information about dictionary about the domain
*/
private List<KnowledgeInfoReq> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultRichConfigResp chatDefaultConfig;
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement;
@@ -6,7 +6,7 @@ import java.util.List;
import lombok.Data; import lombok.Data;
@Data @Data
public class EntityRichInfo { public class EntityRichInfoResp {
/** /**
* entity alias * entity alias
*/ */

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.api.pojo.response;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;

View File

@@ -7,6 +7,7 @@ import lombok.Data;
public class MetricInfo { public class MetricInfo {
private String name; private String name;
private String dimension;
private String value; private String value;
private String date; private String date;
private Map<String, String> statistics; private Map<String, String> statistics;

View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import lombok.*;
import java.util.List;
@Data
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ParseResp {
private Integer chatId;
private String queryText;
private ParseState state;
private List<SemanticParseInfo> selectedParses;
private List<SemanticParseInfo> candidateParses;
public enum ParseState {
COMPLETED,
PENDING,
FAILED
}
}

View File

@@ -4,7 +4,7 @@ import java.util.Date;
import lombok.Data; import lombok.Data;
@Data @Data
public class QueryResponse { public class QueryResp {
private Long questionId; private Long questionId;
private Date createTime; private Date createTime;

View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.api.pojo.response; package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
@@ -8,7 +8,7 @@ import java.util.List;
@Data @Data
@AllArgsConstructor @AllArgsConstructor
public class RecommendQuestion { public class RecommendQuestionResp {
private Long domainId; private Long domainId;
private List<RecommendedQuestion> recommendedQuestions; private List<RecommendedQuestionReq> recommendedQuestions;
} }

View File

@@ -6,7 +6,7 @@ import lombok.Data;
import java.util.List; import java.util.List;
@Data @Data
public class RecommendResponse { public class RecommendResp {
private List<SchemaElement> dimensions; private List<SchemaElement> dimensions;
private List<SchemaElement> metrics; private List<SchemaElement> metrics;
} }

View File

@@ -6,11 +6,11 @@ import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@Data @Data
public class SearchResponse { public class SearchResp {
private List<SearchResult> searchResults; private List<SearchResult> searchResults;
public SearchResponse(List<SearchResult> searchResults) { public SearchResp(List<SearchResult> searchResults) {
this.searchResults = searchResults; this.searchResults = searchResults;
} }
} }

View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" <project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent> <parent>
<artifactId>chat</artifactId> <artifactId>chat</artifactId>
<groupId>com.tencent.supersonic</groupId> <groupId>com.tencent.supersonic</groupId>
@@ -40,6 +40,11 @@
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency>
<groupId>com.github.plexpt</groupId>
<artifactId>chatgpt</artifactId>
<version>4.1.2</version>
</dependency>
<dependency> <dependency>
<groupId>org.junit.jupiter</groupId> <groupId>org.junit.jupiter</groupId>

View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@Configuration
@Data
public class AggregatorConfig {
@Value("${metric.aggregator.ratio.enable:true}")
private Boolean enableRatio;
}

View File

@@ -1,24 +0,0 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import java.util.List;
@Data
public class ChatAggRichConfig {
/**
* invisible dimensions/metrics
*/
private ItemVisibilityInfo visibility;
/**
* information about dictionary about the domain
*/
private List<KnowledgeInfo> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultRichConfig chatDefaultConfig;
}

View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.chat.config; package com.tencent.supersonic.chat.config;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.Data; import lombok.Data;
@@ -22,14 +24,14 @@ public class ChatConfig {
/** /**
* the chatDetailConfig about the domain * the chatDetailConfig about the domain
*/ */
private ChatDetailConfig chatDetailConfig; private ChatDetailConfigReq chatDetailConfig;
/** /**
* the chatAggConfig about the domain * the chatAggConfig about the domain
*/ */
private ChatAggConfig chatAggConfig; private ChatAggConfigReq chatAggConfig;
private List<RecommendedQuestion> recommendedQuestions; private List<RecommendedQuestionReq> recommendedQuestions;
/** /**
* available status * available status

View File

@@ -1,30 +0,0 @@
package com.tencent.supersonic.chat.config;
import lombok.Data;
import java.util.List;
@Data
public class ChatDetailRichConfig {
/**
* invisible dimensions/metrics
*/
private ItemVisibilityInfo visibility;
/**
* the entity info about the domain
*/
private EntityRichInfo entity;
/**
* information about dictionary about the domain
*/
private List<KnowledgeInfo> knowledgeInfos;
private KnowledgeAdvancedConfig globalKnowledgeConfig;
private ChatDefaultRichConfig chatDefaultConfig;
}

View File

@@ -6,7 +6,7 @@ import org.springframework.context.annotation.Configuration;
@Configuration @Configuration
@Data @Data
public class FunctionCallConfig { public class FunctionCallInfoConfig {
@Value("${functionCall.url:}") @Value("${functionCall.url:}")
private String url; private String url;
} }

View File

@@ -0,0 +1,69 @@
package com.tencent.supersonic.chat.mapper;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class EntityMapper implements SchemaMapper {
@Override
public void map(QueryContext queryContext) {
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
for (Long domainId : schemaMapInfo.getMatchedDomains()) {
List<SchemaElementMatch> schemaElementMatchList = schemaMapInfo.getMatchedElements(domainId);
if (CollectionUtils.isEmpty(schemaElementMatchList)) {
continue;
}
SchemaElement entity = getEntity(domainId);
if (entity == null || entity.getId() == null) {
continue;
}
List<SchemaElementMatch> valueSchemaElements = schemaElementMatchList.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : valueSchemaElements) {
if (!entity.getId().equals(schemaElementMatch.getElement().getId())){
continue;
}
if (!checkExistSameEntitySchemaElements(schemaElementMatch, schemaElementMatchList)) {
SchemaElementMatch entitySchemaElementMath = new SchemaElementMatch();
BeanUtils.copyProperties(schemaElementMatch, entitySchemaElementMath);
entitySchemaElementMath.setElement(entity);
schemaElementMatchList.add(entitySchemaElementMath);
}
schemaElementMatch.getElement().setType(SchemaElementType.ID);
}
}
}
private boolean checkExistSameEntitySchemaElements(SchemaElementMatch valueSchemaElementMatch,
List<SchemaElementMatch> schemaElementMatchList) {
List<SchemaElementMatch> entitySchemaElements = schemaElementMatchList.stream().filter(schemaElementMatch ->
SchemaElementType.ENTITY.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : entitySchemaElements) {
if (schemaElementMatch.getElement().getId().equals(valueSchemaElementMatch.getElement().getId())) {
return true;
}
}
return false;
}
private SchemaElement getEntity(Long domainId) {
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
DomainSchema domainSchema = semanticService.getDomainSchema(domainId);
if (domainSchema != null && domainSchema.getEntity() != null) {
return domainSchema.getEntity();
}
return null;
}
}

View File

@@ -2,13 +2,9 @@ package com.tencent.supersonic.chat.mapper;
import com.hankcs.hanlp.seg.common.Term; import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.chat.api.component.SchemaMapper; import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.knowledge.service.SchemaService; import com.tencent.supersonic.knowledge.service.SchemaService;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.knowledge.utils.HanlpHelper; import com.tencent.supersonic.knowledge.utils.HanlpHelper;
import java.util.ArrayList; import java.util.ArrayList;
@@ -44,7 +40,7 @@ public class FuzzyNameMapper implements SchemaMapper {
} }
private void detectAndAddToSchema(QueryContext queryContext, List<Term> terms, List<SchemaElement> domains, private void detectAndAddToSchema(QueryContext queryContext, List<Term> terms, List<SchemaElement> domains,
SchemaElementType schemaElementType) { SchemaElementType schemaElementType) {
try { try {
Map<String, Set<SchemaElement>> domainResultSet = getResultSet(queryContext, terms, domains); Map<String, Set<SchemaElement>> domainResultSet = getResultSet(queryContext, terms, domains);
@@ -57,7 +53,7 @@ public class FuzzyNameMapper implements SchemaMapper {
} }
private Map<String, Set<SchemaElement>> getResultSet(QueryContext queryContext, List<Term> terms, private Map<String, Set<SchemaElement>> getResultSet(QueryContext queryContext, List<Term> terms,
List<SchemaElement> domains) { List<SchemaElement> domains) {
String queryText = queryContext.getRequest().getQueryText(); String queryText = queryContext.getRequest().getQueryText();

View File

@@ -74,6 +74,10 @@ public class HanlpDictMapper implements SchemaMapper {
Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature); Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature);
SchemaElement element = domainSchema.getElement(elementType, elementID); SchemaElement element = domainSchema.getElement(elementType, elementID);
if(Objects.isNull(element)){
log.info("element is null, elementType:{},elementID:{}", elementType, elementID);
continue;
}
if (element.getType().equals(SchemaElementType.VALUE)) { if (element.getType().equals(SchemaElementType.VALUE)) {
element.setName(mapResult.getName()); element.setName(mapResult.getName());
} }

View File

@@ -1,27 +1,41 @@
package com.tencent.supersonic.chat.mapper; package com.tencent.supersonic.chat.mapper;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.component.SchemaMapper; import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors;
@Slf4j @Slf4j
public class QueryFilterMapper implements SchemaMapper { public class QueryFilterMapper implements SchemaMapper {
private Long FREQUENCY = 9999999L;
private double SIMILARITY = 1.0;
@Override @Override
public void map(QueryContext queryContext) { public void map(QueryContext queryContext) {
QueryRequest queryReq = queryContext.getRequest(); QueryReq queryReq = queryContext.getRequest();
Long domainId = queryReq.getDomainId(); Long domainId = queryReq.getDomainId();
if (domainId == null || domainId <= 0) { if (domainId == null || domainId <= 0) {
return; return;
} }
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
clearOtherSchemaElementMatch(domainId, schemaMapInfo); clearOtherSchemaElementMatch(domainId, schemaMapInfo);
List<SchemaElementMatch> schemaElementMatches = schemaMapInfo.getMatchedElements(domainId);
if (schemaElementMatches == null) {
schemaElementMatches = Lists.newArrayList();
schemaMapInfo.setMatchedElements(domainId, schemaElementMatches);
}
addValueSchemaElementMatch(schemaElementMatches, queryReq.getQueryFilters());
} }
private void clearOtherSchemaElementMatch(Long domainId, SchemaMapInfo schemaMapInfo) { private void clearOtherSchemaElementMatch(Long domainId, SchemaMapInfo schemaMapInfo) {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : schemaMapInfo.getDomainElementMatches().entrySet()) { for (Map.Entry<Long, List<SchemaElementMatch>> entry : schemaMapInfo.getDomainElementMatches().entrySet()) {
if (!entry.getKey().equals(domainId)) { if (!entry.getKey().equals(domainId)) {
entry.getValue().clear(); entry.getValue().clear();
@@ -29,4 +43,44 @@ public class QueryFilterMapper implements SchemaMapper {
} }
} }
private List<SchemaElementMatch> addValueSchemaElementMatch(List<SchemaElementMatch> candidateElementMatches,
QueryFilters queryFilter) {
if (queryFilter == null || CollectionUtils.isEmpty(queryFilter.getFilters())) {
return candidateElementMatches;
}
for (QueryFilter filter : queryFilter.getFilters()) {
if (checkExistSameValueSchemaElementMatch(filter, candidateElementMatches)) {
continue;
}
SchemaElement element = SchemaElement.builder()
.id(filter.getElementID())
.name(String.valueOf(filter.getValue()))
.type(SchemaElementType.VALUE)
.bizName(filter.getBizName())
.build();
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.element(element)
.frequency(FREQUENCY)
.word(String.valueOf(filter.getValue()))
.similarity(SIMILARITY)
.detectWord(filter.getName())
.build();
candidateElementMatches.add(schemaElementMatch);
}
return candidateElementMatches;
}
private boolean checkExistSameValueSchemaElementMatch(QueryFilter queryFilter,
List<SchemaElementMatch> schemaElementMatches) {
List<SchemaElementMatch> valueSchemaElements = schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : valueSchemaElements) {
if (schemaElementMatch.getElement().getId().equals(queryFilter.getElementID())
&& schemaElementMatch.getWord().equals(String.valueOf(queryFilter.getValue()))) {
return true;
}
}
return false;
}
} }

View File

@@ -3,13 +3,16 @@ package com.tencent.supersonic.chat.parser;
import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.utils.Lists; import org.apache.commons.compress.utils.Lists;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -25,31 +28,36 @@ import java.util.stream.Collectors;
public class SatisfactionChecker { public class SatisfactionChecker {
private static final double LONG_TEXT_THRESHOLD = 0.8; private static final double LONG_TEXT_THRESHOLD = 0.8;
private static final double SHORT_TEXT_THRESHOLD = 0.6; private static final double SHORT_TEXT_THRESHOLD = 0.5;
private static final int QUERY_TEXT_LENGTH_THRESHOLD = 10; private static final int QUERY_TEXT_LENGTH_THRESHOLD = 10;
public static final double EMBEDDING_THRESHOLD = 0.2;
public static final double BONUS_THRESHOLD = 100;
// check all the parse info in candidate // check all the parse info in candidate
public static boolean check(QueryContext queryCtx) { public static boolean check(QueryContext queryCtx) {
for (SemanticQuery query : queryCtx.getCandidateQueries()) { for (SemanticQuery query : queryCtx.getCandidateQueries()) {
SemanticParseInfo semanticParseInfo = query.getParseInfo(); if (query instanceof RuleSemanticQuery) {
Long domainId = semanticParseInfo.getDomainId(); if (checkRuleThreshHold(queryCtx.getRequest().getQueryText(), query.getParseInfo())) {
List<SchemaElementMatch> schemaElementMatches = queryCtx.getMapInfo() return true;
.getMatchedElements(domainId); }
if (check(queryCtx.getRequest().getQueryText(), semanticParseInfo, schemaElementMatches)) { } else if (query instanceof PluginSemanticQuery) {
return true; if (checkEmbeddingThreshold(query.getParseInfo())) {
log.info("query mode :{} satisfy check", query.getQueryMode());
return true;
}
} }
} }
return false; return false;
} }
private static boolean checkEmbeddingThreshold(SemanticParseInfo semanticParseInfo) {
Object object = semanticParseInfo.getProperties().get(Constants.CONTEXT);
PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(object), PluginParseResult.class);
return EMBEDDING_THRESHOLD > pluginParseResult.getDistance();
}
//check single parse info //check single parse info
private static boolean check(String text, SemanticParseInfo semanticParseInfo, private static boolean checkRuleThreshHold(String text, SemanticParseInfo semanticParseInfo) {
List<SchemaElementMatch> schemaElementMatches) { List<SchemaElementMatch> schemaElementMatches = semanticParseInfo.getElementMatches();
if (semanticParseInfo.getBonus() != null && semanticParseInfo.getBonus() >= BONUS_THRESHOLD) {
return true;
}
if (CollectionUtils.isEmpty(schemaElementMatches)) { if (CollectionUtils.isEmpty(schemaElementMatches)) {
return false; return false;
} }
@@ -71,6 +79,11 @@ public class SatisfactionChecker {
detectWords.add(schemaElementMatch.getDetectWord()); detectWords.add(schemaElementMatch.getDetectWord());
} }
} }
for (SchemaElementMatch schemaElementMatch : schemaElementMatches) {
if (SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) {
detectWords.add(schemaElementMatch.getDetectWord());
}
}
for (SchemaElement schemaItem : semanticParseInfo.getMetrics()) { for (SchemaElement schemaItem : semanticParseInfo.getMetrics()) {
detectWords.add( detectWords.add(
detectWordMap.getOrDefault(Optional.ofNullable(schemaItem.getId()).orElse(0L), "")); detectWordMap.getOrDefault(Optional.ofNullable(schemaItem.getId()).orElse(0L), ""));
@@ -87,7 +100,7 @@ public class SatisfactionChecker {
if (StringUtils.isNotBlank(dateText) && !dateText.equalsIgnoreCase(Constants.NULL)) { if (StringUtils.isNotBlank(dateText) && !dateText.equalsIgnoreCase(Constants.NULL)) {
detectWords.add(dateText); detectWords.add(dateText);
} }
detectWords.removeIf(word -> !text.contains(word)); detectWords.removeIf(word -> !text.contains(word) && !text.contains(StringUtils.reverse(word)));
//compare the length between detect words and query text //compare the length between detect words and query text
return checkThreshold(text, detectWords, semanticParseInfo); return checkThreshold(text, detectWords, semanticParseInfo);
} }

View File

@@ -5,19 +5,21 @@ import com.google.common.collect.Sets;
import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.config.ChatConfigRich; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.config.EntityRichInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.PluginService; import com.tencent.supersonic.chat.service.PluginService;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.ContextUtils;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
@@ -30,30 +32,43 @@ public class EmbeddingBasedParser implements SemanticParser {
@Override @Override
public void parse(QueryContext queryContext, ChatContext chatContext) { public void parse(QueryContext queryContext, ChatContext chatContext) {
EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class);
if (SatisfactionChecker.check(queryContext) || StringUtils.isBlank(embeddingConfig.getUrl())) { if (StringUtils.isBlank(embeddingConfig.getUrl())) {
return; return;
} }
log.info("EmbeddingBasedParser parser query ctx: {}, chat ctx: {}", queryContext, chatContext); log.info("EmbeddingBasedParser parser query ctx: {}, chat ctx: {}", queryContext, chatContext);
for (Long domainId : getDomainMatched(queryContext)) { Set<Long> domainIds = getDomainMatched(queryContext);
String text = replaceText(queryContext, domainId); String text = queryContext.getRequest().getQueryText();
List<RecallRetrieval> embeddingRetrievals = recallResult(text, hasCandidateQuery(queryContext)); if (!CollectionUtils.isEmpty(domainIds)) {
Optional<Plugin> pluginOptional = choosePlugin(embeddingRetrievals, domainId); for (Long domainId : domainIds) {
if (pluginOptional.isPresent()) { List<SchemaElementMatch> schemaElementMatches = getMatchedElements(queryContext, domainId);
Map<String, RecallRetrieval> embeddingRetrievalMap = embeddingRetrievals.stream() String textReplaced = replaceText(text, schemaElementMatches);
.collect(Collectors.toMap(RecallRetrieval::getId, e -> e, (value1, value2) -> value1)); List<RecallRetrieval> embeddingRetrievals = recallResult(textReplaced, hasCandidateQuery(queryContext));
Plugin plugin = pluginOptional.get(); Optional<Plugin> pluginOptional = choosePlugin(embeddingRetrievals, domainId);
log.info("EmbeddingBasedParser text: {} domain: {} choose plugin: [{} {}]", log.info("domain id :{} embedding result, text:{} embeddingResp:{} ",domainId, textReplaced, embeddingRetrievals);
text, domainId, plugin.getId(), plugin.getName()); pluginOptional.ifPresent(plugin -> buildQuery(plugin, embeddingRetrievals, domainId, textReplaced, queryContext, schemaElementMatches));
PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType());
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(queryContext, domainId,
plugin, embeddingRetrievalMap);
semanticParseInfo.setQueryMode(pluginQuery.getQueryMode());
pluginQuery.setParseInfo(semanticParseInfo);
queryContext.getCandidateQueries().add(pluginQuery);
} }
} else {
List<RecallRetrieval> embeddingRetrievals = recallResult(text, hasCandidateQuery(queryContext));
Optional<Plugin> pluginOptional = choosePlugin(embeddingRetrievals, null);
pluginOptional.ifPresent(plugin -> buildQuery(plugin, embeddingRetrievals, null, text, queryContext, Lists.newArrayList()));
} }
} }
private void buildQuery(Plugin plugin, List<RecallRetrieval> embeddingRetrievals,
Long domainId, String text,
QueryContext queryContext, List<SchemaElementMatch> schemaElementMatches) {
Map<String, RecallRetrieval> embeddingRetrievalMap = embeddingRetrievals.stream()
.collect(Collectors.toMap(RecallRetrieval::getId, e -> e, (value1, value2) -> value1));
log.info("EmbeddingBasedParser text: {} domain: {} choose plugin: [{} {}]",
text, domainId, plugin.getId(), plugin.getName());
PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType());
SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(domainId, plugin, text,
queryContext.getRequest(), embeddingRetrievalMap, schemaElementMatches);
semanticParseInfo.setQueryMode(pluginQuery.getQueryMode());
pluginQuery.setParseInfo(semanticParseInfo);
queryContext.getCandidateQueries().add(pluginQuery);
}
private Set<Long> getDomainMatched(QueryContext queryContext) { private Set<Long> getDomainMatched(QueryContext queryContext) {
Long queryDomainId = queryContext.getRequest().getDomainId(); Long queryDomainId = queryContext.getRequest().getDomainId();
if (queryDomainId != null && queryDomainId > 0) { if (queryDomainId != null && queryDomainId > 0) {
@@ -62,51 +77,61 @@ public class EmbeddingBasedParser implements SemanticParser {
return queryContext.getMapInfo().getMatchedDomains(); return queryContext.getMapInfo().getMatchedDomains();
} }
private SemanticParseInfo buildSemanticParseInfo(QueryContext queryContext, Long domainId, Plugin plugin, private SemanticParseInfo buildSemanticParseInfo(Long domainId, Plugin plugin, String text, QueryReq queryReq,
Map<String, RecallRetrieval> embeddingRetrievalMap) { Map<String, RecallRetrieval> embeddingRetrievalMap,
List<SchemaElementMatch> schemaElementMatches) {
SchemaElement schemaElement = new SchemaElement(); SchemaElement schemaElement = new SchemaElement();
schemaElement.setDomain(domainId); schemaElement.setDomain(domainId);
schemaElement.setId(domainId); schemaElement.setId(domainId);
SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setElementMatches(schemaElementMatches);
semanticParseInfo.setDomain(schemaElement); semanticParseInfo.setDomain(schemaElement);
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); double distance = Double.parseDouble(embeddingRetrievalMap.get(plugin.getId().toString()).getDistance());
if (Double.parseDouble(embeddingRetrievalMap.get(plugin.getId().toString()).getDistance()) < THRESHOLD) { double score = text.length() * (1 - distance);
semanticParseInfo.setBonus(SatisfactionChecker.BONUS_THRESHOLD);
}
Map<String, Object> properties = new HashMap<>(); Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, plugin); PluginParseResult pluginParseResult = new PluginParseResult();
pluginParseResult.setPlugin(plugin);
pluginParseResult.setRequest(queryReq);
pluginParseResult.setDistance(distance);
properties.put(Constants.CONTEXT, pluginParseResult);
semanticParseInfo.setProperties(properties); semanticParseInfo.setProperties(properties);
semanticParseInfo.setElementMatches(schemaMapInfo.getMatchedElements(domainId)); semanticParseInfo.setScore(score);
fillSemanticParseInfo(queryContext, semanticParseInfo); fillSemanticParseInfo(semanticParseInfo);
setEntityId(domainId, semanticParseInfo); setEntity(domainId, semanticParseInfo);
return semanticParseInfo; return semanticParseInfo;
} }
private Optional<Long> getEntityElementId(Long domainId) { private List<SchemaElementMatch> getMatchedElements(QueryContext queryContext, Long domainId) {
ConfigService configService = ContextUtils.getBean(ConfigService.class); SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
ChatConfigRich chatConfigRich = configService.getConfigRichInfo(domainId); List<SchemaElementMatch> schemaElementMatches = schemaMapInfo.getMatchedElements(domainId);
EntityRichInfo entityRichInfo = chatConfigRich.getChatDetailRichConfig().getEntity(); if (schemaElementMatches == null) {
if (entityRichInfo != null) { return Lists.newArrayList();
SchemaElement schemaElement = entityRichInfo.getDimItem();
if (schemaElement != null) {
return Optional.of(schemaElement.getId());
}
} }
return Optional.empty(); QueryReq queryReq = queryContext.getRequest();
QueryFilters queryFilters = queryReq.getQueryFilters();
if (queryFilters == null || CollectionUtils.isEmpty(queryFilters.getFilters())) {
return schemaElementMatches;
}
Map<Long, Object> element = queryFilters.getFilters().stream()
.collect(Collectors.toMap(QueryFilter::getElementID, QueryFilter::getValue, (v1, v2) -> v1));
return schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ENTITY.equals(schemaElementMatch.getElement().getType()))
.filter(schemaElementMatch ->
!element.containsKey(schemaElementMatch.getElement().getId()) || (
element.containsKey(schemaElementMatch.getElement().getId()) &&
element.get(schemaElementMatch.getElement().getId()).toString()
.equalsIgnoreCase(schemaElementMatch.getWord())
))
.collect(Collectors.toList());
} }
private void setEntityId(Long domainId, SemanticParseInfo semanticParseInfo) { private void setEntity(Long domainId, SemanticParseInfo semanticParseInfo) {
Optional<Long> entityElementIdOptional = getEntityElementId(domainId); SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
if (entityElementIdOptional.isPresent()) { DomainSchema domainSchema = semanticService.getDomainSchema(domainId);
Long entityElementId = entityElementIdOptional.get(); if (domainSchema != null && domainSchema.getEntity() != null) {
for (QueryFilter filter : semanticParseInfo.getDimensionFilters()) { semanticParseInfo.setEntity(domainSchema.getEntity());
if (entityElementId.equals(filter.getElementID())) {
String value = String.valueOf(filter.getValue());
if (StringUtils.isNumeric(value)) {
semanticParseInfo.setEntity(Long.parseLong(value));
}
}
}
} }
} }
@@ -120,6 +145,12 @@ public class EmbeddingBasedParser implements SemanticParser {
Map<Long, Plugin> pluginMap = plugins.stream().collect(Collectors.toMap(Plugin::getId, p -> p)); Map<Long, Plugin> pluginMap = plugins.stream().collect(Collectors.toMap(Plugin::getId, p -> p));
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) { for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
Plugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId())); Plugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId()));
if (plugin == null) {
continue;
}
if (domainId == null) {
return Optional.of(plugin);
}
if (!CollectionUtils.isEmpty(plugin.getDomainList()) && plugin.getDomainList().contains(domainId)) { if (!CollectionUtils.isEmpty(plugin.getDomainList()) && plugin.getDomainList().contains(domainId)) {
return Optional.of(plugin); return Optional.of(plugin);
} }
@@ -131,7 +162,6 @@ public class EmbeddingBasedParser implements SemanticParser {
try { try {
PluginManager pluginManager = ContextUtils.getBean(PluginManager.class); PluginManager pluginManager = ContextUtils.getBean(PluginManager.class);
EmbeddingResp embeddingResp = pluginManager.recognize(embeddingText); EmbeddingResp embeddingResp = pluginManager.recognize(embeddingText);
log.info("embedding result, text:{} embeddingResp:{}", embeddingText, embeddingResp);
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval(); List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
if(!CollectionUtils.isEmpty(embeddingRetrievals)){ if(!CollectionUtils.isEmpty(embeddingRetrievals)){
if (hasCandidateQuery) { if (hasCandidateQuery) {
@@ -154,25 +184,38 @@ public class EmbeddingBasedParser implements SemanticParser {
return !CollectionUtils.isEmpty(queryContext.getCandidateQueries()); return !CollectionUtils.isEmpty(queryContext.getCandidateQueries());
} }
private void fillSemanticParseInfo(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { private void fillSemanticParseInfo(SemanticParseInfo semanticParseInfo) {
if (queryContext.getRequest().getQueryFilters() != null) { List<SchemaElementMatch> schemaElementMatches = semanticParseInfo.getElementMatches();
semanticParseInfo.getDimensionFilters() if (!CollectionUtils.isEmpty(schemaElementMatches)) {
.addAll(queryContext.getRequest().getQueryFilters().getFilters()); schemaElementMatches.stream().filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.forEach(schemaElementMatch -> {
QueryFilter queryFilter = new QueryFilter();
queryFilter.setValue(schemaElementMatch.getWord());
queryFilter.setElementID(schemaElementMatch.getElement().getId());
queryFilter.setName(schemaElementMatch.getElement().getName());
queryFilter.setOperator(FilterOperatorEnum.EQUALS);
queryFilter.setBizName(schemaElementMatch.getElement().getBizName());
semanticParseInfo.getDimensionFilters().add(queryFilter);
});
} }
} }
protected String replaceText(QueryContext queryContext, Long domainId) { protected String replaceText(String text, List<SchemaElementMatch> schemaElementMatches) {
String text = queryContext.getRequest().getQueryText();
List<SchemaElementMatch> schemaElementMatches = queryContext.getMapInfo().getMatchedElements(domainId);
if (CollectionUtils.isEmpty(schemaElementMatches)) { if (CollectionUtils.isEmpty(schemaElementMatches)) {
return text; return text;
} }
List<SchemaElementMatch> valueSchemaElementMatches = schemaElementMatches.stream() List<SchemaElementMatch> valueSchemaElementMatches = schemaElementMatches.stream()
.filter(schemaElementMatch -> .filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.collect(Collectors.toList()); .collect(Collectors.toList());
for (SchemaElementMatch schemaElementMatch : valueSchemaElementMatches) { for (SchemaElementMatch schemaElementMatch : valueSchemaElementMatches) {
String detectWord = schemaElementMatch.getDetectWord(); String detectWord = schemaElementMatch.getDetectWord();
if (StringUtils.isBlank(detectWord)) {
continue;
}
text = text.replace(detectWord, ""); text = text.replace(detectWord, "");
} }
return text; return text;

View File

@@ -4,14 +4,10 @@ import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
import com.tencent.supersonic.chat.config.ChatConfigRich;
import com.tencent.supersonic.chat.parser.function.DomainResolver;
import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.List; import java.util.List;
@@ -28,18 +24,6 @@ public class EmbeddingEntityResolver {
this.configService = configService; this.configService = configService;
} }
public Pair<Long, Long> getDomainEntityId(QueryContext queryCtx, ChatContext chatCtx) {
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
Long domainId = domainResolver.resolve(queryCtx, chatCtx);
ChatConfigRich chatConfigRichResp = configService.getConfigRichInfo(domainId);
SchemaElement schemaElement = chatConfigRichResp.getChatDetailRichConfig().getEntity().getDimItem();
if (schemaElement == null) {
return Pair.of(domainId, null);
}
Long entityId = getEntityValue(domainId, schemaElement.getId(), queryCtx, chatCtx);
return Pair.of(domainId, entityId);
}
private Long getEntityValue(Long domainId, Long entityElementId, QueryContext queryCtx, ChatContext chatCtx) { private Long getEntityValue(Long domainId, Long entityElementId, QueryContext queryCtx, ChatContext chatCtx) {
Long entityId = null; Long entityId = null;

View File

@@ -3,9 +3,10 @@ package com.tencent.supersonic.chat.parser.function;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.QueryContext;
import java.util.List;
public interface DomainResolver { public interface DomainResolver {
Long resolve(QueryContext queryContext, ChatContext chatCtx); Long resolve(QueryContext queryContext, ChatContext chatCtx, List<Long> restrictiveDomains);
} }

View File

@@ -6,11 +6,11 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.config.FunctionCallConfig; import com.tencent.supersonic.chat.config.FunctionCallInfoConfig;
import com.tencent.supersonic.chat.parser.ParseMode;
import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginManager; import com.tencent.supersonic.chat.plugin.PluginManager;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.plugin.PluginParseResult; import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
@@ -22,12 +22,15 @@ import com.tencent.supersonic.common.util.ContextUtils;
import java.net.URI; import java.net.URI;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.http.HttpEntity; import org.springframework.http.HttpEntity;
@@ -44,9 +47,12 @@ public class FunctionBasedParser implements SemanticParser {
public static final double FUNCTION_BONUS_THRESHOLD = 200; public static final double FUNCTION_BONUS_THRESHOLD = 200;
public static final double SKIP_DSL_LENGTH = 10;
@Override @Override
public void parse(QueryContext queryCtx, ChatContext chatCtx) { public void parse(QueryContext queryCtx, ChatContext chatCtx) {
FunctionCallConfig functionCallConfig = ContextUtils.getBean(FunctionCallConfig.class); FunctionCallInfoConfig functionCallConfig = ContextUtils.getBean(FunctionCallInfoConfig.class);
PluginService pluginService = ContextUtils.getBean(PluginService.class); PluginService pluginService = ContextUtils.getBean(PluginService.class);
String functionUrl = functionCallConfig.getUrl(); String functionUrl = functionCallConfig.getUrl();
@@ -55,39 +61,57 @@ public class FunctionBasedParser implements SemanticParser {
queryCtx.getRequest().getQueryText()); queryCtx.getRequest().getQueryText());
return; return;
} }
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
Long domainId = domainResolver.resolve(queryCtx, chatCtx); Set<Long> matchedDomains = getMatchDomains(queryCtx);
List<String> functionNames = getFunctionNames(domainId); List<String> functionNames = getFunctionNames(matchedDomains);
log.info("domainId:{},functionNames:{}", domainId, functionNames); log.info("matchedDomains:{},functionNames:{}", matchedDomains, functionNames);
if (Objects.isNull(domainId) || domainId <= 0) {
if (CollectionUtils.isEmpty(functionNames) || CollectionUtils.isEmpty(matchedDomains)) {
return; return;
} }
List<PluginParseConfig> functionDOList = getFunctionDO(queryCtx.getRequest().getDomainId());
FunctionReq functionReq = FunctionReq.builder() FunctionReq functionReq = FunctionReq.builder()
.queryText(queryCtx.getRequest().getQueryText()) .queryText(queryCtx.getRequest().getQueryText())
.functionNames(functionNames).build(); .pluginConfigs(functionDOList).build();
FunctionResp functionResp = requestFunction(functionUrl, functionReq); FunctionResp functionResp = requestFunction(functionUrl, functionReq);
log.info("requestFunction result:{}", functionResp.getToolSelection()); log.info("requestFunction result:{}", functionResp.getToolSelection());
if (Objects.isNull(functionResp) || StringUtils.isBlank(functionResp.getToolSelection())) { if (skipFunction(queryCtx, functionResp)) {
return; return;
} }
PluginParseResult functionCallParseResult = new PluginParseResult(); PluginParseResult functionCallParseResult = new PluginParseResult();
String toolSelection = functionResp.getToolSelection(); String toolSelection = functionResp.getToolSelection();
Optional<Plugin> pluginOptional = pluginService.getPluginByName(toolSelection); Optional<Plugin> pluginOptional = pluginService.getPluginByName(toolSelection);
if (pluginOptional.isPresent()) { if (!pluginOptional.isPresent()) {
toolSelection = pluginOptional.get().getType(); log.info("pluginOptional is not exist:{}, skip the parse", toolSelection);
functionCallParseResult.setPlugin(pluginOptional.get()); return;
} }
Plugin plugin = pluginOptional.get();
toolSelection = plugin.getType();
functionCallParseResult.setPlugin(plugin);
log.info("QueryManager PluginQueryModes:{}", QueryManager.getPluginQueryModes());
PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(toolSelection); PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(toolSelection);
DomainResolver domainResolver = ComponentFactory.getDomainResolver();
Long domainId = domainResolver.resolve(queryCtx, chatCtx, plugin.getDomainList());
log.info("FunctionBasedParser domainId:{}",domainId);
if ((Objects.isNull(domainId) || domainId <= 0) && !plugin.isContainsAllDomain()) {
log.info("domain is null, skip the parse, select tool: {}", toolSelection);
return;
}
if (!plugin.getDomainList().contains(domainId) && !plugin.isContainsAllDomain()) {
return;
}
SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(domainId)); if (Objects.nonNull(domainId) && domainId > 0){
parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(domainId));
}
functionCallParseResult.setRequest(queryCtx.getRequest()); functionCallParseResult.setRequest(queryCtx.getRequest());
Map<String, Object> properties = new HashMap<>(); Map<String, Object> properties = new HashMap<>();
properties.put(Constants.CONTEXT, functionCallParseResult); properties.put(Constants.CONTEXT, functionCallParseResult);
parseInfo.setProperties(properties); parseInfo.setProperties(properties);
parseInfo.setBonus(FUNCTION_BONUS_THRESHOLD); parseInfo.setScore(FUNCTION_BONUS_THRESHOLD);
parseInfo.setQueryMode(semanticQuery.getQueryMode());
SchemaElement domain = new SchemaElement(); SchemaElement domain = new SchemaElement();
domain.setDomain(domainId); domain.setDomain(domainId);
domain.setId(domainId); domain.setId(domainId);
@@ -95,13 +119,61 @@ public class FunctionBasedParser implements SemanticParser {
queryCtx.getCandidateQueries().add(semanticQuery); queryCtx.getCandidateQueries().add(semanticQuery);
} }
private List<String> getFunctionNames(Long domainId) { private Set<Long> getMatchDomains(QueryContext queryCtx) {
Set<Long> result = new HashSet<>();
Long domainId = queryCtx.getRequest().getDomainId();
if (Objects.nonNull(domainId) && domainId > 0) {
result.add(domainId);
return result;
}
return queryCtx.getMapInfo().getMatchedDomains();
}
private boolean skipFunction(QueryContext queryCtx, FunctionResp functionResp) {
if (Objects.isNull(functionResp) || StringUtils.isBlank(functionResp.getToolSelection())) {
return true;
}
String queryText = queryCtx.getRequest().getQueryText();
if (functionResp.getToolSelection().equalsIgnoreCase(DSLQuery.QUERY_MODE)
&& queryText.length() < SKIP_DSL_LENGTH) {
log.info("queryText length is :{}, less than the threshold :{}, skip dsl.", queryText.length(),
SKIP_DSL_LENGTH);
return true;
}
return false;
}
private List<PluginParseConfig> getFunctionDO(Long domainId) {
log.info("user decide domain:{}", domainId);
List<Plugin> plugins = PluginManager.getPlugins();
List<PluginParseConfig> functionDOList = plugins.stream().filter(o -> {
if (o.getParseModeConfig() == null) {
return false;
}
if (!CollectionUtils.isEmpty(o.getDomainList())) {//过滤掉没选主题域的插件
return true;
}
if (domainId == null || domainId <= 0L) {
return true;
} else {
return o.getDomainList().contains(domainId);
}
}).map(o -> {
PluginParseConfig functionCallConfig = JsonUtil.toObject(o.getParseModeConfig(),
PluginParseConfig.class);
return functionCallConfig;
}).collect(Collectors.toList());
log.info("getFunctionDO:{}", JsonUtil.toString(functionDOList));
return functionDOList;
}
private List<String> getFunctionNames(Set<Long> matchedDomains) {
List<Plugin> plugins = PluginManager.getPlugins(); List<Plugin> plugins = PluginManager.getPlugins();
Set<String> functionNames = plugins.stream() Set<String> functionNames = plugins.stream()
.filter(entry -> ParseMode.FUNCTION_CALL.equals(entry.getParseMode()))
.filter(entry -> { .filter(entry -> {
if (!CollectionUtils.isEmpty(entry.getDomainList())) { if (!CollectionUtils.isEmpty(entry.getDomainList()) && !CollectionUtils.isEmpty(matchedDomains)) {
return entry.getDomainList().contains(domainId); return entry.getDomainList().stream().anyMatch(matchedDomains::contains);
} }
return true; return true;
} }
@@ -118,7 +190,7 @@ public class FunctionBasedParser implements SemanticParser {
URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri(); URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
try { try {
log.info("requestFunction functionReq:{}", functionReq); log.info("requestFunction functionReq:{}", JsonUtil.toString(functionReq));
ResponseEntity<FunctionResp> responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity, ResponseEntity<FunctionResp> responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity,
FunctionResp.class); FunctionResp.class);
log.info("requestFunction responseEntity:{},cost:{}", responseEntity, log.info("requestFunction responseEntity:{},cost:{}", responseEntity,

View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.chat.parser.function;
import lombok.Data;
@Data
public class FunctionFiled {
private String type;
private String description;
}

View File

@@ -1,6 +1,8 @@
package com.tencent.supersonic.chat.parser.function; package com.tencent.supersonic.chat.parser.function;
import java.util.List; import java.util.List;
import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
@@ -10,6 +12,6 @@ public class FunctionReq {
private String queryText; private String queryText;
private List<String> functionNames; private List<PluginParseConfig> pluginConfigs;
} }

View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat.parser.function; package com.tencent.supersonic.chat.parser.function;
import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -25,10 +25,10 @@ public class HeuristicDomainResolver implements DomainResolver {
Map.Entry<Long, DomainMatchResult> maxDomain = domainTypeMap.entrySet().stream() Map.Entry<Long, DomainMatchResult> maxDomain = domainTypeMap.entrySet().stream()
.filter(entry -> domainQueryModes.containsKey(entry.getKey())) .filter(entry -> domainQueryModes.containsKey(entry.getKey()))
.sorted((o1, o2) -> { .sorted((o1, o2) -> {
int difference = o1.getValue().getCount() - o2.getValue().getCount(); int difference = o2.getValue().getCount() - o1.getValue().getCount();
if (difference == 0) { if (difference == 0) {
return (int) ((o1.getValue().getMaxSimilarity() return (int) ((o2.getValue().getMaxSimilarity()
- o2.getValue().getMaxSimilarity()) * 100); - o1.getValue().getMaxSimilarity()) * 100);
} }
return difference; return difference;
}).findFirst().orElse(null); }).findFirst().orElse(null);
@@ -46,7 +46,7 @@ public class HeuristicDomainResolver implements DomainResolver {
* @return false will use context domain, true will use other domain , maybe include context domain * @return false will use context domain, true will use other domain , maybe include context domain
*/ */
protected static boolean isAllowSwitch(Map<Long, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap, protected static boolean isAllowSwitch(Map<Long, SemanticQuery> domainQueryModes, SchemaMapInfo schemaMap,
ChatContext chatCtx, QueryRequest searchCtx, Long domainId) { ChatContext chatCtx, QueryReq searchCtx, Long domainId, List<Long> restrictiveDomains) {
if (!Objects.nonNull(domainId) || domainId <= 0) { if (!Objects.nonNull(domainId) || domainId <= 0) {
return true; return true;
} }
@@ -81,6 +81,9 @@ public class HeuristicDomainResolver implements DomainResolver {
} }
} }
if (CollectionUtils.isNotEmpty(restrictiveDomains) && !restrictiveDomains.contains(domainId)) {
return true;
}
// if context domain not in schemaMap , will switch // if context domain not in schemaMap , will switch
if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) { if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) {
log.info("domainId not in schemaMap "); log.info("domainId not in schemaMap ");
@@ -118,24 +121,36 @@ public class HeuristicDomainResolver implements DomainResolver {
} }
public Long resolve(QueryContext queryContext, ChatContext chatCtx) { public Long resolve(QueryContext queryContext, ChatContext chatCtx, List<Long> restrictiveDomains) {
Long domainId = queryContext.getRequest().getDomainId(); Long domainId = queryContext.getRequest().getDomainId();
if (Objects.nonNull(domainId) && domainId > 0) { if (Objects.nonNull(domainId) && domainId > 0) {
return domainId; if (CollectionUtils.isNotEmpty(restrictiveDomains) && restrictiveDomains.contains(domainId)) {
return domainId;
} else {
return null;
}
} }
SchemaMapInfo mapInfo = queryContext.getMapInfo(); SchemaMapInfo mapInfo = queryContext.getMapInfo();
Set<Long> matchedDomains = mapInfo.getMatchedDomains(); Set<Long> matchedDomains = mapInfo.getMatchedDomains();
if (CollectionUtils.isNotEmpty(restrictiveDomains)) {
matchedDomains = matchedDomains.stream()
.filter(matchedDomain -> restrictiveDomains.contains(matchedDomain))
.collect(Collectors.toSet());
}
Map<Long, SemanticQuery> domainQueryModes = new HashMap<>(); Map<Long, SemanticQuery> domainQueryModes = new HashMap<>();
for (Long matchedDomain : matchedDomains) { for (Long matchedDomain : matchedDomains) {
domainQueryModes.put(matchedDomain, null); domainQueryModes.put(matchedDomain, null);
} }
if(domainQueryModes.size()==1){
return domainQueryModes.keySet().stream().findFirst().get();
}
return resolve(domainQueryModes, queryContext, chatCtx, return resolve(domainQueryModes, queryContext, chatCtx,
queryContext.getMapInfo()); queryContext.getMapInfo(),restrictiveDomains);
} }
public Long resolve(Map<Long, SemanticQuery> domainQueryModes, QueryContext queryContext, public Long resolve(Map<Long, SemanticQuery> domainQueryModes, QueryContext queryContext,
ChatContext chatCtx, SchemaMapInfo schemaMap) { ChatContext chatCtx, SchemaMapInfo schemaMap, List<Long> restrictiveDomains) {
Long selectDomain = selectDomain(domainQueryModes, queryContext.getRequest(), chatCtx, schemaMap); Long selectDomain = selectDomain(domainQueryModes, queryContext.getRequest(), chatCtx, schemaMap,restrictiveDomains);
if (selectDomain > 0) { if (selectDomain > 0) {
log.info("selectDomain {} ", selectDomain); log.info("selectDomain {} ", selectDomain);
return selectDomain; return selectDomain;
@@ -144,9 +159,9 @@ public class HeuristicDomainResolver implements DomainResolver {
return selectDomainBySchemaElementCount(domainQueryModes, schemaMap); return selectDomainBySchemaElementCount(domainQueryModes, schemaMap);
} }
public Long selectDomain(Map<Long, SemanticQuery> domainQueryModes, QueryRequest queryContext, public Long selectDomain(Map<Long, SemanticQuery> domainQueryModes, QueryReq queryContext,
ChatContext chatCtx, ChatContext chatCtx,
SchemaMapInfo schemaMap) { SchemaMapInfo schemaMap, List<Long> restrictiveDomains) {
// if QueryContext has domainId and in domainQueryModes // if QueryContext has domainId and in domainQueryModes
if (domainQueryModes.containsKey(queryContext.getDomainId())) { if (domainQueryModes.containsKey(queryContext.getDomainId())) {
log.info("selectDomain from QueryContext [{}]", queryContext.getDomainId()); log.info("selectDomain from QueryContext [{}]", queryContext.getDomainId());
@@ -155,7 +170,7 @@ public class HeuristicDomainResolver implements DomainResolver {
// if ChatContext has domainId and in domainQueryModes // if ChatContext has domainId and in domainQueryModes
if (chatCtx.getParseInfo().getDomainId() > 0) { if (chatCtx.getParseInfo().getDomainId() > 0) {
Long domainId = chatCtx.getParseInfo().getDomainId(); Long domainId = chatCtx.getParseInfo().getDomainId();
if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, queryContext, domainId)) { if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, queryContext, domainId,restrictiveDomains)) {
log.info("selectDomain from ChatContext [{}]", domainId); log.info("selectDomain from ChatContext [{}]", domainId);
return domainId; return domainId;
} }
@@ -163,4 +178,4 @@ public class HeuristicDomainResolver implements DomainResolver {
// default 0 // default 0
return 0L; return 0L;
} }
} }

View File

@@ -0,0 +1,17 @@
package com.tencent.supersonic.chat.parser.function;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
public class Parameters {
//default: object
private String type = "object";
private Map<String, FunctionFiled> properties;
private List<String> required;
}

View File

@@ -0,0 +1,49 @@
package com.tencent.supersonic.chat.parser.llm;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.utils.ChatGptHelper;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LLMTimeEnhancementParse implements SemanticParser {
@Override
public void parse(QueryContext queryContext, ChatContext chatContext) {
log.info("before queryContext:{},chatContext:{}",queryContext,chatContext);
ChatGptHelper chatGptHelper = ContextUtils.getBean(ChatGptHelper.class);
String inferredTime = chatGptHelper.inferredTime(queryContext.getRequest().getQueryText());
try {
if (!queryContext.getCandidateQueries().isEmpty()) {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
DateConf dateInfo = query.getParseInfo().getDateInfo();
JSONObject jsonObject = JSON.parseObject(inferredTime);
if (jsonObject.containsKey("date")){
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(jsonObject.getString("date"));
dateInfo.setEndDate(jsonObject.getString("date"));
query.getParseInfo().setDateInfo(dateInfo);
}else if (jsonObject.containsKey("start")){
dateInfo.setDateMode(DateConf.DateMode.BETWEEN);
dateInfo.setStartDate(jsonObject.getString("start"));
dateInfo.setEndDate(jsonObject.getString("end"));
query.getParseInfo().setDateInfo(dateInfo);
}
}
}
}catch (Exception exception){
log.error("{} parse error,this reason is:{}",LLMTimeEnhancementParse.class.getSimpleName(), (Object) exception.getStackTrace());
}
log.info("after queryContext:{},chatContext:{}",queryContext,chatContext);
}
}

View File

@@ -1,22 +1,28 @@
package com.tencent.supersonic.chat.parser.rule; package com.tencent.supersonic.chat.parser.rule;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.AVG;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.COUNT;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.DISTINCT;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MAX;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MIN;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.TOPN;
import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import java.util.AbstractMap;
import java.util.*; import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.*;
@Slf4j @Slf4j
public class AggregateTypeParser implements SemanticParser { public class AggregateTypeParser implements SemanticParser {
@@ -29,7 +35,7 @@ public class AggregateTypeParser implements SemanticParser {
new AbstractMap.SimpleEntry<>(DISTINCT, Pattern.compile("(?i)(uv)")), new AbstractMap.SimpleEntry<>(DISTINCT, Pattern.compile("(?i)(uv)")),
new AbstractMap.SimpleEntry<>(COUNT, Pattern.compile("(?i)(总数|pv)")), new AbstractMap.SimpleEntry<>(COUNT, Pattern.compile("(?i)(总数|pv)")),
new AbstractMap.SimpleEntry<>(NONE, Pattern.compile("(?i)(明细)")) new AbstractMap.SimpleEntry<>(NONE, Pattern.compile("(?i)(明细)"))
).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,(k1,k2)->k2));
@Override @Override
public void parse(QueryContext queryContext, ChatContext chatContext) { public void parse(QueryContext queryContext, ChatContext chatContext) {

View File

@@ -1,17 +1,25 @@
package com.tencent.supersonic.chat.parser.rule; package com.tencent.supersonic.chat.parser.rule;
import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
import java.util.*; import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*; import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
@@ -23,7 +31,8 @@ public class ContextInheritParser implements SemanticParser {
new AbstractMap.SimpleEntry<>(DIMENSION, Arrays.asList(DIMENSION, VALUE)), new AbstractMap.SimpleEntry<>(DIMENSION, Arrays.asList(DIMENSION, VALUE)),
new AbstractMap.SimpleEntry<>(VALUE, Arrays.asList(VALUE, DIMENSION)), new AbstractMap.SimpleEntry<>(VALUE, Arrays.asList(VALUE, DIMENSION)),
new AbstractMap.SimpleEntry<>(ENTITY, Arrays.asList(ENTITY)), new AbstractMap.SimpleEntry<>(ENTITY, Arrays.asList(ENTITY)),
new AbstractMap.SimpleEntry<>(DOMAIN, Arrays.asList(DOMAIN)) new AbstractMap.SimpleEntry<>(DOMAIN, Arrays.asList(DOMAIN)),
new AbstractMap.SimpleEntry<>(ID, Arrays.asList(ID))
).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
@Override @Override
@@ -40,7 +49,9 @@ public class ContextInheritParser implements SemanticParser {
for (SchemaElementMatch match : chatContext.getParseInfo().getElementMatches()) { for (SchemaElementMatch match : chatContext.getParseInfo().getElementMatches()) {
SchemaElementType matchType = match.getElement().getType(); SchemaElementType matchType = match.getElement().getType();
// mutual exclusive element types should not be inherited // mutual exclusive element types should not be inherited
if (!containsTypes(elementMatches, MUTUAL_EXCLUSIVE_MAP.get(matchType))) { RuleSemanticQuery ruleQuery = QueryManager.getRuleQuery(chatContext.getParseInfo().getQueryMode());
if (!containsTypes(elementMatches, matchType, ruleQuery)) {
match.setMode(SchemaElementMatch.MatchMode.INHERIT);
matchesToInherit.add(match); matchesToInherit.add(match);
} }
} }
@@ -53,22 +64,31 @@ public class ContextInheritParser implements SemanticParser {
} }
} }
private boolean containsTypes(List<SchemaElementMatch> matches, List<SchemaElementType> types) { private boolean containsTypes(List<SchemaElementMatch> matches, SchemaElementType matchType,
return matches.stream().anyMatch(m -> types.contains(m.getElement().getType())); RuleSemanticQuery ruleQuery) {
List<SchemaElementType> types = MUTUAL_EXCLUSIVE_MAP.get(matchType);
return matches.stream().anyMatch(m -> {
SchemaElementType type = m.getElement().getType();
if (Objects.nonNull(ruleQuery) && ruleQuery instanceof MetricSemanticQuery
&& !(ruleQuery instanceof MetricEntityQuery)) {
return types.contains(type);
}
return type.equals(matchType);
});
} }
protected boolean shouldInherit(QueryContext queryContext, ChatContext chatContext) { protected boolean shouldInherit(QueryContext queryContext, ChatContext chatContext) {
if (queryContext.getMapInfo().getMatchedElements( Long contextDomainId = chatContext.getParseInfo().getDomainId();
chatContext.getParseInfo().getDomainId()) == null) { if (queryContext.getMapInfo().getMatchedElements(contextDomainId) == null) {
return false; return false;
} }
// if candidates have only one MetricDomain mode and context has value filter , count in context // if candidates have only one MetricDomain mode and context has value filter , count in context
if (queryContext.getCandidateQueries().size() == 1 && (queryContext.getCandidateQueries() List<SemanticQuery> candidateQueries = queryContext.getCandidateQueries().stream()
.get(0) instanceof MetricDomainQuery) .filter(semanticQuery -> semanticQuery.getParseInfo().getDomainId().equals(contextDomainId)).collect(
&& queryContext.getCandidateQueries().get(0).getParseInfo().getDomainId() Collectors.toList());
.equals(chatContext.getParseInfo().getDomainId()) if (candidateQueries.size() == 1 && (candidateQueries.get(0) instanceof MetricDomainQuery)) {
&& !CollectionUtils.isEmpty(chatContext.getParseInfo().getDimensionFilters())) {
return true; return true;
} else { } else {
return queryContext.getCandidateQueries().size() == 0; return queryContext.getCandidateQueries().size() == 0;

View File

@@ -6,7 +6,6 @@ import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import java.util.*; import java.util.*;
import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j

View File

@@ -46,7 +46,7 @@ public class TimeRangeParser implements SemanticParser {
for (SemanticQuery query : queryContext.getCandidateQueries()) { for (SemanticQuery query : queryContext.getCandidateQueries()) {
query.getParseInfo().setDateInfo(dateConf); query.getParseInfo().setDateInfo(dateConf);
} }
} else if(QueryManager.containsRuleQuery(chatContext.getParseInfo().getQueryMode())) { } else if (QueryManager.containsRuleQuery(chatContext.getParseInfo().getQueryMode())) {
RuleSemanticQuery semanticQuery = QueryManager.createRuleQuery( RuleSemanticQuery semanticQuery = QueryManager.createRuleQuery(
chatContext.getParseInfo().getQueryMode()); chatContext.getParseInfo().getQueryMode());
// inherit parse info from context // inherit parse info from context
@@ -64,7 +64,7 @@ public class TimeRangeParser implements SemanticParser {
List<TimeNLP> times = TimeNLPUtil.parse(queryText); List<TimeNLP> times = TimeNLPUtil.parse(queryText);
if (times.size() > 0) { if (times.size() > 0) {
startDate = times.get(0).getTime(); startDate = times.get(0).getTime();
}else { } else {
return null; return null;
} }
@@ -133,7 +133,7 @@ public class TimeRangeParser implements SemanticParser {
info.setPeriod(Constants.DAY); info.setPeriod(Constants.DAY);
} }
days = days * num; days = days * num;
info.setDateMode(DateConf.DateMode.RECENT_UNITS); info.setDateMode(DateConf.DateMode.RECENT);
String text = "" + num + zhPeriod; String text = "" + num + zhPeriod;
if (Strings.isNotEmpty(m.group("periodStr"))) { if (Strings.isNotEmpty(m.group("periodStr"))) {
text = m.group("periodStr"); text = m.group("periodStr");
@@ -175,11 +175,11 @@ public class TimeRangeParser implements SemanticParser {
private DateConf getDateConf(Date startDate, Date endDate) { private DateConf getDateConf(Date startDate, Date endDate) {
if (startDate == null || endDate == null) { if (startDate == null || endDate == null) {
return null; return null;
} }
DateConf info = new DateConf(); DateConf info = new DateConf();
info.setDateMode(DateConf.DateMode.BETWEEN_CONTINUOUS); info.setDateMode(DateConf.DateMode.BETWEEN);
info.setStartDate(DATE_FORMAT.format(startDate)); info.setStartDate(DATE_FORMAT.format(startDate));
info.setEndDate(DATE_FORMAT.format(endDate)); info.setEndDate(DATE_FORMAT.format(endDate));
return info; return info;

View File

@@ -53,11 +53,21 @@ public class PluginDO {
*/ */
private String updatedBy; private String updatedBy;
/**
*
*/
private String parseModeConfig;
/** /**
* *
*/ */
private String config; private String config;
/**
*
*/
private String comment;
/** /**
* *
* @return id * @return id
@@ -218,6 +228,22 @@ public class PluginDO {
this.updatedBy = updatedBy == null ? null : updatedBy.trim(); this.updatedBy = updatedBy == null ? null : updatedBy.trim();
} }
/**
*
* @return parse_mode_config
*/
public String getParseModeConfig() {
return parseModeConfig;
}
/**
*
* @param parseModeConfig
*/
public void setParseModeConfig(String parseModeConfig) {
this.parseModeConfig = parseModeConfig == null ? null : parseModeConfig.trim();
}
/** /**
* *
* @return config * @return config
@@ -233,4 +259,20 @@ public class PluginDO {
public void setConfig(String config) { public void setConfig(String config) {
this.config = config == null ? null : config.trim(); this.config = config == null ? null : config.trim();
} }
/**
*
* @return comment
*/
public String getComment() {
return comment;
}
/**
*
* @param comment
*/
public void setComment(String comment) {
this.comment = comment == null ? null : comment.trim();
}
} }

View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.chat.persistence.mapper; package com.tencent.supersonic.chat.persistence.mapper;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDO; import com.tencent.supersonic.chat.persistence.dataobject.PluginDO;
import com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample; import com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample;
import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Mapper;

View File

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.persistence.repository;
import com.tencent.supersonic.chat.config.ChatConfig; import com.tencent.supersonic.chat.config.ChatConfig;
import com.tencent.supersonic.chat.config.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.config.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import java.util.List; import java.util.List;

View File

@@ -2,17 +2,17 @@ package com.tencent.supersonic.chat.persistence.repository;
import com.github.pagehelper.PageInfo; import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.api.pojo.response.QueryResponse; import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
public interface ChatQueryRepository { public interface ChatQueryRepository {
PageInfo<QueryResponse> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId); PageInfo<QueryResp> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId);
void createChatQuery(QueryResult queryResult, QueryRequest queryContext, ChatContext chatCtx); void createChatQuery(QueryResult queryResult, ChatContext chatCtx);
ChatQueryDO getLastChatQuery(long chatId); ChatQueryDO getLastChatQuery(long chatId);

View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.chat.persistence.repository.impl; package com.tencent.supersonic.chat.persistence.repository.impl;
import com.tencent.supersonic.chat.config.ChatConfig; import com.tencent.supersonic.chat.config.ChatConfig;
import com.tencent.supersonic.chat.config.ChatConfigFilter; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
import com.tencent.supersonic.chat.config.ChatConfigFilterInternal; import com.tencent.supersonic.chat.config.ChatConfigFilterInternal;
import com.tencent.supersonic.chat.config.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO;
import com.tencent.supersonic.chat.persistence.repository.ChatConfigRepository; import com.tencent.supersonic.chat.persistence.repository.ChatConfigRepository;
import com.tencent.supersonic.chat.utils.ChatConfigHelper; import com.tencent.supersonic.chat.utils.ChatConfigHelper;

View File

@@ -3,12 +3,12 @@ package com.tencent.supersonic.chat.persistence.repository.impl;
import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo; import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample;
import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample.Criteria; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample.Criteria;
import com.tencent.supersonic.chat.api.pojo.response.QueryResponse; import com.tencent.supersonic.chat.api.pojo.response.QueryResp;
import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq;
import com.tencent.supersonic.chat.persistence.mapper.ChatQueryDOMapper; import com.tencent.supersonic.chat.persistence.mapper.ChatQueryDOMapper;
import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository; import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository;
@@ -35,7 +35,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
} }
@Override @Override
public PageInfo<QueryResponse> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId) { public PageInfo<QueryResp> getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId) {
ChatQueryDOExample example = new ChatQueryDOExample(); ChatQueryDOExample example = new ChatQueryDOExample();
example.setOrderByClause("question_id desc"); example.setOrderByClause("question_id desc");
Criteria criteria = example.createCriteria(); Criteria criteria = example.createCriteria();
@@ -46,7 +46,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
pageQueryInfoCommend.getPageSize()) pageQueryInfoCommend.getPageSize())
.doSelectPageInfo(() -> chatQueryDOMapper.selectByExampleWithBLOBs(example)); .doSelectPageInfo(() -> chatQueryDOMapper.selectByExampleWithBLOBs(example));
PageInfo<QueryResponse> chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo); PageInfo<QueryResp> chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo);
chatQueryVOPageInfo.setList( chatQueryVOPageInfo.setList(
pageInfo.getList().stream().map(this::convertTo) pageInfo.getList().stream().map(this::convertTo)
.sorted(Comparator.comparingInt(o -> o.getQuestionId().intValue())) .sorted(Comparator.comparingInt(o -> o.getQuestionId().intValue()))
@@ -54,8 +54,8 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
return chatQueryVOPageInfo; return chatQueryVOPageInfo;
} }
private QueryResponse convertTo(ChatQueryDO chatQueryDO) { private QueryResp convertTo(ChatQueryDO chatQueryDO) {
QueryResponse queryResponse = new QueryResponse(); QueryResp queryResponse = new QueryResp();
BeanUtils.copyProperties(chatQueryDO, queryResponse); BeanUtils.copyProperties(chatQueryDO, queryResponse);
QueryResult queryResult = JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class); QueryResult queryResult = JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class);
queryResult.setQueryId(chatQueryDO.getQuestionId()); queryResult.setQueryId(chatQueryDO.getQuestionId());
@@ -64,16 +64,16 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository {
} }
@Override @Override
public void createChatQuery(QueryResult queryResult, QueryRequest queryRequest, ChatContext chatCtx) { public void createChatQuery(QueryResult queryResult, ChatContext chatCtx) {
ChatQueryDO chatQueryDO = new ChatQueryDO(); ChatQueryDO chatQueryDO = new ChatQueryDO();
chatQueryDO.setChatId(Long.valueOf(queryRequest.getChatId())); chatQueryDO.setChatId(Long.valueOf(chatCtx.getChatId()));
chatQueryDO.setCreateTime(new java.util.Date()); chatQueryDO.setCreateTime(new java.util.Date());
chatQueryDO.setUserName(queryRequest.getUser().getName()); chatQueryDO.setUserName(chatCtx.getUser());
chatQueryDO.setQueryState(queryResult.getQueryState().ordinal()); chatQueryDO.setQueryState(queryResult.getQueryState().ordinal());
chatQueryDO.setQueryText(queryRequest.getQueryText()); chatQueryDO.setQueryText(chatCtx.getQueryText());
chatQueryDO.setQueryResult(JsonUtil.toString(queryResult)); chatQueryDO.setQueryResult(JsonUtil.toString(queryResult));
chatQueryDOMapper.insert(chatQueryDO); chatQueryDOMapper.insert(chatQueryDO);
ChatQueryDO lastChatQuery = getLastChatQuery(queryRequest.getChatId()); ChatQueryDO lastChatQuery = getLastChatQuery(chatCtx.getChatId());
Long queryId = lastChatQuery.getQuestionId(); Long queryId = lastChatQuery.getQuestionId();
queryResult.setQueryId(queryId); queryResult.setQueryId(queryId);
} }

View File

@@ -1,37 +1,58 @@
package com.tencent.supersonic.chat.plugin; package com.tencent.supersonic.chat.plugin;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.parser.ParseMode; import com.tencent.supersonic.chat.parser.ParseMode;
import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.common.pojo.RecordInfo;
import lombok.Data; import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Data @Data
public class Plugin extends RecordInfo { public class Plugin extends RecordInfo {
private Long id; private Long id;
//plugin type WEB_PAGE WEB_SERVICE /***
* plugin type WEB_PAGE WEB_SERVICE
*/
private String type; private String type;
private List<Long> domainList; private List<Long> domainList = Lists.newArrayList();
//description, for parsing /**
* description, for parsing
*/
private String pattern; private String pattern;
//parse /**
* parse
*/
private ParseMode parseMode; private ParseMode parseMode;
private String parseModeConfig;
private String name; private String name;
//config for different plugin type /**
* config for different plugin type
*/
private String config; private String config;
public List<String> getPatterns() { private String comment;
return Stream.of(getPattern().split("\\|")).collect(Collectors.toList());
public List<String> getExampleQuestionList() {
if (StringUtils.isNotBlank(parseModeConfig)) {
PluginParseConfig pluginParseConfig = JSONObject.parseObject(parseModeConfig, PluginParseConfig.class);
return pluginParseConfig.getExamples();
}
return Lists.newArrayList();
}
public boolean isContainsAllDomain() {
return CollectionUtils.isNotEmpty(domainList) && domainList.contains(-1L);
} }
} }

View File

@@ -16,6 +16,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import org.springframework.core.ParameterizedTypeReference; import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*; import org.springframework.http.*;
@@ -40,7 +41,8 @@ public class PluginManager {
public static List<Plugin> getPlugins() { public static List<Plugin> getPlugins() {
PluginService pluginService = ContextUtils.getBean(PluginService.class); PluginService pluginService = ContextUtils.getBean(PluginService.class);
List<Plugin> pluginList = pluginService.getPluginList(); List<Plugin> pluginList = pluginService.getPluginList().stream().filter(plugin ->
CollectionUtils.isNotEmpty(plugin.getDomainList())).collect(Collectors.toList());
pluginList.addAll(internalPluginMap.values()); pluginList.addAll(internalPluginMap.values());
return new ArrayList<>(pluginList); return new ArrayList<>(pluginList);
} }
@@ -48,7 +50,7 @@ public class PluginManager {
@EventListener @EventListener
public void addPlugin(PluginAddEvent pluginAddEvent) { public void addPlugin(PluginAddEvent pluginAddEvent) {
Plugin plugin = pluginAddEvent.getPlugin(); Plugin plugin = pluginAddEvent.getPlugin();
if (ParseMode.EMBEDDING_RECALL.equals(plugin.getParseMode())) { if (CollectionUtils.isNotEmpty(plugin.getExampleQuestionList())) {
requestEmbeddingPluginAdd(convert(Lists.newArrayList(plugin))); requestEmbeddingPluginAdd(convert(Lists.newArrayList(plugin)));
} }
} }
@@ -57,10 +59,10 @@ public class PluginManager {
public void updatePlugin(PluginUpdateEvent pluginUpdateEvent) { public void updatePlugin(PluginUpdateEvent pluginUpdateEvent) {
Plugin oldPlugin = pluginUpdateEvent.getOldPlugin(); Plugin oldPlugin = pluginUpdateEvent.getOldPlugin();
Plugin newPlugin = pluginUpdateEvent.getNewPlugin(); Plugin newPlugin = pluginUpdateEvent.getNewPlugin();
if (ParseMode.EMBEDDING_RECALL.equals(oldPlugin.getParseMode())) { if (CollectionUtils.isNotEmpty(oldPlugin.getExampleQuestionList())) {
requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(oldPlugin))); requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(oldPlugin)));
} }
if (ParseMode.EMBEDDING_RECALL.equals(newPlugin.getParseMode())) { if (CollectionUtils.isNotEmpty(newPlugin.getExampleQuestionList())) {
requestEmbeddingPluginAdd(convert(Lists.newArrayList(newPlugin))); requestEmbeddingPluginAdd(convert(Lists.newArrayList(newPlugin)));
} }
} }
@@ -68,29 +70,30 @@ public class PluginManager {
@EventListener @EventListener
public void delPlugin(PluginAddEvent pluginAddEvent) { public void delPlugin(PluginAddEvent pluginAddEvent) {
Plugin plugin = pluginAddEvent.getPlugin(); Plugin plugin = pluginAddEvent.getPlugin();
if (ParseMode.EMBEDDING_RECALL.equals(plugin.getParseMode())) { if (CollectionUtils.isNotEmpty(plugin.getExampleQuestionList())) {
requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(plugin))); requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(plugin)));
} }
} }
public void requestEmbeddingPluginDelete(Set<String> ids) { public void requestEmbeddingPluginDelete(Set<String> ids) {
if(CollectionUtils.isEmpty(ids)){ if (CollectionUtils.isEmpty(ids)) {
return; return;
} }
doRequest(embeddingConfig.getDeletePath(), JSONObject.toJSONString(ids)); doRequest(embeddingConfig.getDeletePath(), JSONObject.toJSONString(ids));
} }
public void requestEmbeddingPluginAdd(List<Map<String, String>> maps) {
public void requestEmbeddingPluginAdd(List<Map<String,String>> maps) { if (CollectionUtils.isEmpty(maps)) {
if(CollectionUtils.isEmpty(maps)){
return; return;
} }
doRequest(embeddingConfig.getAddPath(), JSONObject.toJSONString(maps)); doRequest(embeddingConfig.getAddPath(), JSONObject.toJSONString(maps));
} }
public void doRequest(String path, String jsonBody) { public void doRequest(String path, String jsonBody) {
String url = embeddingConfig.getUrl()+ path; if (Strings.isEmpty(embeddingConfig.getUrl())) {
return;
}
String url = embeddingConfig.getUrl() + path;
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON); headers.setContentType(MediaType.APPLICATION_JSON);
headers.setLocation(URI.create(url)); headers.setLocation(URI.create(url));
@@ -99,7 +102,8 @@ public class PluginManager {
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers); HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
log.info("[embedding] equest body :{}, url:{}", jsonBody, url); log.info("[embedding] equest body :{}, url:{}", jsonBody, url);
ResponseEntity<String> responseEntity = ResponseEntity<String> responseEntity =
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference<String>() {}); restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference<String>() {
});
log.info("[embedding] result body:{}", responseEntity); log.info("[embedding] result body:{}", responseEntity);
} }
@@ -111,7 +115,7 @@ public class PluginManager {
} }
public EmbeddingResp recognize(String embeddingText) { public EmbeddingResp recognize(String embeddingText) {
String url = embeddingConfig.getUrl()+ embeddingConfig.getRecognizePath() + "?n_results=" + embeddingConfig.getNResult(); String url = embeddingConfig.getUrl() + embeddingConfig.getRecognizePath() + "?n_results=" + embeddingConfig.getNResult();
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON); headers.setContentType(MediaType.APPLICATION_JSON);
headers.setLocation(URI.create(url)); headers.setLocation(URI.create(url));
@@ -121,10 +125,11 @@ public class PluginManager {
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers); HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
log.info("[embedding] request body:{}, url:{}", jsonBody, url); log.info("[embedding] request body:{}, url:{}", jsonBody, url);
ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity = ResponseEntity<List<EmbeddingResp>> embeddingResponseEntity =
restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference<List<EmbeddingResp>>() {}); restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference<List<EmbeddingResp>>() {
log.info("[embedding] recognize result body:{}",embeddingResponseEntity); });
log.info("[embedding] recognize result body:{}", embeddingResponseEntity);
List<EmbeddingResp> embeddingResps = embeddingResponseEntity.getBody(); List<EmbeddingResp> embeddingResps = embeddingResponseEntity.getBody();
if(CollectionUtils.isNotEmpty(embeddingResps)){ if (CollectionUtils.isNotEmpty(embeddingResps)) {
for (EmbeddingResp embeddingResp : embeddingResps) { for (EmbeddingResp embeddingResp : embeddingResps) {
List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval(); List<RecallRetrieval> embeddingRetrievals = embeddingResp.getRetrieval();
for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) { for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) {
@@ -136,13 +141,13 @@ public class PluginManager {
throw new RuntimeException("get embedding result failed"); throw new RuntimeException("get embedding result failed");
} }
public List<Map<String, String>> convert(List<Plugin> plugins){ public List<Map<String, String>> convert(List<Plugin> plugins) {
List<Map<String, String>> maps = Lists.newArrayList(); List<Map<String, String>> maps = Lists.newArrayList();
for(Plugin plugin : plugins){ for (Plugin plugin : plugins) {
List<String> patterns = plugin.getPatterns(); List<String> exampleQuestions = plugin.getExampleQuestionList();
int num = 0; int num = 0;
for(String pattern : patterns){ for (String pattern : exampleQuestions) {
Map<String,String> map = new HashMap<>(); Map<String, String> map = new HashMap<>();
map.put("preset_query_id", generateUniqueEmbeddingId(num, plugin.getId())); map.put("preset_query_id", generateUniqueEmbeddingId(num, plugin.getId()));
map.put("preset_query", pattern); map.put("preset_query", pattern);
maps.add(map); maps.add(map);
@@ -155,7 +160,7 @@ public class PluginManager {
private Set<String> getEmbeddingId(List<Plugin> plugins) { private Set<String> getEmbeddingId(List<Plugin> plugins) {
Set<String> embeddingIdSet = new HashSet<>(); Set<String> embeddingIdSet = new HashSet<>();
for (Map<String, String> map : convert(plugins)) { for (Map<String, String> map : convert(plugins)) {
embeddingIdSet.addAll(map.keySet()); embeddingIdSet.add(map.get("preset_query_id"));
} }
return embeddingIdSet; return embeddingIdSet;
} }

View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.chat.plugin;
import com.tencent.supersonic.chat.parser.function.Parameters;
import lombok.Data;
import java.io.Serializable;
import java.util.List;
@Data
public class PluginParseConfig implements Serializable {
private String name;
private String description;
public Parameters parameters;
public List<String> examples;
}

View File

@@ -1,11 +1,12 @@
package com.tencent.supersonic.chat.plugin; package com.tencent.supersonic.chat.plugin;
import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import lombok.Data; import lombok.Data;
@Data @Data
public class PluginParseResult { public class PluginParseResult {
private Plugin plugin; private Plugin plugin;
private QueryRequest request; private QueryReq request;
private double distance;
} }

View File

@@ -5,39 +5,46 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
import java.util.HashMap;
import java.util.List; import java.util.*;
import java.util.Map;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
@Slf4j @Slf4j
public class HeuristicQuerySelector implements QuerySelector { public class HeuristicQuerySelector implements QuerySelector {
@Override private static final double MATCH_INHERIT_PENALTY = 0.5;
public SemanticQuery select(List<SemanticQuery> candidateQueries) { private static final double MATCH_CURRENT_REWORD = 2;
double maxScore = 0; private static final double CANDIDATE_THRESHOLD = 0.2;
SemanticQuery pickedQuery = null;
if (CollectionUtils.isNotEmpty(candidateQueries) && candidateQueries.size() == 1) {
return candidateQueries.get(0);
}
for (SemanticQuery query : candidateQueries) {
SemanticParseInfo semanticParse = query.getParseInfo();
double score = computeScore(semanticParse);
if (score > maxScore) {
maxScore = score;
pickedQuery = query;
}
log.info("candidate query (domain={}, queryMode={}) with score={}",
semanticParse.getDomainName(), semanticParse.getQueryMode(), score);
}
return pickedQuery; @Override
public List<SemanticQuery> select(List<SemanticQuery> candidateQueries) {
List<SemanticQuery> selectedQueries = new ArrayList<>();
if (CollectionUtils.isNotEmpty(candidateQueries) && candidateQueries.size() == 1) {
selectedQueries.addAll(candidateQueries);
} else {
OptionalDouble maxScoreOp = candidateQueries.stream().mapToDouble(
q -> computeScore(q.getParseInfo())).max();
if (maxScoreOp.isPresent()) {
double maxScore = maxScoreOp.getAsDouble();
candidateQueries.stream().forEach(query -> {
SemanticParseInfo semanticParse = query.getParseInfo();
if ((maxScore - semanticParse.getScore()) / maxScore <= CANDIDATE_THRESHOLD) {
selectedQueries.add(query);
}
log.info("candidate query (domain={}, queryMode={}) with score={}",
semanticParse.getDomainName(), semanticParse.getQueryMode(), semanticParse.getScore());
});
}
}
return selectedQueries;
} }
private double computeScore(SemanticParseInfo semanticParse) { private double computeScore(SemanticParseInfo semanticParse) {
double score = 0; double totalScore = 0;
Map<SchemaElementType, SchemaElementMatch> maxSimilarityMatch = new HashMap<>(); Map<SchemaElementType, SchemaElementMatch> maxSimilarityMatch = new HashMap<>();
for (SchemaElementMatch match : semanticParse.getElementMatches()) { for (SchemaElementMatch match : semanticParse.getElementMatches()) {
@@ -49,13 +56,19 @@ public class HeuristicQuerySelector implements QuerySelector {
} }
for (SchemaElementMatch match : maxSimilarityMatch.values()) { for (SchemaElementMatch match : maxSimilarityMatch.values()) {
score += double matchScore = Optional.ofNullable(match.getDetectWord()).orElse(Constants.EMPTY).length() * match.getSimilarity();
Optional.ofNullable(match.getDetectWord()).orElse(Constants.EMPTY).length() * match.getSimilarity(); if (match.equals(SchemaElementMatch.MatchMode.INHERIT)) {
matchScore *= MATCH_INHERIT_PENALTY;
} else {
matchScore *= MATCH_CURRENT_REWORD;
}
totalScore += matchScore;
} }
// bonus is a special construct to control the final score // original score in parse info acts like an extra bonus
score += semanticParse.getBonus(); totalScore += semanticParse.getScore();
semanticParse.setScore(totalScore);
return score; return totalScore;
} }
} }

View File

@@ -3,6 +3,9 @@ package com.tencent.supersonic.chat.query;
import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.query.rule.entity.EntitySemanticQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -22,6 +25,14 @@ public class QueryManager {
} }
} }
public static SemanticQuery createQuery(String queryMode) {
if (containsRuleQuery(queryMode)) {
return createRuleQuery(queryMode);
} else {
return createPluginQuery(queryMode);
}
}
public static RuleSemanticQuery createRuleQuery(String queryMode) { public static RuleSemanticQuery createRuleQuery(String queryMode) {
RuleSemanticQuery semanticQuery = ruleQueryMap.get(queryMode); RuleSemanticQuery semanticQuery = ruleQueryMap.get(queryMode);
if (Objects.isNull(semanticQuery)) { if (Objects.isNull(semanticQuery)) {
@@ -45,7 +56,6 @@ public class QueryManager {
throw new RuntimeException("no supported queryMode :" + queryMode); throw new RuntimeException("no supported queryMode :" + queryMode);
} }
} }
public static boolean containsRuleQuery(String queryMode) { public static boolean containsRuleQuery(String queryMode) {
if (queryMode == null) { if (queryMode == null) {
return false; return false;
@@ -53,6 +63,27 @@ public class QueryManager {
return ruleQueryMap.containsKey(queryMode); return ruleQueryMap.containsKey(queryMode);
} }
public static boolean isMetricQuery(String queryMode) {
if (queryMode == null || !ruleQueryMap.containsKey(queryMode)) {
return false;
}
return ruleQueryMap.get(queryMode) instanceof MetricSemanticQuery;
}
public static boolean isEntityQuery(String queryMode) {
if (queryMode == null || !ruleQueryMap.containsKey(queryMode)) {
return false;
}
return ruleQueryMap.get(queryMode) instanceof EntitySemanticQuery;
}
public static RuleSemanticQuery getRuleQuery(String queryMode) {
if (queryMode == null) {
return null;
}
return ruleQueryMap.get(queryMode);
}
public static List<RuleSemanticQuery> getRuleQueries() { public static List<RuleSemanticQuery> getRuleQueries() {
return new ArrayList<>(ruleQueryMap.values()); return new ArrayList<>(ruleQueryMap.values());
} }

View File

@@ -9,5 +9,5 @@ import java.util.List;
**/ **/
public interface QuerySelector { public interface QuerySelector {
SemanticQuery select(List<SemanticQuery> candidateQueries); List<SemanticQuery> select(List<SemanticQuery> candidateQueries);
} }

View File

@@ -0,0 +1,37 @@
package com.tencent.supersonic.chat.query.plugin;
import lombok.Data;
@Data
public class ParamOption {
private ParamType paramType;
private OptionType optionType;
private String key;
private String name;
private String keyAlias;
private Long domainId;
private Long elementId;
private Object value;
/**
* CUSTOM: the value is specified by the user
* SEMANTIC: the value of element
* FORWARD: only forward
*/
public enum ParamType {
CUSTOM, SEMANTIC, FORWARD
}
public enum OptionType {
REQUIRED, OPTIONAL
}
}

View File

@@ -1,22 +1,14 @@
package com.tencent.supersonic.chat.query.plugin; package com.tencent.supersonic.chat.query.plugin;
import com.google.common.collect.Lists;
import lombok.Data; import lombok.Data;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
@Data @Data
public class WebBase { public class WebBase {
private String url; private String url;
//key, id of schema element private List<ParamOption> paramOptions = Lists.newArrayList();
private Map<String, Object> params = new HashMap<>();
//key, value of shcema element
private Map<String, Object> valueParams = new HashMap<>();
//only forward
private Map<String, Object> forwardParam = new HashMap<>();
} }

View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.chat.query.plugin;
import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
@Data
public class WebBaseResult {
private String url;
private List<ParamOption> params = Lists.newArrayList();
}

View File

@@ -62,8 +62,7 @@ public class DSLQuery extends PluginSemanticQuery {
@Override @Override
public QueryResult execute(User user) { public QueryResult execute(User user) {
PluginParseResult functionCallParseResult = (PluginParseResult) parseInfo.getProperties() PluginParseResult functionCallParseResult =JsonUtil.toObject(JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)),PluginParseResult.class);
.get(Constants.CONTEXT);
Long domainId = parseInfo.getDomainId(); Long domainId = parseInfo.getDomainId();
LLMResp llmResp = requestLLM(functionCallParseResult, domainId); LLMResp llmResp = requestLLM(functionCallParseResult, domainId);
if (Objects.isNull(llmResp)) { if (Objects.isNull(llmResp)) {

View File

@@ -1,15 +1,19 @@
package com.tencent.supersonic.chat.query.plugin.webpage; package com.tencent.supersonic.chat.query.plugin.webpage;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.api.pojo.response.QueryState;
import com.tencent.supersonic.chat.config.ChatConfigRich; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.Plugin;
import com.tencent.supersonic.chat.plugin.PluginParseResult;
import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.plugin.ParamOption;
import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery;
import com.tencent.supersonic.chat.query.plugin.WebBase; import com.tencent.supersonic.chat.query.plugin.WebBase;
import com.tencent.supersonic.chat.query.plugin.WebBaseResult;
import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Constants;
@@ -18,10 +22,9 @@ import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.Comparator;
import java.util.HashMap; import java.util.*;
import java.util.List; import java.util.stream.Collectors;
import java.util.Map;
@Slf4j @Slf4j
@Component @Component
@@ -44,13 +47,14 @@ public class WebPageQuery extends PluginSemanticQuery {
QueryResult queryResult = new QueryResult(); QueryResult queryResult = new QueryResult();
queryResult.setQueryMode(QUERY_MODE); queryResult.setQueryMode(QUERY_MODE);
Map<String, Object> properties = parseInfo.getProperties(); Map<String, Object> properties = parseInfo.getProperties();
Plugin plugin = (Plugin) properties.get(Constants.CONTEXT); PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class);
WebPageResponse webPageResponse = buildResponse(plugin); WebPageResponse webPageResponse = buildResponse(pluginParseResult.getPlugin());
queryResult.setResponse(webPageResponse); queryResult.setResponse(webPageResponse);
if (parseInfo.getDomainId() != null && parseInfo.getDomainId() > 0 if (parseInfo.getDomainId() != null && parseInfo.getDomainId() > 0
&& parseInfo.getEntity() != null && parseInfo.getEntity() > 0) { && parseInfo.getEntity() != null && Objects.nonNull(parseInfo.getEntity().getId())
ChatConfigRich chatConfigRichResp = configService.getConfigRichInfo(parseInfo.getDomainId()); && parseInfo.getEntity().getId() > 0) {
updateSemanticParse(chatConfigRichResp, parseInfo.getEntity()); ChatConfigRichResp chatConfigRichResp = configService.getConfigRichInfo(parseInfo.getDomainId());
updateSemanticParse(chatConfigRichResp);
EntityInfo entityInfo = ContextUtils.getBean(SemanticService.class).getEntityInfo(parseInfo, user); EntityInfo entityInfo = ContextUtils.getBean(SemanticService.class).getEntityInfo(parseInfo, user);
queryResult.setEntityInfo(entityInfo); queryResult.setEntityInfo(entityInfo);
} else { } else {
@@ -60,8 +64,7 @@ public class WebPageQuery extends PluginSemanticQuery {
return queryResult; return queryResult;
} }
private void updateSemanticParse(ChatConfigRich chatConfigRichResp, Long entityId) { private void updateSemanticParse(ChatConfigRichResp chatConfigRichResp) {
parseInfo.setEntity(entityId);
SchemaElement domain = new SchemaElement(); SchemaElement domain = new SchemaElement();
domain.setId(chatConfigRichResp.getDomainId()); domain.setId(chatConfigRichResp.getDomainId());
domain.setName(chatConfigRichResp.getDomainName()); domain.setName(chatConfigRichResp.getDomainName());
@@ -74,35 +77,43 @@ public class WebPageQuery extends PluginSemanticQuery {
webPageResponse.setPluginId(plugin.getId()); webPageResponse.setPluginId(plugin.getId());
webPageResponse.setPluginType(plugin.getType()); webPageResponse.setPluginType(plugin.getType());
WebBase webPage = JsonUtil.toObject(plugin.getConfig(), WebBase.class); WebBase webPage = JsonUtil.toObject(plugin.getConfig(), WebBase.class);
fillWebPage(webPage); WebBaseResult webBaseResult = buildWebPageResult(webPage);
webPageResponse.setWebPage(webPage); webPageResponse.setWebPage(webBaseResult);
return webPageResponse; return webPageResponse;
} }
private void fillWebPage(WebBase webPage) { private WebBaseResult buildWebPageResult(WebBase webPage) {
List<SchemaElementMatch> schemaElementMatchList = parseInfo.getElementMatches(); WebBaseResult webBaseResult = new WebBaseResult();
webBaseResult.setUrl(webPage.getUrl());
Map<String, Object> elementValueMap = getElementMap();
if (!CollectionUtils.isEmpty(webPage.getParamOptions()) && !CollectionUtils.isEmpty(elementValueMap)) {
for (ParamOption paramOption : webPage.getParamOptions()) {
if (!ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType())) {
continue;
}
String elementId = String.valueOf(paramOption.getElementId());
Object elementValue = elementValueMap.get(elementId);
paramOption.setValue(elementValue);
}
}
webBaseResult.setParams(webPage.getParamOptions());
return webBaseResult;
}
private Map<String, Object> getElementMap() {
Map<String, Object> elementValueMap = new HashMap<>(); Map<String, Object> elementValueMap = new HashMap<>();
if (!CollectionUtils.isEmpty(schemaElementMatchList) && !CollectionUtils.isEmpty(webPage.getParams()) ) { List<SchemaElementMatch> schemaElementMatchList = parseInfo.getElementMatches();
if (!CollectionUtils.isEmpty(schemaElementMatchList)) {
schemaElementMatchList.stream() schemaElementMatchList.stream()
.filter(schemaElementMatch -> .filter(schemaElementMatch ->
SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())
|| SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()))
.sorted(Comparator.comparingDouble(SchemaElementMatch::getSimilarity)) .sorted(Comparator.comparingDouble(SchemaElementMatch::getSimilarity))
.forEach(schemaElementMatch -> .forEach(schemaElementMatch ->
elementValueMap.put(String.valueOf(schemaElementMatch.getElement().getId()), elementValueMap.put(String.valueOf(schemaElementMatch.getElement().getId()),
schemaElementMatch.getWord())); schemaElementMatch.getWord()));
} }
if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())) { return elementValueMap;
parseInfo.getDimensionFilters().forEach(
filter -> elementValueMap.put(String.valueOf(filter.getElementID()), filter.getValue())
);
}
Map<String, Object> params = webPage.getParams();
for (Map.Entry<String, Object> entry : params.entrySet()) {
String key = entry.getKey();
String elementId = String.valueOf(entry.getValue());
Object elementValue = elementValueMap.get(elementId);
webPage.getValueParams().put(key, elementValue);
}
} }
} }

View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.chat.query.plugin.webpage; package com.tencent.supersonic.chat.query.plugin.webpage;
import com.tencent.supersonic.chat.query.plugin.WebBase; import com.tencent.supersonic.chat.query.plugin.WebBase;
import com.tencent.supersonic.chat.query.plugin.WebBaseResult;
import lombok.Data; import lombok.Data;
import java.util.List; import java.util.List;
@@ -15,8 +16,8 @@ public class WebPageResponse {
private String description; private String description;
private WebBase webPage; private WebBaseResult webPage;
private List<WebBase> moreWebPage; private List<WebBaseResult> moreWebPage;
} }

Some files were not shown because too many files have changed in this diff Show More