diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/adaptor/UserAdaptor.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/adaptor/UserAdaptor.java new file mode 100644 index 000000000..33c4cbe6a --- /dev/null +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/adaptor/UserAdaptor.java @@ -0,0 +1,25 @@ +package com.tencent.supersonic.auth.api.authentication.adaptor; + +import com.tencent.supersonic.auth.api.authentication.pojo.Organization; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.request.UserReq; + +import java.util.List; +import java.util.Set; + +public interface UserAdaptor { + + List getUserNames(); + + List getUserList(); + + List getOrganizationTree(); + + void register(UserReq userReq); + + String login(UserReq userReq); + + List getUserByOrg(String key); + + Set getUserAllOrgId(String userName); +} diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/Organization.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/Organization.java new file mode 100644 index 000000000..a72870955 --- /dev/null +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/pojo/Organization.java @@ -0,0 +1,23 @@ +package com.tencent.supersonic.auth.api.authentication.pojo; + +import com.google.common.collect.Lists; +import lombok.Data; + +import java.util.List; + +@Data +public class Organization { + + private String id; + + private String parentId; + + private String name; + + private String fullName; + + private List subOrganizations = Lists.newArrayList(); + + private boolean isRoot; + +} diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java index c2c625fc9..dcfb3b8ef 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authentication/service/UserService.java @@ -1,9 +1,11 @@ package com.tencent.supersonic.auth.api.authentication.service; +import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.request.UserReq; import java.util.List; +import java.util.Set; public interface UserService { @@ -14,4 +16,10 @@ public interface UserService { void register(UserReq userCmd); String login(UserReq userCmd); + + Set getUserAllOrgId(String userName); + + List getUserByOrg(String key); + + List getOrganizationTree(); } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/application/UserServiceImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java similarity index 60% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/application/UserServiceImpl.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java index b8c4d6fb1..e762ca9a3 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/application/UserServiceImpl.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java @@ -1,35 +1,30 @@ -package com.tencent.supersonic.auth.authentication.application; +package com.tencent.supersonic.auth.authentication.adaptor; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor; +import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword; import com.tencent.supersonic.auth.api.authentication.request.UserReq; -import com.tencent.supersonic.auth.api.authentication.service.UserService; -import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO; -import com.tencent.supersonic.auth.authentication.domain.repository.UserRepository; -import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils; -import java.util.List; -import java.util.stream.Collectors; +import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO; +import com.tencent.supersonic.auth.authentication.persistence.repository.UserRepository; +import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils; +import com.tencent.supersonic.common.util.ContextUtils; import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Service; - -@Service -public class UserServiceImpl implements UserService { - - private UserRepository userRepository; - - private UserTokenUtils userTokenUtils; - - public UserServiceImpl(UserRepository userRepository, UserTokenUtils userTokenUtils) { - this.userRepository = userRepository; - this.userTokenUtils = userTokenUtils; - } +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +public class DefaultUserAdaptor implements UserAdaptor { private List getUserDOList() { + UserRepository userRepository = ContextUtils.getBean(UserRepository.class); return userRepository.getUserList(); } private UserDO getUser(String name) { + UserRepository userRepository = ContextUtils.getBean(UserRepository.class); return userRepository.getUser(name); } @@ -38,22 +33,26 @@ public class UserServiceImpl implements UserService { return getUserDOList().stream().map(UserDO::getName).collect(Collectors.toList()); } - @Override public List getUserList() { List userDOS = getUserDOList(); return userDOS.stream().map(this::convert).collect(Collectors.toList()); } + @Override + public List getOrganizationTree() { + return Lists.newArrayList(); + } + private User convert(UserDO userDO) { User user = new User(); BeanUtils.copyProperties(userDO, user); return user; } - @Override public void register(UserReq userReq) { + UserRepository userRepository = ContextUtils.getBean(UserRepository.class); List userDOS = getUserNames(); if (userDOS.contains(userReq.getName())) { throw new RuntimeException(String.format("user %s exist", userReq.getName())); @@ -65,6 +64,7 @@ public class UserServiceImpl implements UserService { @Override public String login(UserReq userReq) { + UserTokenUtils userTokenUtils = ContextUtils.getBean(UserTokenUtils.class); UserDO userDO = getUser(userReq.getName()); if (userDO == null) { throw new RuntimeException("user not exist,please register"); @@ -77,5 +77,14 @@ public class UserServiceImpl implements UserService { throw new RuntimeException("password not correct, please try again"); } + @Override + public List getUserByOrg(String key) { + return Lists.newArrayList(); + } -} + @Override + public Set getUserAllOrgId(String userName) { + return Sets.newHashSet(); + } + +} \ No newline at end of file diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/AuthenticationIgnore.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationIgnore.java similarity index 79% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/AuthenticationIgnore.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationIgnore.java index 86a45e200..4223c54f6 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/AuthenticationIgnore.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationIgnore.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.auth.authentication.domain.interceptor; +package com.tencent.supersonic.auth.authentication.interceptor; import java.lang.annotation.ElementType; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/AuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java similarity index 94% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/AuthenticationInterceptor.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java index 433a0ddd1..0d52c9566 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/AuthenticationInterceptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java @@ -1,9 +1,9 @@ -package com.tencent.supersonic.auth.authentication.domain.interceptor; +package com.tencent.supersonic.auth.authentication.interceptor; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.constant.UserConstants; -import com.tencent.supersonic.auth.authentication.application.UserServiceImpl; -import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils; +import com.tencent.supersonic.auth.authentication.service.UserServiceImpl; +import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils; import com.tencent.supersonic.common.util.S2ThreadContext; import java.lang.reflect.Field; import java.util.Arrays; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/DefaultAuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java similarity index 92% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/DefaultAuthenticationInterceptor.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java index 23a72cf49..968085d2b 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/DefaultAuthenticationInterceptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java @@ -1,11 +1,11 @@ -package com.tencent.supersonic.auth.authentication.domain.interceptor; +package com.tencent.supersonic.auth.authentication.interceptor; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword; -import com.tencent.supersonic.auth.authentication.application.UserServiceImpl; -import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils; +import com.tencent.supersonic.auth.authentication.service.UserServiceImpl; +import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils; import com.tencent.supersonic.common.pojo.exception.AccessException; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.S2ThreadContext; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/InterceptorFactory.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/InterceptorFactory.java similarity index 93% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/InterceptorFactory.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/InterceptorFactory.java index a67d5caec..b754c3980 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/InterceptorFactory.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/InterceptorFactory.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.auth.authentication.domain.interceptor; +package com.tencent.supersonic.auth.authentication.interceptor; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.support.SpringFactoriesLoader; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/dataobject/UserDO.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDO.java similarity index 95% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/dataobject/UserDO.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDO.java index 334b4fde6..77b4ae9e7 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/dataobject/UserDO.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDO.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.auth.authentication.domain.dataobject; +package com.tencent.supersonic.auth.authentication.persistence.dataobject; public class UserDO { diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/dataobject/UserDOExample.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDOExample.java similarity index 99% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/dataobject/UserDOExample.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDOExample.java index 522407eff..21f01f4ca 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/dataobject/UserDOExample.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/dataobject/UserDOExample.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.auth.authentication.domain.dataobject; +package com.tencent.supersonic.auth.authentication.persistence.dataobject; import java.util.ArrayList; import java.util.List; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/infrastructure/mapper/UserDOMapper.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/mapper/UserDOMapper.java similarity index 53% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/infrastructure/mapper/UserDOMapper.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/mapper/UserDOMapper.java index c3b5dc43e..bbda545d2 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/infrastructure/mapper/UserDOMapper.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/mapper/UserDOMapper.java @@ -1,8 +1,8 @@ -package com.tencent.supersonic.auth.authentication.infrastructure.mapper; +package com.tencent.supersonic.auth.authentication.persistence.mapper; -import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO; -import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDOExample; +import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO; +import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample; import java.util.List; import org.apache.ibatis.annotations.Mapper; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/infrastructure/repository/UserRepositoryImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/Impl/UserRepositoryImpl.java similarity index 68% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/infrastructure/repository/UserRepositoryImpl.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/Impl/UserRepositoryImpl.java index 4c9d8a958..97360894b 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/infrastructure/repository/UserRepositoryImpl.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/Impl/UserRepositoryImpl.java @@ -1,10 +1,10 @@ -package com.tencent.supersonic.auth.authentication.infrastructure.repository; +package com.tencent.supersonic.auth.authentication.persistence.repository.Impl; -import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO; -import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDOExample; -import com.tencent.supersonic.auth.authentication.domain.repository.UserRepository; -import com.tencent.supersonic.auth.authentication.infrastructure.mapper.UserDOMapper; +import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO; +import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDOExample; +import com.tencent.supersonic.auth.authentication.persistence.repository.UserRepository; +import com.tencent.supersonic.auth.authentication.persistence.mapper.UserDOMapper; import java.util.List; import java.util.Optional; import org.springframework.stereotype.Component; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/repository/UserRepository.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java similarity index 50% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/repository/UserRepository.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java index fbc63c595..14037edfb 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/repository/UserRepository.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.auth.authentication.domain.repository; +package com.tencent.supersonic.auth.authentication.persistence.repository; -import com.tencent.supersonic.auth.authentication.domain.dataobject.UserDO; +import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserDO; import java.util.List; public interface UserRepository { diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java index 508297110..14cc106d8 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/rest/UserController.java @@ -1,19 +1,17 @@ package com.tencent.supersonic.auth.authentication.rest; +import com.tencent.supersonic.auth.api.authentication.pojo.Organization; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.request.UserReq; import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import java.util.List; +import java.util.Set; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; @RestController @RequestMapping("/api/auth/user") @@ -32,7 +30,6 @@ public class UserController { return UserHolder.findUser(httpServletRequest, httpServletResponse); } - @GetMapping("/getUserNames") public List getUserNames() { return userService.getUserNames(); @@ -43,6 +40,21 @@ public class UserController { return userService.getUserList(); } + @GetMapping("/getOrganizationTree") + public List getOrganizationTree() { + return userService.getOrganizationTree(); + } + + @GetMapping("/getUserAllOrgId/{userName}") + public Set getUserAllOrgId(@PathVariable("userName") String userName) { + return userService.getUserAllOrgId(userName); + } + + @GetMapping("/getUserByOrg/{org}") + public List getUserByOrg(@PathVariable("org") String org) { + return userService.getUserByOrg(org); + } + @PostMapping("/register") public void register(@RequestBody UserReq userCmd) { userService.register(userCmd); diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java new file mode 100644 index 000000000..15b102991 --- /dev/null +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/service/UserServiceImpl.java @@ -0,0 +1,52 @@ +package com.tencent.supersonic.auth.authentication.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.Organization; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.request.UserReq; +import com.tencent.supersonic.auth.api.authentication.service.UserService; +import com.tencent.supersonic.auth.authentication.utils.ComponentFactory; +import java.util.List; +import java.util.Set; + +import org.springframework.stereotype.Service; + +@Service +public class UserServiceImpl implements UserService { + + + @Override + public List getUserNames() { + return ComponentFactory.getUserAdaptor().getUserNames(); + } + + @Override + public List getUserList() { + return ComponentFactory.getUserAdaptor().getUserList(); + } + + @Override + public Set getUserAllOrgId(String userName) { + return ComponentFactory.getUserAdaptor().getUserAllOrgId(userName); + } + + @Override + public List getUserByOrg(String key) { + return ComponentFactory.getUserAdaptor().getUserByOrg(key); + } + + @Override + public List getOrganizationTree() { + return ComponentFactory.getUserAdaptor().getOrganizationTree(); + } + + @Override + public void register(UserReq userReq) { + ComponentFactory.getUserAdaptor().register(userReq); + } + + @Override + public String login(UserReq userReq) { + return ComponentFactory.getUserAdaptor().login(userReq); + } + +} diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/FakeUserStrategy.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/FakeUserStrategy.java similarity index 90% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/FakeUserStrategy.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/FakeUserStrategy.java index 9054d8968..a494c158d 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/FakeUserStrategy.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/FakeUserStrategy.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.auth.authentication.domain.strategy; +package com.tencent.supersonic.auth.authentication.strategy; import com.tencent.supersonic.auth.api.authentication.pojo.User; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/HttpHeaderUserStrategy.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/HttpHeaderUserStrategy.java similarity index 84% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/HttpHeaderUserStrategy.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/HttpHeaderUserStrategy.java index e1af56208..8da6fe324 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/HttpHeaderUserStrategy.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/HttpHeaderUserStrategy.java @@ -1,8 +1,8 @@ -package com.tencent.supersonic.auth.authentication.domain.strategy; +package com.tencent.supersonic.auth.authentication.strategy; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserStrategy; -import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils; +import com.tencent.supersonic.auth.authentication.utils.UserTokenUtils; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.springframework.stereotype.Service; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/UserStrategyFactory.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/UserStrategyFactory.java similarity index 93% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/UserStrategyFactory.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/UserStrategyFactory.java index cacc32fe0..7732a51b2 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/strategy/UserStrategyFactory.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/strategy/UserStrategyFactory.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.auth.authentication.domain.strategy; +package com.tencent.supersonic.auth.authentication.strategy; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/ComponentFactory.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/ComponentFactory.java new file mode 100644 index 000000000..e6cee4640 --- /dev/null +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/ComponentFactory.java @@ -0,0 +1,23 @@ +package com.tencent.supersonic.auth.authentication.utils; + +import com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor; +import org.springframework.core.io.support.SpringFactoriesLoader; +import java.util.Objects; + +public class ComponentFactory { + + private static UserAdaptor userAdaptor; + + public static UserAdaptor getUserAdaptor() { + if (Objects.isNull(userAdaptor)) { + userAdaptor = init(UserAdaptor.class); + } + return userAdaptor; + } + + private static T init(Class factoryType) { + return SpringFactoriesLoader.loadFactories(factoryType, + Thread.currentThread().getContextClassLoader()).get(0); + } + +} diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/utils/UserTokenUtils.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/UserTokenUtils.java similarity index 98% rename from auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/utils/UserTokenUtils.java rename to auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/UserTokenUtils.java index 94f76b91c..4c55a84b1 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/utils/UserTokenUtils.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/UserTokenUtils.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.auth.authentication.domain.utils; +package com.tencent.supersonic.auth.authentication.utils; import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_ALGORITHM; import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_CREATE_TIME; diff --git a/auth/authentication/src/main/resources/mapper/UserDOMapper.xml b/auth/authentication/src/main/resources/mapper/UserDOMapper.xml index ebef375fd..15eb2b49c 100644 --- a/auth/authentication/src/main/resources/mapper/UserDOMapper.xml +++ b/auth/authentication/src/main/resources/mapper/UserDOMapper.xml @@ -1,7 +1,7 @@ - - + + @@ -40,7 +40,7 @@ id, name, password, display_name, email - select distinct @@ -67,13 +67,13 @@ delete from s2_user where id = #{id,jdbcType=BIGINT} - + insert into s2_user (id, name, password, display_name, email) values (#{id,jdbcType=BIGINT}, #{name,jdbcType=VARCHAR}, #{password,jdbcType=VARCHAR}, #{displayName,jdbcType=VARCHAR}, #{email,jdbcType=VARCHAR}) - + insert into s2_user @@ -110,13 +110,13 @@ - select count(*) from s2_user - + update s2_user @@ -134,7 +134,7 @@ where id = #{id,jdbcType=BIGINT} - + update s2_user set name = #{name,jdbcType=VARCHAR}, password = #{password,jdbcType=VARCHAR}, diff --git a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java index d797b2f1e..878c7cfd4 100644 --- a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java +++ b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/application/AuthServiceImpl.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.auth.authorization.application; import com.google.common.base.Strings; import com.google.gson.Gson; +import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp; import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter; @@ -18,10 +19,7 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; @Service @@ -30,8 +28,12 @@ public class AuthServiceImpl implements AuthService { private JdbcTemplate jdbcTemplate; - public AuthServiceImpl(JdbcTemplate jdbcTemplate) { + private UserService userService; + + public AuthServiceImpl(JdbcTemplate jdbcTemplate, + UserService userService) { this.jdbcTemplate = jdbcTemplate; + this.userService = userService; } private List load() { @@ -75,6 +77,10 @@ public class AuthServiceImpl implements AuthService { @Override public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, HttpServletRequest request) { + Set userOrgIds = userService.getUserAllOrgId(req.getUser()); + if (!CollectionUtils.isEmpty(userOrgIds)) { + req.setDepartmentIds(new ArrayList<>(userOrgIds)); + } List groups = getAuthGroups(req); AuthorizedResourceResp resource = new AuthorizedResourceResp(); Map> authGroupsByDomainId = groups.stream() @@ -119,7 +125,6 @@ public class AuthServiceImpl implements AuthService { } } } - return resource; } @@ -133,9 +138,9 @@ public class AuthServiceImpl implements AuthService { .contains(req.getUser())) { return true; } - for (String deparmentId : req.getDepartmentIds()) { + for (String departmentId : req.getDepartmentIds()) { if (!CollectionUtils.isEmpty(group.getAuthorizedDepartmentIds()) - && group.getAuthorizedDepartmentIds().contains(deparmentId)) { + && group.getAuthorizedDepartmentIds().contains(departmentId)) { return true; } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java index f8ddf147a..def007dca 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java @@ -16,4 +16,6 @@ public interface SemanticQuery { QueryResult execute(User user) throws SqlParseException; SemanticParseInfo getParseInfo(); + + void setParseInfo(SemanticParseInfo parseInfo); } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DomainSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DomainSchema.java index d1aeb1855..83a1368d3 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DomainSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/DomainSchema.java @@ -13,11 +13,15 @@ public class DomainSchema { private Set metrics = new HashSet<>(); private Set dimensions = new HashSet<>(); private Set dimensionValues = new HashSet<>(); - private Set entities = new HashSet<>(); + private SchemaElement entity = new SchemaElement(); public SchemaElement getElement(SchemaElementType elementType, long elementID) { Optional element = Optional.empty(); + switch (elementType) { + case ENTITY: + element = Optional.ofNullable(entity); + break; case DOMAIN: element = Optional.of(domain); break; @@ -27,9 +31,6 @@ public class DomainSchema { case DIMENSION: element = dimensions.stream().filter(e -> e.getId() == elementID).findFirst(); break; - case ENTITY: - element = entities.stream().filter(e -> e.getId() == elementID).findFirst(); - break; case VALUE: element = dimensionValues.stream().filter(e -> e.getId() == elementID).findFirst(); default: diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java index ba92177af..244468b9b 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryContext.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.api.pojo; import com.tencent.supersonic.chat.api.component.SemanticQuery; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import lombok.Data; import java.util.ArrayList; @@ -10,11 +10,11 @@ import java.util.List; @Data public class QueryContext { - private QueryRequest request; + private QueryReq request; private List candidateQueries = new ArrayList<>(); private SchemaMapInfo mapInfo = new SchemaMapInfo(); - public QueryContext(QueryRequest request) { + public QueryContext(QueryReq request) { this.request = request; } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java index 0907c25c9..6390545fd 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElement.java @@ -5,11 +5,13 @@ import com.google.common.base.Objects; import java.io.Serializable; import java.util.List; -import lombok.Builder; -import lombok.Data; +import lombok.*; @Data +@Getter @Builder +@NoArgsConstructor +//@AllArgsConstructor public class SchemaElement implements Serializable { private Long domain; @@ -20,8 +22,8 @@ public class SchemaElement implements Serializable { private SchemaElementType type; private List alias; - public SchemaElement() { - } +// public SchemaElement() { +// } public SchemaElement(Long domain, Long id, String name, String bizName, Long useCnt, SchemaElementType type, List alias) { diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java index 605023199..e958e24e4 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java @@ -18,4 +18,10 @@ public class SchemaElementMatch { String detectWord; String word; Long frequency; + MatchMode mode = MatchMode.CURRENT; + + public enum MatchMode { + CURRENT, + INHERIT + } } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index b0059cfc7..797800d5f 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -1,36 +1,32 @@ package com.tencent.supersonic.chat.api.pojo; +import java.util.*; + import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import lombok.Data; @Data public class SemanticParseInfo { - String queryMode; - SchemaElement domain; - Set metrics = new LinkedHashSet(); - Set dimensions = new LinkedHashSet(); - Long entity = 0L; - AggregateTypeEnum aggType = AggregateTypeEnum.NONE; - Set dimensionFilters = new LinkedHashSet(); - Set metricFilters = new LinkedHashSet(); + private String queryMode; + private SchemaElement domain; + private Set metrics = new TreeSet<>(new SchemaNameLengthComparator()); + private Set dimensions = new LinkedHashSet(); + private SchemaElement entity; + private AggregateTypeEnum aggType = AggregateTypeEnum.NONE; + private Set dimensionFilters = new LinkedHashSet(); + private Set metricFilters = new LinkedHashSet(); private Set orders = new LinkedHashSet(); private DateConf dateInfo; private Long limit; private Boolean nativeQuery = false; - private Double bonus = 0d; + private double score; private List elementMatches = new ArrayList<>(); - private Map properties; + private Map properties = new HashMap<>(); public Long getDomainId() { return domain != null ? domain.getId() : 0L; @@ -40,8 +36,9 @@ public class SemanticParseInfo { return domain != null ? domain.getName() : "null"; } - public Set getMetrics() { - this.metrics = this.metrics.stream().sorted((o1, o2) -> { + private static class SchemaNameLengthComparator implements Comparator { + @Override + public int compare(SchemaElement o1, SchemaElement o2) { int len1 = o1.getName().length(); int len2 = o2.getName().length(); if (len1 != len2) { @@ -49,7 +46,7 @@ public class SemanticParseInfo { } else { return o1.getName().compareTo(o2.getName()); } - }).collect(Collectors.toCollection(LinkedHashSet::new)); - return this.metrics; + } } + } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java index 7e199f00d..6c9e86d44 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java @@ -48,7 +48,7 @@ public class SemanticSchema implements Serializable { public List getEntities() { List entities = new ArrayList<>(); - domainSchemaList.stream().forEach(d -> entities.addAll(d.getEntities())); + domainSchemaList.stream().forEach(d -> entities.add(d.getEntity())); return entities; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggConfig.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatAggConfigReq.java similarity index 59% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggConfig.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatAggConfigReq.java index 32f3ba11c..56d83c326 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggConfig.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatAggConfigReq.java @@ -1,11 +1,11 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import lombok.Data; import java.util.List; @Data -public class ChatAggConfig { +public class ChatAggConfigReq { /** * invisible dimensions/metrics @@ -15,10 +15,10 @@ public class ChatAggConfig { /** * information about dictionary about the domain */ - private List knowledgeInfos; + private List knowledgeInfos; private KnowledgeAdvancedConfig globalKnowledgeConfig; - private ChatDefaultConfig chatDefaultConfig; + private ChatDefaultConfigReq chatDefaultConfig; } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigBaseReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java similarity index 66% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigBaseReq.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java index c172bf8b4..b1b64d647 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigBaseReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigBaseReq.java @@ -1,6 +1,5 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; -import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import lombok.Data; @@ -20,18 +19,18 @@ public class ChatConfigBaseReq { /** * the chatDetailConfig about the domain */ - private ChatDetailConfig chatDetailConfig; + private ChatDetailConfigReq chatDetailConfig; /** * the chatAggConfig about the domain */ - private ChatAggConfig chatAggConfig; + private ChatAggConfigReq chatAggConfig; /** * the recommended questions about the domain */ - private List recommendedQuestions; + private List recommendedQuestions; /** * available status diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigEditReqReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigEditReqReq.java similarity index 67% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigEditReqReq.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigEditReqReq.java index 441e8f59d..638b03a64 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigEditReqReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigEditReqReq.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigFilter.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigFilter.java similarity index 83% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigFilter.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigFilter.java index 721bd4ce0..61cbdfe2c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigFilter.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatConfigFilter.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultConfig.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java similarity index 59% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultConfig.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java index c69eeade3..33ca4889c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultConfig.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDefaultConfigReq.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import com.tencent.supersonic.common.pojo.Constants; @@ -8,7 +8,7 @@ import java.util.ArrayList; import java.util.List; @Data -public class ChatDefaultConfig { +public class ChatDefaultConfigReq { private List dimensionIds = new ArrayList<>(); private List metricIds = new ArrayList<>(); @@ -24,4 +24,15 @@ public class ChatDefaultConfig { */ private String period = Constants.DAY; + private TimeMode timeMode = TimeMode.LAST; + + public enum TimeMode { + /** + * date mode + * LAST - a certain time + * RECENT - a period time + */ + LAST, RECENT + } + } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailConfig.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDetailConfigReq.java similarity index 51% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailConfig.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDetailConfigReq.java index 42b6f7d98..6d3468c7f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailConfig.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ChatDetailConfigReq.java @@ -1,11 +1,11 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import lombok.Data; import java.util.List; @Data -public class ChatDetailConfig { +public class ChatDetailConfigReq { /** * invisible dimensions/metrics @@ -15,15 +15,10 @@ public class ChatDetailConfig { /** * information about dictionary about the domain */ - private List knowledgeInfos; + private List knowledgeInfos; private KnowledgeAdvancedConfig globalKnowledgeConfig; - private ChatDefaultConfig chatDefaultConfig; - - /** - * the entity info about the domain - */ - private Entity entity; + private ChatDefaultConfigReq chatDefaultConfig; } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/Entity.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/Entity.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java index 26a1d415c..373b3cc11 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/Entity.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/Entity.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import java.util.List; import lombok.AllArgsConstructor; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ExecuteQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ExecuteQueryReq.java new file mode 100644 index 000000000..b1f4d6013 --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ExecuteQueryReq.java @@ -0,0 +1,15 @@ +package com.tencent.supersonic.chat.api.pojo.request; + + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import lombok.Data; + +@Data +public class ExecuteQueryReq { + private User user; + private Integer chatId; + private String queryText; + private SemanticParseInfo parseInfo; + private boolean saveAnswer = true; +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibility.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemVisibility.java similarity index 86% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibility.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemVisibility.java index e554e92ed..ee3a1eb13 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibility.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/ItemVisibility.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import java.util.ArrayList; import java.util.List; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeAdvancedConfig.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeAdvancedConfig.java similarity index 85% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeAdvancedConfig.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeAdvancedConfig.java index 8ba0dabf1..3c5d8d501 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeAdvancedConfig.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeAdvancedConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import lombok.Data; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java similarity index 88% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeInfo.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java index 7a0686ed0..0ec5707db 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/KnowledgeInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/KnowledgeInfoReq.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.request; import com.tencent.supersonic.common.pojo.enums.TypeEnums; @@ -11,7 +11,7 @@ import lombok.Data; */ @Data -public class KnowledgeInfo { +public class KnowledgeInfoReq { /** * metricId、DimensionId、domainId diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java index f3ba04966..14939bc63 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/PluginQueryReq.java @@ -7,10 +7,9 @@ import lombok.Data; public class PluginQueryReq { - private String showElementId; + private String name; - //DASHBOARD WIDGET - private String showType; + private String parseMode; private String type; @@ -18,5 +17,5 @@ public class PluginQueryReq { private String pattern; - + private String createdBy; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataRequest.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java similarity index 95% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataRequest.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java index 73ef07130..1dbaa4137 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataRequest.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java @@ -10,7 +10,7 @@ import com.tencent.supersonic.common.pojo.Order; import lombok.Data; @Data -public class QueryDataRequest { +public class QueryDataReq { String queryMode; SchemaElement domain; Set metrics = new HashSet<>(); diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryRequest.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java similarity index 92% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryRequest.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java index d51ccee2f..0e8c686ad 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryRequest.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryReq.java @@ -4,8 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import lombok.Data; @Data -public class QueryRequest { - +public class QueryReq { private String queryText; private Integer chatId; private Long domainId = 0L; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestion.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestionReq.java similarity index 50% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestion.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestionReq.java index 0aa8a6bdf..cbfebb407 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestion.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/RecommendedQuestionReq.java @@ -1,11 +1,15 @@ package com.tencent.supersonic.chat.api.pojo.request; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.ToString; @Data @ToString -public class RecommendedQuestion { +@AllArgsConstructor +@NoArgsConstructor +public class RecommendedQuestionReq { private String question; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatAggRichConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatAggRichConfigResp.java new file mode 100644 index 000000000..b69a16f3c --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatAggRichConfigResp.java @@ -0,0 +1,26 @@ +package com.tencent.supersonic.chat.api.pojo.response; + +import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig; +import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; +import lombok.Data; + +import java.util.List; + +@Data +public class ChatAggRichConfigResp { + + /** + * invisible dimensions/metrics + */ + private ItemVisibilityInfo visibility; + + /** + * information about dictionary about the domain + */ + private List knowledgeInfos; + + private KnowledgeAdvancedConfig globalKnowledgeConfig; + + private ChatDefaultRichConfigResp chatDefaultConfig; + +} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigResp.java similarity index 56% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigResp.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigResp.java index 6c7ecba88..c32efdd1b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigResp.java @@ -1,6 +1,8 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; +import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import java.util.Date; @@ -15,11 +17,11 @@ public class ChatConfigResp { private Long domainId; - private ChatDetailConfig chatDetailConfig; + private ChatDetailConfigReq chatDetailConfig; - private ChatAggConfig chatAggConfig; + private ChatAggConfigReq chatAggConfig; - private List recommendedQuestions; + private List recommendedQuestions; /** * available status diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigRich.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigRichResp.java similarity index 64% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigRich.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigRichResp.java index f2913012b..e6f8e75a3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfigRich.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatConfigRichResp.java @@ -1,6 +1,6 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; +import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import java.util.Date; import java.util.List; @@ -8,7 +8,7 @@ import java.util.List; import lombok.Data; @Data -public class ChatConfigRich { +public class ChatConfigRichResp { private Long id; @@ -17,11 +17,11 @@ public class ChatConfigRich { private String domainName; private String bizName; - private ChatAggRichConfig chatAggRichConfig; + private ChatAggRichConfigResp chatAggRichConfig; - private ChatDetailRichConfig chatDetailRichConfig; + private ChatDetailRichConfigResp chatDetailRichConfig; - private List recommendedQuestions; + private List recommendedQuestions; /** * available status diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultRichConfig.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java similarity index 66% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultRichConfig.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java index 5b63e9ee7..6ef966681 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDefaultRichConfig.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java @@ -1,14 +1,15 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.response; import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; import com.tencent.supersonic.common.pojo.Constants; import lombok.Data; import java.util.List; @Data -public class ChatDefaultRichConfig { +public class ChatDefaultRichConfigResp { private List dimensions; private List metrics; @@ -25,4 +26,6 @@ public class ChatDefaultRichConfig { */ private String period = Constants.DAY; + private ChatDefaultConfigReq.TimeMode timeMode; + } \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDetailRichConfigResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDetailRichConfigResp.java new file mode 100644 index 000000000..a3dbd60d8 --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDetailRichConfigResp.java @@ -0,0 +1,27 @@ +package com.tencent.supersonic.chat.api.pojo.response; + +import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig; +import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; +import lombok.Data; + +import java.util.List; + +@Data +public class ChatDetailRichConfigResp { + + /** + * invisible dimensions/metrics + */ + private ItemVisibilityInfo visibility; + + /** + * information about dictionary about the domain + */ + private List knowledgeInfos; + + private KnowledgeAdvancedConfig globalKnowledgeConfig; + + private ChatDefaultRichConfigResp chatDefaultConfig; + + +} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/EntityRichInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java similarity index 71% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/EntityRichInfo.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java index 023b6e770..a8610fed1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/EntityRichInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.response; import com.tencent.supersonic.chat.api.pojo.SchemaElement; @@ -6,7 +6,7 @@ import java.util.List; import lombok.Data; @Data -public class EntityRichInfo { +public class EntityRichInfoResp { /** * entity alias */ diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibilityInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ItemVisibilityInfo.java similarity index 81% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibilityInfo.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ItemVisibilityInfo.java index 7a66b15df..3bb4e0798 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ItemVisibilityInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ItemVisibilityInfo.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.config; +package com.tencent.supersonic.chat.api.pojo.response; import java.util.List; import lombok.Data; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/MetricInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/MetricInfo.java index ef9264145..1c1c4c580 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/MetricInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/MetricInfo.java @@ -7,6 +7,7 @@ import lombok.Data; public class MetricInfo { private String name; + private String dimension; private String value; private String date; private Map statistics; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java new file mode 100644 index 000000000..6fd7f33e2 --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java @@ -0,0 +1,25 @@ +package com.tencent.supersonic.chat.api.pojo.response; + +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import lombok.*; + +import java.util.List; + +@Data +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ParseResp { + private Integer chatId; + private String queryText; + private ParseState state; + private List selectedParses; + private List candidateParses; + + public enum ParseState { + COMPLETED, + PENDING, + FAILED + } +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResponse.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java similarity index 91% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResponse.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java index 2ab39fb93..1c50eeb05 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResponse.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResp.java @@ -4,7 +4,7 @@ import java.util.Date; import lombok.Data; @Data -public class QueryResponse { +public class QueryResp { private Long questionId; private Date createTime; diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendQuestion.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendQuestionResp.java similarity index 69% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendQuestion.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendQuestionResp.java index 67cec7302..62d106faf 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendQuestion.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendQuestionResp.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.api.pojo.response; -import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; +import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; import lombok.AllArgsConstructor; import lombok.Data; @@ -8,7 +8,7 @@ import java.util.List; @Data @AllArgsConstructor -public class RecommendQuestion { +public class RecommendQuestionResp { private Long domainId; - private List recommendedQuestions; + private List recommendedQuestions; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResponse.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java similarity index 88% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResponse.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java index 89fd38a34..8490ab549 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResponse.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java @@ -6,7 +6,7 @@ import lombok.Data; import java.util.List; @Data -public class RecommendResponse { +public class RecommendResp { private List dimensions; private List metrics; } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResponse.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResp.java similarity index 73% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResponse.java rename to chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResp.java index 5f6b406ee..7cf443bd7 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResponse.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResp.java @@ -6,11 +6,11 @@ import lombok.Getter; import lombok.Setter; @Data -public class SearchResponse { +public class SearchResp { private List searchResults; - public SearchResponse(List searchResults) { + public SearchResp(List searchResults) { this.searchResults = searchResults; } } diff --git a/chat/core/pom.xml b/chat/core/pom.xml index f30672455..860369e16 100644 --- a/chat/core/pom.xml +++ b/chat/core/pom.xml @@ -1,7 +1,7 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> chat com.tencent.supersonic @@ -40,6 +40,11 @@ compile + + com.github.plexpt + chatgpt + 4.1.2 + org.junit.jupiter diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/AggregatorConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/config/AggregatorConfig.java new file mode 100644 index 000000000..934b0ae21 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/config/AggregatorConfig.java @@ -0,0 +1,13 @@ +package com.tencent.supersonic.chat.config; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +@Configuration +@Data +public class AggregatorConfig { + @Value("${metric.aggregator.ratio.enable:true}") + private Boolean enableRatio; +} + + diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggRichConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggRichConfig.java deleted file mode 100644 index fbe4b6f96..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatAggRichConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.tencent.supersonic.chat.config; - -import lombok.Data; - -import java.util.List; - -@Data -public class ChatAggRichConfig { - - /** - * invisible dimensions/metrics - */ - private ItemVisibilityInfo visibility; - - /** - * information about dictionary about the domain - */ - private List knowledgeInfos; - - private KnowledgeAdvancedConfig globalKnowledgeConfig; - - private ChatDefaultRichConfig chatDefaultConfig; - -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfig.java index c41c08960..ecffde18b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfig.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatConfig.java @@ -1,6 +1,8 @@ package com.tencent.supersonic.chat.config; -import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; +import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; @@ -22,14 +24,14 @@ public class ChatConfig { /** * the chatDetailConfig about the domain */ - private ChatDetailConfig chatDetailConfig; + private ChatDetailConfigReq chatDetailConfig; /** * the chatAggConfig about the domain */ - private ChatAggConfig chatAggConfig; + private ChatAggConfigReq chatAggConfig; - private List recommendedQuestions; + private List recommendedQuestions; /** * available status diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailRichConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailRichConfig.java deleted file mode 100644 index e7f372baf..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/ChatDetailRichConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.supersonic.chat.config; - -import lombok.Data; - -import java.util.List; - -@Data -public class ChatDetailRichConfig { - - /** - * invisible dimensions/metrics - */ - private ItemVisibilityInfo visibility; - - /** - * the entity info about the domain - */ - private EntityRichInfo entity; - - /** - * information about dictionary about the domain - */ - private List knowledgeInfos; - - private KnowledgeAdvancedConfig globalKnowledgeConfig; - - private ChatDefaultRichConfig chatDefaultConfig; - - -} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java similarity index 87% rename from chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallConfig.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java index 1fd4a4083..ec23773bd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallConfig.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/config/FunctionCallInfoConfig.java @@ -6,7 +6,7 @@ import org.springframework.context.annotation.Configuration; @Configuration @Data -public class FunctionCallConfig { +public class FunctionCallInfoConfig { @Value("${functionCall.url:}") private String url; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java new file mode 100644 index 000000000..6ae9ceeba --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/EntityMapper.java @@ -0,0 +1,69 @@ +package com.tencent.supersonic.chat.mapper; + + +import com.tencent.supersonic.chat.api.component.SchemaMapper; +import com.tencent.supersonic.chat.api.pojo.*; +import com.tencent.supersonic.chat.service.SemanticService; +import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; +import java.util.List; +import java.util.stream.Collectors; + + +@Slf4j +public class EntityMapper implements SchemaMapper { + + @Override + public void map(QueryContext queryContext) { + SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); + for (Long domainId : schemaMapInfo.getMatchedDomains()) { + List schemaElementMatchList = schemaMapInfo.getMatchedElements(domainId); + if (CollectionUtils.isEmpty(schemaElementMatchList)) { + continue; + } + SchemaElement entity = getEntity(domainId); + if (entity == null || entity.getId() == null) { + continue; + } + List valueSchemaElements = schemaElementMatchList.stream().filter(schemaElementMatch -> + SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); + for (SchemaElementMatch schemaElementMatch : valueSchemaElements) { + if (!entity.getId().equals(schemaElementMatch.getElement().getId())){ + continue; + } + if (!checkExistSameEntitySchemaElements(schemaElementMatch, schemaElementMatchList)) { + SchemaElementMatch entitySchemaElementMath = new SchemaElementMatch(); + BeanUtils.copyProperties(schemaElementMatch, entitySchemaElementMath); + entitySchemaElementMath.setElement(entity); + schemaElementMatchList.add(entitySchemaElementMath); + } + schemaElementMatch.getElement().setType(SchemaElementType.ID); + } + } + } + + private boolean checkExistSameEntitySchemaElements(SchemaElementMatch valueSchemaElementMatch, + List schemaElementMatchList) { + List entitySchemaElements = schemaElementMatchList.stream().filter(schemaElementMatch -> + SchemaElementType.ENTITY.equals(schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); + for (SchemaElementMatch schemaElementMatch : entitySchemaElements) { + if (schemaElementMatch.getElement().getId().equals(valueSchemaElementMatch.getElement().getId())) { + return true; + } + } + return false; + } + + private SchemaElement getEntity(Long domainId) { + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + DomainSchema domainSchema = semanticService.getDomainSchema(domainId); + if (domainSchema != null && domainSchema.getEntity() != null) { + return domainSchema.getEntity(); + } + return null; + } +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java index cf3a6129e..470f43236 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/FuzzyNameMapper.java @@ -2,13 +2,9 @@ package com.tencent.supersonic.chat.mapper; import com.hankcs.hanlp.seg.common.Term; import com.tencent.supersonic.chat.api.component.SchemaMapper; +import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; import com.tencent.supersonic.knowledge.service.SchemaService; -import com.tencent.supersonic.chat.api.pojo.SemanticSchema; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.knowledge.utils.HanlpHelper; import java.util.ArrayList; @@ -44,7 +40,7 @@ public class FuzzyNameMapper implements SchemaMapper { } private void detectAndAddToSchema(QueryContext queryContext, List terms, List domains, - SchemaElementType schemaElementType) { + SchemaElementType schemaElementType) { try { Map> domainResultSet = getResultSet(queryContext, terms, domains); @@ -57,7 +53,7 @@ public class FuzzyNameMapper implements SchemaMapper { } private Map> getResultSet(QueryContext queryContext, List terms, - List domains) { + List domains) { String queryText = queryContext.getRequest().getQueryText(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java index fb9c94d36..c58e177c4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/HanlpDictMapper.java @@ -74,6 +74,10 @@ public class HanlpDictMapper implements SchemaMapper { Long frequency = wordNatureToFrequency.get(mapResult.getName() + nature); SchemaElement element = domainSchema.getElement(elementType, elementID); + if(Objects.isNull(element)){ + log.info("element is null, elementType:{},elementID:{}", elementType, elementID); + continue; + } if (element.getType().equals(SchemaElementType.VALUE)) { element.setName(mapResult.getName()); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/QueryFilterMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/QueryFilterMapper.java index 74f785f30..9134d1541 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/QueryFilterMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/mapper/QueryFilterMapper.java @@ -1,27 +1,41 @@ package com.tencent.supersonic.chat.mapper; +import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.component.SchemaMapper; import com.tencent.supersonic.chat.api.pojo.*; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; +import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Slf4j public class QueryFilterMapper implements SchemaMapper { + private Long FREQUENCY = 9999999L; + private double SIMILARITY = 1.0; + @Override public void map(QueryContext queryContext) { - QueryRequest queryReq = queryContext.getRequest(); + QueryReq queryReq = queryContext.getRequest(); Long domainId = queryReq.getDomainId(); if (domainId == null || domainId <= 0) { return; } SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); clearOtherSchemaElementMatch(domainId, schemaMapInfo); + List schemaElementMatches = schemaMapInfo.getMatchedElements(domainId); + if (schemaElementMatches == null) { + schemaElementMatches = Lists.newArrayList(); + schemaMapInfo.setMatchedElements(domainId, schemaElementMatches); + } + addValueSchemaElementMatch(schemaElementMatches, queryReq.getQueryFilters()); } - private void clearOtherSchemaElementMatch(Long domainId, SchemaMapInfo schemaMapInfo) { + private void clearOtherSchemaElementMatch(Long domainId, SchemaMapInfo schemaMapInfo) { for (Map.Entry> entry : schemaMapInfo.getDomainElementMatches().entrySet()) { if (!entry.getKey().equals(domainId)) { entry.getValue().clear(); @@ -29,4 +43,44 @@ public class QueryFilterMapper implements SchemaMapper { } } + private List addValueSchemaElementMatch(List candidateElementMatches, + QueryFilters queryFilter) { + if (queryFilter == null || CollectionUtils.isEmpty(queryFilter.getFilters())) { + return candidateElementMatches; + } + for (QueryFilter filter : queryFilter.getFilters()) { + if (checkExistSameValueSchemaElementMatch(filter, candidateElementMatches)) { + continue; + } + SchemaElement element = SchemaElement.builder() + .id(filter.getElementID()) + .name(String.valueOf(filter.getValue())) + .type(SchemaElementType.VALUE) + .bizName(filter.getBizName()) + .build(); + SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() + .element(element) + .frequency(FREQUENCY) + .word(String.valueOf(filter.getValue())) + .similarity(SIMILARITY) + .detectWord(filter.getName()) + .build(); + candidateElementMatches.add(schemaElementMatch); + } + return candidateElementMatches; + } + + private boolean checkExistSameValueSchemaElementMatch(QueryFilter queryFilter, + List schemaElementMatches) { + List valueSchemaElements = schemaElementMatches.stream().filter(schemaElementMatch -> + SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) + .collect(Collectors.toList()); + for (SchemaElementMatch schemaElementMatch : valueSchemaElements) { + if (schemaElementMatch.getElement().getId().equals(queryFilter.getElementID()) + && schemaElementMatch.getWord().equals(String.valueOf(queryFilter.getValue()))) { + return true; + } + } + return false; + } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java index 495a73597..b5fb123dd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/SatisfactionChecker.java @@ -3,13 +3,16 @@ package com.tencent.supersonic.chat.parser; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.*; +import com.tencent.supersonic.chat.plugin.PluginParseResult; +import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; +import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.compress.utils.Lists; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; - import java.util.HashSet; import java.util.List; import java.util.Map; @@ -25,31 +28,36 @@ import java.util.stream.Collectors; public class SatisfactionChecker { private static final double LONG_TEXT_THRESHOLD = 0.8; - private static final double SHORT_TEXT_THRESHOLD = 0.6; + private static final double SHORT_TEXT_THRESHOLD = 0.5; private static final int QUERY_TEXT_LENGTH_THRESHOLD = 10; - - public static final double BONUS_THRESHOLD = 100; + public static final double EMBEDDING_THRESHOLD = 0.2; // check all the parse info in candidate public static boolean check(QueryContext queryCtx) { for (SemanticQuery query : queryCtx.getCandidateQueries()) { - SemanticParseInfo semanticParseInfo = query.getParseInfo(); - Long domainId = semanticParseInfo.getDomainId(); - List schemaElementMatches = queryCtx.getMapInfo() - .getMatchedElements(domainId); - if (check(queryCtx.getRequest().getQueryText(), semanticParseInfo, schemaElementMatches)) { - return true; + if (query instanceof RuleSemanticQuery) { + if (checkRuleThreshHold(queryCtx.getRequest().getQueryText(), query.getParseInfo())) { + return true; + } + } else if (query instanceof PluginSemanticQuery) { + if (checkEmbeddingThreshold(query.getParseInfo())) { + log.info("query mode :{} satisfy check", query.getQueryMode()); + return true; + } } } return false; } + private static boolean checkEmbeddingThreshold(SemanticParseInfo semanticParseInfo) { + Object object = semanticParseInfo.getProperties().get(Constants.CONTEXT); + PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(object), PluginParseResult.class); + return EMBEDDING_THRESHOLD > pluginParseResult.getDistance(); + } + //check single parse info - private static boolean check(String text, SemanticParseInfo semanticParseInfo, - List schemaElementMatches) { - if (semanticParseInfo.getBonus() != null && semanticParseInfo.getBonus() >= BONUS_THRESHOLD) { - return true; - } + private static boolean checkRuleThreshHold(String text, SemanticParseInfo semanticParseInfo) { + List schemaElementMatches = semanticParseInfo.getElementMatches(); if (CollectionUtils.isEmpty(schemaElementMatches)) { return false; } @@ -71,6 +79,11 @@ public class SatisfactionChecker { detectWords.add(schemaElementMatch.getDetectWord()); } } + for (SchemaElementMatch schemaElementMatch : schemaElementMatches) { + if (SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) { + detectWords.add(schemaElementMatch.getDetectWord()); + } + } for (SchemaElement schemaItem : semanticParseInfo.getMetrics()) { detectWords.add( detectWordMap.getOrDefault(Optional.ofNullable(schemaItem.getId()).orElse(0L), "")); @@ -87,7 +100,7 @@ public class SatisfactionChecker { if (StringUtils.isNotBlank(dateText) && !dateText.equalsIgnoreCase(Constants.NULL)) { detectWords.add(dateText); } - detectWords.removeIf(word -> !text.contains(word)); + detectWords.removeIf(word -> !text.contains(word) && !text.contains(StringUtils.reverse(word))); //compare the length between detect words and query text return checkThreshold(text, detectWords, semanticParseInfo); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingBasedParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingBasedParser.java index 6bbdf299f..e054bd9df 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingBasedParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingBasedParser.java @@ -5,19 +5,21 @@ import com.google.common.collect.Sets; import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.config.ChatConfigRich; -import com.tencent.supersonic.chat.config.EntityRichInfo; +import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.PluginManager; +import com.tencent.supersonic.chat.plugin.PluginParseResult; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; -import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.PluginService; +import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.util.ContextUtils; import java.util.*; import java.util.stream.Collectors; +import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -30,30 +32,43 @@ public class EmbeddingBasedParser implements SemanticParser { @Override public void parse(QueryContext queryContext, ChatContext chatContext) { EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); - if (SatisfactionChecker.check(queryContext) || StringUtils.isBlank(embeddingConfig.getUrl())) { + if (StringUtils.isBlank(embeddingConfig.getUrl())) { return; } log.info("EmbeddingBasedParser parser query ctx: {}, chat ctx: {}", queryContext, chatContext); - for (Long domainId : getDomainMatched(queryContext)) { - String text = replaceText(queryContext, domainId); - List embeddingRetrievals = recallResult(text, hasCandidateQuery(queryContext)); - Optional pluginOptional = choosePlugin(embeddingRetrievals, domainId); - if (pluginOptional.isPresent()) { - Map embeddingRetrievalMap = embeddingRetrievals.stream() - .collect(Collectors.toMap(RecallRetrieval::getId, e -> e, (value1, value2) -> value1)); - Plugin plugin = pluginOptional.get(); - log.info("EmbeddingBasedParser text: {} domain: {} choose plugin: [{} {}]", - text, domainId, plugin.getId(), plugin.getName()); - PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType()); - SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(queryContext, domainId, - plugin, embeddingRetrievalMap); - semanticParseInfo.setQueryMode(pluginQuery.getQueryMode()); - pluginQuery.setParseInfo(semanticParseInfo); - queryContext.getCandidateQueries().add(pluginQuery); + Set domainIds = getDomainMatched(queryContext); + String text = queryContext.getRequest().getQueryText(); + if (!CollectionUtils.isEmpty(domainIds)) { + for (Long domainId : domainIds) { + List schemaElementMatches = getMatchedElements(queryContext, domainId); + String textReplaced = replaceText(text, schemaElementMatches); + List embeddingRetrievals = recallResult(textReplaced, hasCandidateQuery(queryContext)); + Optional pluginOptional = choosePlugin(embeddingRetrievals, domainId); + log.info("domain id :{} embedding result, text:{} embeddingResp:{} ",domainId, textReplaced, embeddingRetrievals); + pluginOptional.ifPresent(plugin -> buildQuery(plugin, embeddingRetrievals, domainId, textReplaced, queryContext, schemaElementMatches)); } + } else { + List embeddingRetrievals = recallResult(text, hasCandidateQuery(queryContext)); + Optional pluginOptional = choosePlugin(embeddingRetrievals, null); + pluginOptional.ifPresent(plugin -> buildQuery(plugin, embeddingRetrievals, null, text, queryContext, Lists.newArrayList())); } } + private void buildQuery(Plugin plugin, List embeddingRetrievals, + Long domainId, String text, + QueryContext queryContext, List schemaElementMatches) { + Map embeddingRetrievalMap = embeddingRetrievals.stream() + .collect(Collectors.toMap(RecallRetrieval::getId, e -> e, (value1, value2) -> value1)); + log.info("EmbeddingBasedParser text: {} domain: {} choose plugin: [{} {}]", + text, domainId, plugin.getId(), plugin.getName()); + PluginSemanticQuery pluginQuery = QueryManager.createPluginQuery(plugin.getType()); + SemanticParseInfo semanticParseInfo = buildSemanticParseInfo(domainId, plugin, text, + queryContext.getRequest(), embeddingRetrievalMap, schemaElementMatches); + semanticParseInfo.setQueryMode(pluginQuery.getQueryMode()); + pluginQuery.setParseInfo(semanticParseInfo); + queryContext.getCandidateQueries().add(pluginQuery); + } + private Set getDomainMatched(QueryContext queryContext) { Long queryDomainId = queryContext.getRequest().getDomainId(); if (queryDomainId != null && queryDomainId > 0) { @@ -62,51 +77,61 @@ public class EmbeddingBasedParser implements SemanticParser { return queryContext.getMapInfo().getMatchedDomains(); } - private SemanticParseInfo buildSemanticParseInfo(QueryContext queryContext, Long domainId, Plugin plugin, - Map embeddingRetrievalMap) { + private SemanticParseInfo buildSemanticParseInfo(Long domainId, Plugin plugin, String text, QueryReq queryReq, + Map embeddingRetrievalMap, + List schemaElementMatches) { SchemaElement schemaElement = new SchemaElement(); schemaElement.setDomain(domainId); schemaElement.setId(domainId); SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); + semanticParseInfo.setElementMatches(schemaElementMatches); semanticParseInfo.setDomain(schemaElement); - SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); - if (Double.parseDouble(embeddingRetrievalMap.get(plugin.getId().toString()).getDistance()) < THRESHOLD) { - semanticParseInfo.setBonus(SatisfactionChecker.BONUS_THRESHOLD); - } + double distance = Double.parseDouble(embeddingRetrievalMap.get(plugin.getId().toString()).getDistance()); + double score = text.length() * (1 - distance); Map properties = new HashMap<>(); - properties.put(Constants.CONTEXT, plugin); + PluginParseResult pluginParseResult = new PluginParseResult(); + pluginParseResult.setPlugin(plugin); + pluginParseResult.setRequest(queryReq); + pluginParseResult.setDistance(distance); + properties.put(Constants.CONTEXT, pluginParseResult); semanticParseInfo.setProperties(properties); - semanticParseInfo.setElementMatches(schemaMapInfo.getMatchedElements(domainId)); - fillSemanticParseInfo(queryContext, semanticParseInfo); - setEntityId(domainId, semanticParseInfo); + semanticParseInfo.setScore(score); + fillSemanticParseInfo(semanticParseInfo); + setEntity(domainId, semanticParseInfo); return semanticParseInfo; } - private Optional getEntityElementId(Long domainId) { - ConfigService configService = ContextUtils.getBean(ConfigService.class); - ChatConfigRich chatConfigRich = configService.getConfigRichInfo(domainId); - EntityRichInfo entityRichInfo = chatConfigRich.getChatDetailRichConfig().getEntity(); - if (entityRichInfo != null) { - SchemaElement schemaElement = entityRichInfo.getDimItem(); - if (schemaElement != null) { - return Optional.of(schemaElement.getId()); - } + private List getMatchedElements(QueryContext queryContext, Long domainId) { + SchemaMapInfo schemaMapInfo = queryContext.getMapInfo(); + List schemaElementMatches = schemaMapInfo.getMatchedElements(domainId); + if (schemaElementMatches == null) { + return Lists.newArrayList(); } - return Optional.empty(); + QueryReq queryReq = queryContext.getRequest(); + QueryFilters queryFilters = queryReq.getQueryFilters(); + if (queryFilters == null || CollectionUtils.isEmpty(queryFilters.getFilters())) { + return schemaElementMatches; + } + Map element = queryFilters.getFilters().stream() + .collect(Collectors.toMap(QueryFilter::getElementID, QueryFilter::getValue, (v1, v2) -> v1)); + return schemaElementMatches.stream().filter(schemaElementMatch -> + SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()) + || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType()) + || SchemaElementType.ENTITY.equals(schemaElementMatch.getElement().getType())) + .filter(schemaElementMatch -> + !element.containsKey(schemaElementMatch.getElement().getId()) || ( + element.containsKey(schemaElementMatch.getElement().getId()) && + element.get(schemaElementMatch.getElement().getId()).toString() + .equalsIgnoreCase(schemaElementMatch.getWord()) + )) + .collect(Collectors.toList()); } - private void setEntityId(Long domainId, SemanticParseInfo semanticParseInfo) { - Optional entityElementIdOptional = getEntityElementId(domainId); - if (entityElementIdOptional.isPresent()) { - Long entityElementId = entityElementIdOptional.get(); - for (QueryFilter filter : semanticParseInfo.getDimensionFilters()) { - if (entityElementId.equals(filter.getElementID())) { - String value = String.valueOf(filter.getValue()); - if (StringUtils.isNumeric(value)) { - semanticParseInfo.setEntity(Long.parseLong(value)); - } - } - } + private void setEntity(Long domainId, SemanticParseInfo semanticParseInfo) { + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + DomainSchema domainSchema = semanticService.getDomainSchema(domainId); + if (domainSchema != null && domainSchema.getEntity() != null) { + semanticParseInfo.setEntity(domainSchema.getEntity()); } } @@ -120,6 +145,12 @@ public class EmbeddingBasedParser implements SemanticParser { Map pluginMap = plugins.stream().collect(Collectors.toMap(Plugin::getId, p -> p)); for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) { Plugin plugin = pluginMap.get(Long.parseLong(embeddingRetrieval.getId())); + if (plugin == null) { + continue; + } + if (domainId == null) { + return Optional.of(plugin); + } if (!CollectionUtils.isEmpty(plugin.getDomainList()) && plugin.getDomainList().contains(domainId)) { return Optional.of(plugin); } @@ -131,7 +162,6 @@ public class EmbeddingBasedParser implements SemanticParser { try { PluginManager pluginManager = ContextUtils.getBean(PluginManager.class); EmbeddingResp embeddingResp = pluginManager.recognize(embeddingText); - log.info("embedding result, text:{} embeddingResp:{}", embeddingText, embeddingResp); List embeddingRetrievals = embeddingResp.getRetrieval(); if(!CollectionUtils.isEmpty(embeddingRetrievals)){ if (hasCandidateQuery) { @@ -154,25 +184,38 @@ public class EmbeddingBasedParser implements SemanticParser { return !CollectionUtils.isEmpty(queryContext.getCandidateQueries()); } - private void fillSemanticParseInfo(QueryContext queryContext, SemanticParseInfo semanticParseInfo) { - if (queryContext.getRequest().getQueryFilters() != null) { - semanticParseInfo.getDimensionFilters() - .addAll(queryContext.getRequest().getQueryFilters().getFilters()); + private void fillSemanticParseInfo(SemanticParseInfo semanticParseInfo) { + List schemaElementMatches = semanticParseInfo.getElementMatches(); + if (!CollectionUtils.isEmpty(schemaElementMatches)) { + schemaElementMatches.stream().filter(schemaElementMatch -> + SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()) + || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) + .forEach(schemaElementMatch -> { + QueryFilter queryFilter = new QueryFilter(); + queryFilter.setValue(schemaElementMatch.getWord()); + queryFilter.setElementID(schemaElementMatch.getElement().getId()); + queryFilter.setName(schemaElementMatch.getElement().getName()); + queryFilter.setOperator(FilterOperatorEnum.EQUALS); + queryFilter.setBizName(schemaElementMatch.getElement().getBizName()); + semanticParseInfo.getDimensionFilters().add(queryFilter); + }); } } - protected String replaceText(QueryContext queryContext, Long domainId) { - String text = queryContext.getRequest().getQueryText(); - List schemaElementMatches = queryContext.getMapInfo().getMatchedElements(domainId); + protected String replaceText(String text, List schemaElementMatches) { if (CollectionUtils.isEmpty(schemaElementMatches)) { return text; } List valueSchemaElementMatches = schemaElementMatches.stream() .filter(schemaElementMatch -> - SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) + SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()) + || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) .collect(Collectors.toList()); for (SchemaElementMatch schemaElementMatch : valueSchemaElementMatches) { String detectWord = schemaElementMatch.getDetectWord(); + if (StringUtils.isBlank(detectWord)) { + continue; + } text = text.replace(detectWord, ""); } return text; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingEntityResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingEntityResolver.java index 6ac6509d5..6a52efead 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingEntityResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/embedding/EmbeddingEntityResolver.java @@ -4,14 +4,10 @@ import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.config.ChatConfigRich; -import com.tencent.supersonic.chat.parser.function.DomainResolver; import com.tencent.supersonic.chat.service.ConfigService; -import com.tencent.supersonic.chat.utils.ComponentFactory; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Component; import java.util.List; @@ -28,18 +24,6 @@ public class EmbeddingEntityResolver { this.configService = configService; } - public Pair getDomainEntityId(QueryContext queryCtx, ChatContext chatCtx) { - DomainResolver domainResolver = ComponentFactory.getDomainResolver(); - Long domainId = domainResolver.resolve(queryCtx, chatCtx); - ChatConfigRich chatConfigRichResp = configService.getConfigRichInfo(domainId); - SchemaElement schemaElement = chatConfigRichResp.getChatDetailRichConfig().getEntity().getDimItem(); - if (schemaElement == null) { - return Pair.of(domainId, null); - } - Long entityId = getEntityValue(domainId, schemaElement.getId(), queryCtx, chatCtx); - return Pair.of(domainId, entityId); - } - private Long getEntityValue(Long domainId, Long entityElementId, QueryContext queryCtx, ChatContext chatCtx) { Long entityId = null; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/DomainResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/DomainResolver.java index 4ecd3e9db..45fc864c8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/DomainResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/DomainResolver.java @@ -3,9 +3,10 @@ package com.tencent.supersonic.chat.parser.function; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; +import java.util.List; public interface DomainResolver { - Long resolve(QueryContext queryContext, ChatContext chatCtx); + Long resolve(QueryContext queryContext, ChatContext chatCtx, List restrictiveDomains); } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionBasedParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionBasedParser.java index e32ccc9fd..beb862d25 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionBasedParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionBasedParser.java @@ -6,11 +6,11 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.config.FunctionCallConfig; -import com.tencent.supersonic.chat.parser.ParseMode; +import com.tencent.supersonic.chat.config.FunctionCallInfoConfig; import com.tencent.supersonic.chat.parser.SatisfactionChecker; import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.PluginManager; +import com.tencent.supersonic.chat.plugin.PluginParseConfig; import com.tencent.supersonic.chat.plugin.PluginParseResult; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; @@ -22,12 +22,15 @@ import com.tencent.supersonic.common.util.ContextUtils; import java.net.URI; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; + +import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpEntity; @@ -44,9 +47,12 @@ public class FunctionBasedParser implements SemanticParser { public static final double FUNCTION_BONUS_THRESHOLD = 200; + public static final double SKIP_DSL_LENGTH = 10; + + @Override public void parse(QueryContext queryCtx, ChatContext chatCtx) { - FunctionCallConfig functionCallConfig = ContextUtils.getBean(FunctionCallConfig.class); + FunctionCallInfoConfig functionCallConfig = ContextUtils.getBean(FunctionCallInfoConfig.class); PluginService pluginService = ContextUtils.getBean(PluginService.class); String functionUrl = functionCallConfig.getUrl(); @@ -55,39 +61,57 @@ public class FunctionBasedParser implements SemanticParser { queryCtx.getRequest().getQueryText()); return; } - DomainResolver domainResolver = ComponentFactory.getDomainResolver(); - Long domainId = domainResolver.resolve(queryCtx, chatCtx); - List functionNames = getFunctionNames(domainId); - log.info("domainId:{},functionNames:{}", domainId, functionNames); - if (Objects.isNull(domainId) || domainId <= 0) { + + Set matchedDomains = getMatchDomains(queryCtx); + List functionNames = getFunctionNames(matchedDomains); + log.info("matchedDomains:{},functionNames:{}", matchedDomains, functionNames); + + if (CollectionUtils.isEmpty(functionNames) || CollectionUtils.isEmpty(matchedDomains)) { return; } + List functionDOList = getFunctionDO(queryCtx.getRequest().getDomainId()); FunctionReq functionReq = FunctionReq.builder() .queryText(queryCtx.getRequest().getQueryText()) - .functionNames(functionNames).build(); - + .pluginConfigs(functionDOList).build(); FunctionResp functionResp = requestFunction(functionUrl, functionReq); log.info("requestFunction result:{}", functionResp.getToolSelection()); - if (Objects.isNull(functionResp) || StringUtils.isBlank(functionResp.getToolSelection())) { + if (skipFunction(queryCtx, functionResp)) { return; } PluginParseResult functionCallParseResult = new PluginParseResult(); String toolSelection = functionResp.getToolSelection(); Optional pluginOptional = pluginService.getPluginByName(toolSelection); - if (pluginOptional.isPresent()) { - toolSelection = pluginOptional.get().getType(); - functionCallParseResult.setPlugin(pluginOptional.get()); + if (!pluginOptional.isPresent()) { + log.info("pluginOptional is not exist:{}, skip the parse", toolSelection); + return; } + Plugin plugin = pluginOptional.get(); + toolSelection = plugin.getType(); + functionCallParseResult.setPlugin(plugin); + log.info("QueryManager PluginQueryModes:{}", QueryManager.getPluginQueryModes()); PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(toolSelection); + DomainResolver domainResolver = ComponentFactory.getDomainResolver(); + Long domainId = domainResolver.resolve(queryCtx, chatCtx, plugin.getDomainList()); + log.info("FunctionBasedParser domainId:{}",domainId); + if ((Objects.isNull(domainId) || domainId <= 0) && !plugin.isContainsAllDomain()) { + log.info("domain is null, skip the parse, select tool: {}", toolSelection); + return; + } + if (!plugin.getDomainList().contains(domainId) && !plugin.isContainsAllDomain()) { + return; + } SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); - parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(domainId)); + if (Objects.nonNull(domainId) && domainId > 0){ + parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(domainId)); + } functionCallParseResult.setRequest(queryCtx.getRequest()); Map properties = new HashMap<>(); properties.put(Constants.CONTEXT, functionCallParseResult); parseInfo.setProperties(properties); - parseInfo.setBonus(FUNCTION_BONUS_THRESHOLD); + parseInfo.setScore(FUNCTION_BONUS_THRESHOLD); + parseInfo.setQueryMode(semanticQuery.getQueryMode()); SchemaElement domain = new SchemaElement(); domain.setDomain(domainId); domain.setId(domainId); @@ -95,13 +119,61 @@ public class FunctionBasedParser implements SemanticParser { queryCtx.getCandidateQueries().add(semanticQuery); } - private List getFunctionNames(Long domainId) { + private Set getMatchDomains(QueryContext queryCtx) { + Set result = new HashSet<>(); + Long domainId = queryCtx.getRequest().getDomainId(); + if (Objects.nonNull(domainId) && domainId > 0) { + result.add(domainId); + return result; + } + return queryCtx.getMapInfo().getMatchedDomains(); + } + + private boolean skipFunction(QueryContext queryCtx, FunctionResp functionResp) { + if (Objects.isNull(functionResp) || StringUtils.isBlank(functionResp.getToolSelection())) { + return true; + } + String queryText = queryCtx.getRequest().getQueryText(); + + if (functionResp.getToolSelection().equalsIgnoreCase(DSLQuery.QUERY_MODE) + && queryText.length() < SKIP_DSL_LENGTH) { + log.info("queryText length is :{}, less than the threshold :{}, skip dsl.", queryText.length(), + SKIP_DSL_LENGTH); + return true; + } + return false; + } + + private List getFunctionDO(Long domainId) { + log.info("user decide domain:{}", domainId); + List plugins = PluginManager.getPlugins(); + List functionDOList = plugins.stream().filter(o -> { + if (o.getParseModeConfig() == null) { + return false; + } + if (!CollectionUtils.isEmpty(o.getDomainList())) {//过滤掉没选主题域的插件 + return true; + } + if (domainId == null || domainId <= 0L) { + return true; + } else { + return o.getDomainList().contains(domainId); + } + }).map(o -> { + PluginParseConfig functionCallConfig = JsonUtil.toObject(o.getParseModeConfig(), + PluginParseConfig.class); + return functionCallConfig; + }).collect(Collectors.toList()); + log.info("getFunctionDO:{}", JsonUtil.toString(functionDOList)); + return functionDOList; + } + + private List getFunctionNames(Set matchedDomains) { List plugins = PluginManager.getPlugins(); Set functionNames = plugins.stream() - .filter(entry -> ParseMode.FUNCTION_CALL.equals(entry.getParseMode())) .filter(entry -> { - if (!CollectionUtils.isEmpty(entry.getDomainList())) { - return entry.getDomainList().contains(domainId); + if (!CollectionUtils.isEmpty(entry.getDomainList()) && !CollectionUtils.isEmpty(matchedDomains)) { + return entry.getDomainList().stream().anyMatch(matchedDomains::contains); } return true; } @@ -118,7 +190,7 @@ public class FunctionBasedParser implements SemanticParser { URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri(); RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); try { - log.info("requestFunction functionReq:{}", functionReq); + log.info("requestFunction functionReq:{}", JsonUtil.toString(functionReq)); ResponseEntity responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity, FunctionResp.class); log.info("requestFunction responseEntity:{},cost:{}", responseEntity, diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionFiled.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionFiled.java new file mode 100644 index 000000000..5c250ca51 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionFiled.java @@ -0,0 +1,13 @@ +package com.tencent.supersonic.chat.parser.function; + + +import lombok.Data; + +@Data +public class FunctionFiled { + + private String type; + + private String description; + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionReq.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionReq.java index 65c382cc0..ef52541e6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionReq.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/FunctionReq.java @@ -1,6 +1,8 @@ package com.tencent.supersonic.chat.parser.function; import java.util.List; + +import com.tencent.supersonic.chat.plugin.PluginParseConfig; import lombok.Builder; import lombok.Data; @@ -10,6 +12,6 @@ public class FunctionReq { private String queryText; - private List functionNames; + private List pluginConfigs; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/HeuristicDomainResolver.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/HeuristicDomainResolver.java index db7a486e1..c56508941 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/HeuristicDomainResolver.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/HeuristicDomainResolver.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.parser.function; import com.tencent.supersonic.chat.api.pojo.*; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.component.SemanticQuery; import lombok.extern.slf4j.Slf4j; @@ -25,10 +25,10 @@ public class HeuristicDomainResolver implements DomainResolver { Map.Entry maxDomain = domainTypeMap.entrySet().stream() .filter(entry -> domainQueryModes.containsKey(entry.getKey())) .sorted((o1, o2) -> { - int difference = o1.getValue().getCount() - o2.getValue().getCount(); + int difference = o2.getValue().getCount() - o1.getValue().getCount(); if (difference == 0) { - return (int) ((o1.getValue().getMaxSimilarity() - - o2.getValue().getMaxSimilarity()) * 100); + return (int) ((o2.getValue().getMaxSimilarity() + - o1.getValue().getMaxSimilarity()) * 100); } return difference; }).findFirst().orElse(null); @@ -46,7 +46,7 @@ public class HeuristicDomainResolver implements DomainResolver { * @return false will use context domain, true will use other domain , maybe include context domain */ protected static boolean isAllowSwitch(Map domainQueryModes, SchemaMapInfo schemaMap, - ChatContext chatCtx, QueryRequest searchCtx, Long domainId) { + ChatContext chatCtx, QueryReq searchCtx, Long domainId, List restrictiveDomains) { if (!Objects.nonNull(domainId) || domainId <= 0) { return true; } @@ -81,6 +81,9 @@ public class HeuristicDomainResolver implements DomainResolver { } } + if (CollectionUtils.isNotEmpty(restrictiveDomains) && !restrictiveDomains.contains(domainId)) { + return true; + } // if context domain not in schemaMap , will switch if (schemaMap.getMatchedElements(domainId) == null || schemaMap.getMatchedElements(domainId).size() <= 0) { log.info("domainId not in schemaMap "); @@ -118,24 +121,36 @@ public class HeuristicDomainResolver implements DomainResolver { } - public Long resolve(QueryContext queryContext, ChatContext chatCtx) { + public Long resolve(QueryContext queryContext, ChatContext chatCtx, List restrictiveDomains) { Long domainId = queryContext.getRequest().getDomainId(); if (Objects.nonNull(domainId) && domainId > 0) { - return domainId; + if (CollectionUtils.isNotEmpty(restrictiveDomains) && restrictiveDomains.contains(domainId)) { + return domainId; + } else { + return null; + } } SchemaMapInfo mapInfo = queryContext.getMapInfo(); Set matchedDomains = mapInfo.getMatchedDomains(); + if (CollectionUtils.isNotEmpty(restrictiveDomains)) { + matchedDomains = matchedDomains.stream() + .filter(matchedDomain -> restrictiveDomains.contains(matchedDomain)) + .collect(Collectors.toSet()); + } Map domainQueryModes = new HashMap<>(); for (Long matchedDomain : matchedDomains) { domainQueryModes.put(matchedDomain, null); } + if(domainQueryModes.size()==1){ + return domainQueryModes.keySet().stream().findFirst().get(); + } return resolve(domainQueryModes, queryContext, chatCtx, - queryContext.getMapInfo()); + queryContext.getMapInfo(),restrictiveDomains); } public Long resolve(Map domainQueryModes, QueryContext queryContext, - ChatContext chatCtx, SchemaMapInfo schemaMap) { - Long selectDomain = selectDomain(domainQueryModes, queryContext.getRequest(), chatCtx, schemaMap); + ChatContext chatCtx, SchemaMapInfo schemaMap, List restrictiveDomains) { + Long selectDomain = selectDomain(domainQueryModes, queryContext.getRequest(), chatCtx, schemaMap,restrictiveDomains); if (selectDomain > 0) { log.info("selectDomain {} ", selectDomain); return selectDomain; @@ -144,9 +159,9 @@ public class HeuristicDomainResolver implements DomainResolver { return selectDomainBySchemaElementCount(domainQueryModes, schemaMap); } - public Long selectDomain(Map domainQueryModes, QueryRequest queryContext, + public Long selectDomain(Map domainQueryModes, QueryReq queryContext, ChatContext chatCtx, - SchemaMapInfo schemaMap) { + SchemaMapInfo schemaMap, List restrictiveDomains) { // if QueryContext has domainId and in domainQueryModes if (domainQueryModes.containsKey(queryContext.getDomainId())) { log.info("selectDomain from QueryContext [{}]", queryContext.getDomainId()); @@ -155,7 +170,7 @@ public class HeuristicDomainResolver implements DomainResolver { // if ChatContext has domainId and in domainQueryModes if (chatCtx.getParseInfo().getDomainId() > 0) { Long domainId = chatCtx.getParseInfo().getDomainId(); - if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, queryContext, domainId)) { + if (!isAllowSwitch(domainQueryModes, schemaMap, chatCtx, queryContext, domainId,restrictiveDomains)) { log.info("selectDomain from ChatContext [{}]", domainId); return domainId; } @@ -163,4 +178,4 @@ public class HeuristicDomainResolver implements DomainResolver { // default 0 return 0L; } -} \ No newline at end of file +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/Parameters.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/Parameters.java new file mode 100644 index 000000000..bd9f88e01 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/function/Parameters.java @@ -0,0 +1,17 @@ +package com.tencent.supersonic.chat.parser.function; + +import lombok.Data; +import java.util.List; +import java.util.Map; + +@Data +public class Parameters { + + //default: object + private String type = "object"; + + private Map properties; + + private List required; + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/LLMTimeEnhancementParse.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/LLMTimeEnhancementParse.java new file mode 100644 index 000000000..0a9440ee9 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/LLMTimeEnhancementParse.java @@ -0,0 +1,49 @@ +package com.tencent.supersonic.chat.parser.llm; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.tencent.supersonic.chat.api.component.SemanticParser; +import com.tencent.supersonic.chat.api.component.SemanticQuery; +import com.tencent.supersonic.chat.api.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.utils.ChatGptHelper; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.util.ContextUtils; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LLMTimeEnhancementParse implements SemanticParser { + + + @Override + public void parse(QueryContext queryContext, ChatContext chatContext) { + log.info("before queryContext:{},chatContext:{}",queryContext,chatContext); + ChatGptHelper chatGptHelper = ContextUtils.getBean(ChatGptHelper.class); + String inferredTime = chatGptHelper.inferredTime(queryContext.getRequest().getQueryText()); + try { + if (!queryContext.getCandidateQueries().isEmpty()) { + for (SemanticQuery query : queryContext.getCandidateQueries()) { + DateConf dateInfo = query.getParseInfo().getDateInfo(); + JSONObject jsonObject = JSON.parseObject(inferredTime); + if (jsonObject.containsKey("date")){ + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); + dateInfo.setStartDate(jsonObject.getString("date")); + dateInfo.setEndDate(jsonObject.getString("date")); + query.getParseInfo().setDateInfo(dateInfo); + }else if (jsonObject.containsKey("start")){ + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); + dateInfo.setStartDate(jsonObject.getString("start")); + dateInfo.setEndDate(jsonObject.getString("end")); + query.getParseInfo().setDateInfo(dateInfo); + } + } + } + }catch (Exception exception){ + log.error("{} parse error,this reason is:{}",LLMTimeEnhancementParse.class.getSimpleName(), (Object) exception.getStackTrace()); + } + + log.info("after queryContext:{},chatContext:{}",queryContext,chatContext); + } + + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AggregateTypeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AggregateTypeParser.java index 9302ab920..4b70804fe 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AggregateTypeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/AggregateTypeParser.java @@ -1,22 +1,28 @@ package com.tencent.supersonic.chat.parser.rule; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.AVG; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.COUNT; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.DISTINCT; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MAX; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.MIN; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.TOPN; + import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; - -import java.util.*; +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; - -import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.*; - @Slf4j public class AggregateTypeParser implements SemanticParser { @@ -29,7 +35,7 @@ public class AggregateTypeParser implements SemanticParser { new AbstractMap.SimpleEntry<>(DISTINCT, Pattern.compile("(?i)(uv)")), new AbstractMap.SimpleEntry<>(COUNT, Pattern.compile("(?i)(总数|pv)")), new AbstractMap.SimpleEntry<>(NONE, Pattern.compile("(?i)(明细)")) - ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,(k1,k2)->k2)); @Override public void parse(QueryContext queryContext, ChatContext chatContext) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java index 073a4906c..74a7713f9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/ContextInheritParser.java @@ -1,17 +1,25 @@ package com.tencent.supersonic.chat.parser.rule; import com.tencent.supersonic.chat.api.component.SemanticParser; -import com.tencent.supersonic.chat.api.pojo.*; -import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery; +import com.tencent.supersonic.chat.api.component.SemanticQuery; +import com.tencent.supersonic.chat.api.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.SchemaElementType; +import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.common.util.JsonUtil; - -import java.util.*; +import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery; +import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery; +import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; - import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*; @@ -23,7 +31,8 @@ public class ContextInheritParser implements SemanticParser { new AbstractMap.SimpleEntry<>(DIMENSION, Arrays.asList(DIMENSION, VALUE)), new AbstractMap.SimpleEntry<>(VALUE, Arrays.asList(VALUE, DIMENSION)), new AbstractMap.SimpleEntry<>(ENTITY, Arrays.asList(ENTITY)), - new AbstractMap.SimpleEntry<>(DOMAIN, Arrays.asList(DOMAIN)) + new AbstractMap.SimpleEntry<>(DOMAIN, Arrays.asList(DOMAIN)), + new AbstractMap.SimpleEntry<>(ID, Arrays.asList(ID)) ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @Override @@ -40,7 +49,9 @@ public class ContextInheritParser implements SemanticParser { for (SchemaElementMatch match : chatContext.getParseInfo().getElementMatches()) { SchemaElementType matchType = match.getElement().getType(); // mutual exclusive element types should not be inherited - if (!containsTypes(elementMatches, MUTUAL_EXCLUSIVE_MAP.get(matchType))) { + RuleSemanticQuery ruleQuery = QueryManager.getRuleQuery(chatContext.getParseInfo().getQueryMode()); + if (!containsTypes(elementMatches, matchType, ruleQuery)) { + match.setMode(SchemaElementMatch.MatchMode.INHERIT); matchesToInherit.add(match); } } @@ -53,22 +64,31 @@ public class ContextInheritParser implements SemanticParser { } } - private boolean containsTypes(List matches, List types) { - return matches.stream().anyMatch(m -> types.contains(m.getElement().getType())); + private boolean containsTypes(List matches, SchemaElementType matchType, + RuleSemanticQuery ruleQuery) { + List types = MUTUAL_EXCLUSIVE_MAP.get(matchType); + + return matches.stream().anyMatch(m -> { + SchemaElementType type = m.getElement().getType(); + if (Objects.nonNull(ruleQuery) && ruleQuery instanceof MetricSemanticQuery + && !(ruleQuery instanceof MetricEntityQuery)) { + return types.contains(type); + } + return type.equals(matchType); + }); } protected boolean shouldInherit(QueryContext queryContext, ChatContext chatContext) { - if (queryContext.getMapInfo().getMatchedElements( - chatContext.getParseInfo().getDomainId()) == null) { + Long contextDomainId = chatContext.getParseInfo().getDomainId(); + if (queryContext.getMapInfo().getMatchedElements(contextDomainId) == null) { return false; } // if candidates have only one MetricDomain mode and context has value filter , count in context - if (queryContext.getCandidateQueries().size() == 1 && (queryContext.getCandidateQueries() - .get(0) instanceof MetricDomainQuery) - && queryContext.getCandidateQueries().get(0).getParseInfo().getDomainId() - .equals(chatContext.getParseInfo().getDomainId()) - && !CollectionUtils.isEmpty(chatContext.getParseInfo().getDimensionFilters())) { + List candidateQueries = queryContext.getCandidateQueries().stream() + .filter(semanticQuery -> semanticQuery.getParseInfo().getDomainId().equals(contextDomainId)).collect( + Collectors.toList()); + if (candidateQueries.size() == 1 && (candidateQueries.get(0) instanceof MetricDomainQuery)) { return true; } else { return queryContext.getCandidateQueries().size() == 0; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java index af1d7e0f8..290a7851b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/QueryModeParser.java @@ -6,7 +6,6 @@ import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import java.util.*; -import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java index fec43ee27..1d20a3208 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/rule/TimeRangeParser.java @@ -46,7 +46,7 @@ public class TimeRangeParser implements SemanticParser { for (SemanticQuery query : queryContext.getCandidateQueries()) { query.getParseInfo().setDateInfo(dateConf); } - } else if(QueryManager.containsRuleQuery(chatContext.getParseInfo().getQueryMode())) { + } else if (QueryManager.containsRuleQuery(chatContext.getParseInfo().getQueryMode())) { RuleSemanticQuery semanticQuery = QueryManager.createRuleQuery( chatContext.getParseInfo().getQueryMode()); // inherit parse info from context @@ -64,7 +64,7 @@ public class TimeRangeParser implements SemanticParser { List times = TimeNLPUtil.parse(queryText); if (times.size() > 0) { startDate = times.get(0).getTime(); - }else { + } else { return null; } @@ -133,7 +133,7 @@ public class TimeRangeParser implements SemanticParser { info.setPeriod(Constants.DAY); } days = days * num; - info.setDateMode(DateConf.DateMode.RECENT_UNITS); + info.setDateMode(DateConf.DateMode.RECENT); String text = "近" + num + zhPeriod; if (Strings.isNotEmpty(m.group("periodStr"))) { text = m.group("periodStr"); @@ -175,11 +175,11 @@ public class TimeRangeParser implements SemanticParser { private DateConf getDateConf(Date startDate, Date endDate) { if (startDate == null || endDate == null) { - return null; + return null; } DateConf info = new DateConf(); - info.setDateMode(DateConf.DateMode.BETWEEN_CONTINUOUS); + info.setDateMode(DateConf.DateMode.BETWEEN); info.setStartDate(DATE_FORMAT.format(startDate)); info.setEndDate(DATE_FORMAT.format(endDate)); return info; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/PluginDO.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/PluginDO.java index 30e6a60bf..3cdf0bb14 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/PluginDO.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/dataobject/PluginDO.java @@ -53,11 +53,21 @@ public class PluginDO { */ private String updatedBy; + /** + * + */ + private String parseModeConfig; + /** * */ private String config; + /** + * + */ + private String comment; + /** * * @return id @@ -218,6 +228,22 @@ public class PluginDO { this.updatedBy = updatedBy == null ? null : updatedBy.trim(); } + /** + * + * @return parse_mode_config + */ + public String getParseModeConfig() { + return parseModeConfig; + } + + /** + * + * @param parseModeConfig + */ + public void setParseModeConfig(String parseModeConfig) { + this.parseModeConfig = parseModeConfig == null ? null : parseModeConfig.trim(); + } + /** * * @return config @@ -233,4 +259,20 @@ public class PluginDO { public void setConfig(String config) { this.config = config == null ? null : config.trim(); } + + /** + * + * @return comment + */ + public String getComment() { + return comment; + } + + /** + * + * @param comment + */ + public void setComment(String comment) { + this.comment = comment == null ? null : comment.trim(); + } } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/PluginDOMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/PluginDOMapper.java index 426be06d8..9f410ed00 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/PluginDOMapper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/mapper/PluginDOMapper.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.chat.persistence.mapper; - import com.tencent.supersonic.chat.persistence.dataobject.PluginDO; import com.tencent.supersonic.chat.persistence.dataobject.PluginDOExample; import org.apache.ibatis.annotations.Mapper; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatConfigRepository.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatConfigRepository.java index 33de13ec6..3a9250e44 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatConfigRepository.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatConfigRepository.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.persistence.repository; import com.tencent.supersonic.chat.config.ChatConfig; -import com.tencent.supersonic.chat.config.ChatConfigFilter; -import com.tencent.supersonic.chat.config.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import java.util.List; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java index 4b818c464..3d9dac6a4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/ChatQueryRepository.java @@ -2,17 +2,17 @@ package com.tencent.supersonic.chat.persistence.repository; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; -import com.tencent.supersonic.chat.api.pojo.response.QueryResponse; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; public interface ChatQueryRepository { - PageInfo getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId); + PageInfo getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId); - void createChatQuery(QueryResult queryResult, QueryRequest queryContext, ChatContext chatCtx); + void createChatQuery(QueryResult queryResult, ChatContext chatCtx); ChatQueryDO getLastChatQuery(long chatId); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatConfigRepositoryImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatConfigRepositoryImpl.java index cec064a1f..2670d8be2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatConfigRepositoryImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatConfigRepositoryImpl.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.persistence.repository.impl; import com.tencent.supersonic.chat.config.ChatConfig; -import com.tencent.supersonic.chat.config.ChatConfigFilter; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; import com.tencent.supersonic.chat.config.ChatConfigFilterInternal; -import com.tencent.supersonic.chat.config.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO; import com.tencent.supersonic.chat.persistence.repository.ChatConfigRepository; import com.tencent.supersonic.chat.utils.ChatConfigHelper; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java index a5e87980b..dad207009 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/persistence/repository/impl/ChatQueryRepositoryImpl.java @@ -3,12 +3,12 @@ package com.tencent.supersonic.chat.persistence.repository.impl; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.chat.api.pojo.ChatContext; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDOExample.Criteria; -import com.tencent.supersonic.chat.api.pojo.response.QueryResponse; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.persistence.mapper.ChatQueryDOMapper; import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository; @@ -35,7 +35,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { } @Override - public PageInfo getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId) { + public PageInfo getChatQuery(PageQueryInfoReq pageQueryInfoCommend, long chatId) { ChatQueryDOExample example = new ChatQueryDOExample(); example.setOrderByClause("question_id desc"); Criteria criteria = example.createCriteria(); @@ -46,7 +46,7 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { pageQueryInfoCommend.getPageSize()) .doSelectPageInfo(() -> chatQueryDOMapper.selectByExampleWithBLOBs(example)); - PageInfo chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo); + PageInfo chatQueryVOPageInfo = PageUtils.pageInfo2PageInfoVo(pageInfo); chatQueryVOPageInfo.setList( pageInfo.getList().stream().map(this::convertTo) .sorted(Comparator.comparingInt(o -> o.getQuestionId().intValue())) @@ -54,8 +54,8 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { return chatQueryVOPageInfo; } - private QueryResponse convertTo(ChatQueryDO chatQueryDO) { - QueryResponse queryResponse = new QueryResponse(); + private QueryResp convertTo(ChatQueryDO chatQueryDO) { + QueryResp queryResponse = new QueryResp(); BeanUtils.copyProperties(chatQueryDO, queryResponse); QueryResult queryResult = JsonUtil.toObject(chatQueryDO.getQueryResult(), QueryResult.class); queryResult.setQueryId(chatQueryDO.getQuestionId()); @@ -64,16 +64,16 @@ public class ChatQueryRepositoryImpl implements ChatQueryRepository { } @Override - public void createChatQuery(QueryResult queryResult, QueryRequest queryRequest, ChatContext chatCtx) { + public void createChatQuery(QueryResult queryResult, ChatContext chatCtx) { ChatQueryDO chatQueryDO = new ChatQueryDO(); - chatQueryDO.setChatId(Long.valueOf(queryRequest.getChatId())); + chatQueryDO.setChatId(Long.valueOf(chatCtx.getChatId())); chatQueryDO.setCreateTime(new java.util.Date()); - chatQueryDO.setUserName(queryRequest.getUser().getName()); + chatQueryDO.setUserName(chatCtx.getUser()); chatQueryDO.setQueryState(queryResult.getQueryState().ordinal()); - chatQueryDO.setQueryText(queryRequest.getQueryText()); + chatQueryDO.setQueryText(chatCtx.getQueryText()); chatQueryDO.setQueryResult(JsonUtil.toString(queryResult)); chatQueryDOMapper.insert(chatQueryDO); - ChatQueryDO lastChatQuery = getLastChatQuery(queryRequest.getChatId()); + ChatQueryDO lastChatQuery = getLastChatQuery(chatCtx.getChatId()); Long queryId = lastChatQuery.getQuestionId(); queryResult.setQueryId(queryId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java index 469e11bfc..5ce89f197 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/Plugin.java @@ -1,37 +1,58 @@ package com.tencent.supersonic.chat.plugin; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Lists; import com.tencent.supersonic.chat.parser.ParseMode; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; - +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; @Data public class Plugin extends RecordInfo { private Long id; - //plugin type WEB_PAGE WEB_SERVICE + /*** + * plugin type WEB_PAGE WEB_SERVICE + */ private String type; - private List domainList; + private List domainList = Lists.newArrayList(); - //description, for parsing + /** + * description, for parsing + */ private String pattern; - //parse + /** + * parse + */ private ParseMode parseMode; + private String parseModeConfig; + private String name; - //config for different plugin type + /** + * config for different plugin type + */ private String config; - public List getPatterns() { - return Stream.of(getPattern().split("\\|")).collect(Collectors.toList()); + private String comment; + + public List getExampleQuestionList() { + if (StringUtils.isNotBlank(parseModeConfig)) { + PluginParseConfig pluginParseConfig = JSONObject.parseObject(parseModeConfig, PluginParseConfig.class); + return pluginParseConfig.getExamples(); + } + return Lists.newArrayList(); + } + + public boolean isContainsAllDomain() { + return CollectionUtils.isNotEmpty(domainList) && domainList.contains(-1L); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java index 57540ebb5..67bb6bb18 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginManager.java @@ -16,6 +16,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.util.Strings; import org.springframework.context.event.EventListener; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.*; @@ -40,7 +41,8 @@ public class PluginManager { public static List getPlugins() { PluginService pluginService = ContextUtils.getBean(PluginService.class); - List pluginList = pluginService.getPluginList(); + List pluginList = pluginService.getPluginList().stream().filter(plugin -> + CollectionUtils.isNotEmpty(plugin.getDomainList())).collect(Collectors.toList()); pluginList.addAll(internalPluginMap.values()); return new ArrayList<>(pluginList); } @@ -48,7 +50,7 @@ public class PluginManager { @EventListener public void addPlugin(PluginAddEvent pluginAddEvent) { Plugin plugin = pluginAddEvent.getPlugin(); - if (ParseMode.EMBEDDING_RECALL.equals(plugin.getParseMode())) { + if (CollectionUtils.isNotEmpty(plugin.getExampleQuestionList())) { requestEmbeddingPluginAdd(convert(Lists.newArrayList(plugin))); } } @@ -57,10 +59,10 @@ public class PluginManager { public void updatePlugin(PluginUpdateEvent pluginUpdateEvent) { Plugin oldPlugin = pluginUpdateEvent.getOldPlugin(); Plugin newPlugin = pluginUpdateEvent.getNewPlugin(); - if (ParseMode.EMBEDDING_RECALL.equals(oldPlugin.getParseMode())) { + if (CollectionUtils.isNotEmpty(oldPlugin.getExampleQuestionList())) { requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(oldPlugin))); } - if (ParseMode.EMBEDDING_RECALL.equals(newPlugin.getParseMode())) { + if (CollectionUtils.isNotEmpty(newPlugin.getExampleQuestionList())) { requestEmbeddingPluginAdd(convert(Lists.newArrayList(newPlugin))); } } @@ -68,29 +70,30 @@ public class PluginManager { @EventListener public void delPlugin(PluginAddEvent pluginAddEvent) { Plugin plugin = pluginAddEvent.getPlugin(); - if (ParseMode.EMBEDDING_RECALL.equals(plugin.getParseMode())) { + if (CollectionUtils.isNotEmpty(plugin.getExampleQuestionList())) { requestEmbeddingPluginDelete(getEmbeddingId(Lists.newArrayList(plugin))); } - } public void requestEmbeddingPluginDelete(Set ids) { - if(CollectionUtils.isEmpty(ids)){ + if (CollectionUtils.isEmpty(ids)) { return; } doRequest(embeddingConfig.getDeletePath(), JSONObject.toJSONString(ids)); } - - public void requestEmbeddingPluginAdd(List> maps) { - if(CollectionUtils.isEmpty(maps)){ + public void requestEmbeddingPluginAdd(List> maps) { + if (CollectionUtils.isEmpty(maps)) { return; } - doRequest(embeddingConfig.getAddPath(), JSONObject.toJSONString(maps)); + doRequest(embeddingConfig.getAddPath(), JSONObject.toJSONString(maps)); } public void doRequest(String path, String jsonBody) { - String url = embeddingConfig.getUrl()+ path; + if (Strings.isEmpty(embeddingConfig.getUrl())) { + return; + } + String url = embeddingConfig.getUrl() + path; HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); headers.setLocation(URI.create(url)); @@ -99,7 +102,8 @@ public class PluginManager { HttpEntity entity = new HttpEntity<>(jsonBody, headers); log.info("[embedding] equest body :{}, url:{}", jsonBody, url); ResponseEntity responseEntity = - restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference() {}); + restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference() { + }); log.info("[embedding] result body:{}", responseEntity); } @@ -111,7 +115,7 @@ public class PluginManager { } public EmbeddingResp recognize(String embeddingText) { - String url = embeddingConfig.getUrl()+ embeddingConfig.getRecognizePath() + "?n_results=" + embeddingConfig.getNResult(); + String url = embeddingConfig.getUrl() + embeddingConfig.getRecognizePath() + "?n_results=" + embeddingConfig.getNResult(); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); headers.setLocation(URI.create(url)); @@ -121,10 +125,11 @@ public class PluginManager { HttpEntity entity = new HttpEntity<>(jsonBody, headers); log.info("[embedding] request body:{}, url:{}", jsonBody, url); ResponseEntity> embeddingResponseEntity = - restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference>() {}); - log.info("[embedding] recognize result body:{}",embeddingResponseEntity); + restTemplate.exchange(requestUrl, HttpMethod.POST, entity, new ParameterizedTypeReference>() { + }); + log.info("[embedding] recognize result body:{}", embeddingResponseEntity); List embeddingResps = embeddingResponseEntity.getBody(); - if(CollectionUtils.isNotEmpty(embeddingResps)){ + if (CollectionUtils.isNotEmpty(embeddingResps)) { for (EmbeddingResp embeddingResp : embeddingResps) { List embeddingRetrievals = embeddingResp.getRetrieval(); for (RecallRetrieval embeddingRetrieval : embeddingRetrievals) { @@ -136,13 +141,13 @@ public class PluginManager { throw new RuntimeException("get embedding result failed"); } - public List> convert(List plugins){ + public List> convert(List plugins) { List> maps = Lists.newArrayList(); - for(Plugin plugin : plugins){ - List patterns = plugin.getPatterns(); + for (Plugin plugin : plugins) { + List exampleQuestions = plugin.getExampleQuestionList(); int num = 0; - for(String pattern : patterns){ - Map map = new HashMap<>(); + for (String pattern : exampleQuestions) { + Map map = new HashMap<>(); map.put("preset_query_id", generateUniqueEmbeddingId(num, plugin.getId())); map.put("preset_query", pattern); maps.add(map); @@ -155,7 +160,7 @@ public class PluginManager { private Set getEmbeddingId(List plugins) { Set embeddingIdSet = new HashSet<>(); for (Map map : convert(plugins)) { - embeddingIdSet.addAll(map.keySet()); + embeddingIdSet.add(map.get("preset_query_id")); } return embeddingIdSet; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseConfig.java b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseConfig.java new file mode 100644 index 000000000..d6482eadb --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseConfig.java @@ -0,0 +1,21 @@ +package com.tencent.supersonic.chat.plugin; + + +import com.tencent.supersonic.chat.parser.function.Parameters; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +@Data +public class PluginParseConfig implements Serializable { + + private String name; + + private String description; + + public Parameters parameters; + + public List examples; + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseResult.java index ba7a4a9c7..ab93d2908 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseResult.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/plugin/PluginParseResult.java @@ -1,11 +1,12 @@ package com.tencent.supersonic.chat.plugin; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import lombok.Data; @Data public class PluginParseResult { private Plugin plugin; - private QueryRequest request; + private QueryReq request; + private double distance; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/HeuristicQuerySelector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/HeuristicQuerySelector.java index 82ef74840..cb6602263 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/HeuristicQuerySelector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/HeuristicQuerySelector.java @@ -5,39 +5,46 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.common.pojo.Constants; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; + +import java.util.*; + import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; @Slf4j public class HeuristicQuerySelector implements QuerySelector { - @Override - public SemanticQuery select(List candidateQueries) { - double maxScore = 0; - SemanticQuery pickedQuery = null; - if (CollectionUtils.isNotEmpty(candidateQueries) && candidateQueries.size() == 1) { - return candidateQueries.get(0); - } - for (SemanticQuery query : candidateQueries) { - SemanticParseInfo semanticParse = query.getParseInfo(); - double score = computeScore(semanticParse); - if (score > maxScore) { - maxScore = score; - pickedQuery = query; - } - log.info("candidate query (domain={}, queryMode={}) with score={}", - semanticParse.getDomainName(), semanticParse.getQueryMode(), score); - } + private static final double MATCH_INHERIT_PENALTY = 0.5; + private static final double MATCH_CURRENT_REWORD = 2; + private static final double CANDIDATE_THRESHOLD = 0.2; - return pickedQuery; + @Override + public List select(List candidateQueries) { + List selectedQueries = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(candidateQueries) && candidateQueries.size() == 1) { + selectedQueries.addAll(candidateQueries); + } else { + OptionalDouble maxScoreOp = candidateQueries.stream().mapToDouble( + q -> computeScore(q.getParseInfo())).max(); + if (maxScoreOp.isPresent()) { + double maxScore = maxScoreOp.getAsDouble(); + + candidateQueries.stream().forEach(query -> { + SemanticParseInfo semanticParse = query.getParseInfo(); + if ((maxScore - semanticParse.getScore()) / maxScore <= CANDIDATE_THRESHOLD) { + selectedQueries.add(query); + } + log.info("candidate query (domain={}, queryMode={}) with score={}", + semanticParse.getDomainName(), semanticParse.getQueryMode(), semanticParse.getScore()); + }); + } + } + return selectedQueries; } private double computeScore(SemanticParseInfo semanticParse) { - double score = 0; + double totalScore = 0; Map maxSimilarityMatch = new HashMap<>(); for (SchemaElementMatch match : semanticParse.getElementMatches()) { @@ -49,13 +56,19 @@ public class HeuristicQuerySelector implements QuerySelector { } for (SchemaElementMatch match : maxSimilarityMatch.values()) { - score += - Optional.ofNullable(match.getDetectWord()).orElse(Constants.EMPTY).length() * match.getSimilarity(); + double matchScore = Optional.ofNullable(match.getDetectWord()).orElse(Constants.EMPTY).length() * match.getSimilarity(); + if (match.equals(SchemaElementMatch.MatchMode.INHERIT)) { + matchScore *= MATCH_INHERIT_PENALTY; + } else { + matchScore *= MATCH_CURRENT_REWORD; + } + totalScore += matchScore; } - // bonus is a special construct to control the final score - score += semanticParse.getBonus(); + // original score in parse info acts like an extra bonus + totalScore += semanticParse.getScore(); + semanticParse.setScore(totalScore); - return score; + return totalScore; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryManager.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryManager.java index 4cd897fb3..9b808f7f1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryManager.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/QueryManager.java @@ -3,6 +3,9 @@ package com.tencent.supersonic.chat.query; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; +import com.tencent.supersonic.chat.query.rule.entity.EntitySemanticQuery; +import com.tencent.supersonic.chat.query.rule.metric.MetricSemanticQuery; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -22,6 +25,14 @@ public class QueryManager { } } + public static SemanticQuery createQuery(String queryMode) { + if (containsRuleQuery(queryMode)) { + return createRuleQuery(queryMode); + } else { + return createPluginQuery(queryMode); + } + } + public static RuleSemanticQuery createRuleQuery(String queryMode) { RuleSemanticQuery semanticQuery = ruleQueryMap.get(queryMode); if (Objects.isNull(semanticQuery)) { @@ -45,7 +56,6 @@ public class QueryManager { throw new RuntimeException("no supported queryMode :" + queryMode); } } - public static boolean containsRuleQuery(String queryMode) { if (queryMode == null) { return false; @@ -53,6 +63,27 @@ public class QueryManager { return ruleQueryMap.containsKey(queryMode); } + public static boolean isMetricQuery(String queryMode) { + if (queryMode == null || !ruleQueryMap.containsKey(queryMode)) { + return false; + } + return ruleQueryMap.get(queryMode) instanceof MetricSemanticQuery; + } + + public static boolean isEntityQuery(String queryMode) { + if (queryMode == null || !ruleQueryMap.containsKey(queryMode)) { + return false; + } + return ruleQueryMap.get(queryMode) instanceof EntitySemanticQuery; + } + + public static RuleSemanticQuery getRuleQuery(String queryMode) { + if (queryMode == null) { + return null; + } + return ruleQueryMap.get(queryMode); + } + public static List getRuleQueries() { return new ArrayList<>(ruleQueryMap.values()); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/QuerySelector.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/QuerySelector.java index dcc8ae79c..906938548 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/QuerySelector.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/QuerySelector.java @@ -9,5 +9,5 @@ import java.util.List; **/ public interface QuerySelector { - SemanticQuery select(List candidateQueries); + List select(List candidateQueries); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/ParamOption.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/ParamOption.java new file mode 100644 index 000000000..4e786eefb --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/ParamOption.java @@ -0,0 +1,37 @@ +package com.tencent.supersonic.chat.query.plugin; + +import lombok.Data; + +@Data +public class ParamOption { + + private ParamType paramType; + + private OptionType optionType; + + private String key; + + private String name; + + private String keyAlias; + + private Long domainId; + + private Long elementId; + + private Object value; + + /** + * CUSTOM: the value is specified by the user + * SEMANTIC: the value of element + * FORWARD: only forward + */ + public enum ParamType { + CUSTOM, SEMANTIC, FORWARD + } + + public enum OptionType { + REQUIRED, OPTIONAL + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBase.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBase.java index a9aec8bf4..dd26c2ff0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBase.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBase.java @@ -1,22 +1,14 @@ package com.tencent.supersonic.chat.query.plugin; +import com.google.common.collect.Lists; import lombok.Data; - -import java.util.HashMap; -import java.util.Map; +import java.util.List; @Data public class WebBase { private String url; - //key, id of schema element - private Map params = new HashMap<>(); - - //key, value of shcema element - private Map valueParams = new HashMap<>(); - - //only forward - private Map forwardParam = new HashMap<>(); + private List paramOptions = Lists.newArrayList(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBaseResult.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBaseResult.java new file mode 100644 index 000000000..8ba6cde74 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/WebBaseResult.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.chat.query.plugin; + +import com.google.common.collect.Lists; +import lombok.Data; +import java.util.List; + +@Data +public class WebBaseResult { + + private String url; + + private List params = Lists.newArrayList(); + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/dsl/DSLQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/dsl/DSLQuery.java index 16d1de9d9..a690c32c6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/dsl/DSLQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/dsl/DSLQuery.java @@ -62,8 +62,7 @@ public class DSLQuery extends PluginSemanticQuery { @Override public QueryResult execute(User user) { - PluginParseResult functionCallParseResult = (PluginParseResult) parseInfo.getProperties() - .get(Constants.CONTEXT); + PluginParseResult functionCallParseResult =JsonUtil.toObject(JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)),PluginParseResult.class); Long domainId = parseInfo.getDomainId(); LLMResp llmResp = requestLLM(functionCallParseResult, domainId); if (Objects.isNull(llmResp)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java index 7c981d808..f28b796fd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageQuery.java @@ -1,15 +1,19 @@ package com.tencent.supersonic.chat.query.plugin.webpage; +import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.*; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.config.ChatConfigRich; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; import com.tencent.supersonic.chat.plugin.Plugin; +import com.tencent.supersonic.chat.plugin.PluginParseResult; import com.tencent.supersonic.chat.query.QueryManager; +import com.tencent.supersonic.chat.query.plugin.ParamOption; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.WebBase; +import com.tencent.supersonic.chat.query.plugin.WebBaseResult; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.Constants; @@ -18,10 +22,9 @@ import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; + +import java.util.*; +import java.util.stream.Collectors; @Slf4j @Component @@ -44,13 +47,14 @@ public class WebPageQuery extends PluginSemanticQuery { QueryResult queryResult = new QueryResult(); queryResult.setQueryMode(QUERY_MODE); Map properties = parseInfo.getProperties(); - Plugin plugin = (Plugin) properties.get(Constants.CONTEXT); - WebPageResponse webPageResponse = buildResponse(plugin); + PluginParseResult pluginParseResult = JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)), PluginParseResult.class); + WebPageResponse webPageResponse = buildResponse(pluginParseResult.getPlugin()); queryResult.setResponse(webPageResponse); if (parseInfo.getDomainId() != null && parseInfo.getDomainId() > 0 - && parseInfo.getEntity() != null && parseInfo.getEntity() > 0) { - ChatConfigRich chatConfigRichResp = configService.getConfigRichInfo(parseInfo.getDomainId()); - updateSemanticParse(chatConfigRichResp, parseInfo.getEntity()); + && parseInfo.getEntity() != null && Objects.nonNull(parseInfo.getEntity().getId()) + && parseInfo.getEntity().getId() > 0) { + ChatConfigRichResp chatConfigRichResp = configService.getConfigRichInfo(parseInfo.getDomainId()); + updateSemanticParse(chatConfigRichResp); EntityInfo entityInfo = ContextUtils.getBean(SemanticService.class).getEntityInfo(parseInfo, user); queryResult.setEntityInfo(entityInfo); } else { @@ -60,8 +64,7 @@ public class WebPageQuery extends PluginSemanticQuery { return queryResult; } - private void updateSemanticParse(ChatConfigRich chatConfigRichResp, Long entityId) { - parseInfo.setEntity(entityId); + private void updateSemanticParse(ChatConfigRichResp chatConfigRichResp) { SchemaElement domain = new SchemaElement(); domain.setId(chatConfigRichResp.getDomainId()); domain.setName(chatConfigRichResp.getDomainName()); @@ -74,35 +77,43 @@ public class WebPageQuery extends PluginSemanticQuery { webPageResponse.setPluginId(plugin.getId()); webPageResponse.setPluginType(plugin.getType()); WebBase webPage = JsonUtil.toObject(plugin.getConfig(), WebBase.class); - fillWebPage(webPage); - webPageResponse.setWebPage(webPage); + WebBaseResult webBaseResult = buildWebPageResult(webPage); + webPageResponse.setWebPage(webBaseResult); return webPageResponse; } - private void fillWebPage(WebBase webPage) { - List schemaElementMatchList = parseInfo.getElementMatches(); + private WebBaseResult buildWebPageResult(WebBase webPage) { + WebBaseResult webBaseResult = new WebBaseResult(); + webBaseResult.setUrl(webPage.getUrl()); + Map elementValueMap = getElementMap(); + if (!CollectionUtils.isEmpty(webPage.getParamOptions()) && !CollectionUtils.isEmpty(elementValueMap)) { + for (ParamOption paramOption : webPage.getParamOptions()) { + if (!ParamOption.ParamType.SEMANTIC.equals(paramOption.getParamType())) { + continue; + } + String elementId = String.valueOf(paramOption.getElementId()); + Object elementValue = elementValueMap.get(elementId); + paramOption.setValue(elementValue); + } + } + webBaseResult.setParams(webPage.getParamOptions()); + return webBaseResult; + } + + private Map getElementMap() { Map elementValueMap = new HashMap<>(); - if (!CollectionUtils.isEmpty(schemaElementMatchList) && !CollectionUtils.isEmpty(webPage.getParams()) ) { + List schemaElementMatchList = parseInfo.getElementMatches(); + if (!CollectionUtils.isEmpty(schemaElementMatchList)) { schemaElementMatchList.stream() .filter(schemaElementMatch -> - SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType())) + SchemaElementType.VALUE.equals(schemaElementMatch.getElement().getType()) + || SchemaElementType.ID.equals(schemaElementMatch.getElement().getType())) .sorted(Comparator.comparingDouble(SchemaElementMatch::getSimilarity)) .forEach(schemaElementMatch -> elementValueMap.put(String.valueOf(schemaElementMatch.getElement().getId()), schemaElementMatch.getWord())); } - if (!CollectionUtils.isEmpty(parseInfo.getDimensionFilters())) { - parseInfo.getDimensionFilters().forEach( - filter -> elementValueMap.put(String.valueOf(filter.getElementID()), filter.getValue()) - ); - } - Map params = webPage.getParams(); - for (Map.Entry entry : params.entrySet()) { - String key = entry.getKey(); - String elementId = String.valueOf(entry.getValue()); - Object elementValue = elementValueMap.get(elementId); - webPage.getValueParams().put(key, elementValue); - } + return elementValueMap; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageResponse.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageResponse.java index 6c75a37f9..8a728ae91 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageResponse.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webpage/WebPageResponse.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.chat.query.plugin.webpage; import com.tencent.supersonic.chat.query.plugin.WebBase; +import com.tencent.supersonic.chat.query.plugin.WebBaseResult; import lombok.Data; import java.util.List; @@ -15,8 +16,8 @@ public class WebPageResponse { private String description; - private WebBase webPage; + private WebBaseResult webPage; - private List moreWebPage; + private List moreWebPage; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java index d9ebe1f95..427f7b5db 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/webservice/WebServiceQuery.java @@ -1,22 +1,28 @@ package com.tencent.supersonic.chat.query.plugin.webservice; +import com.alibaba.fastjson.JSON; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.plugin.PluginParseResult; import com.tencent.supersonic.chat.query.QueryManager; +import com.tencent.supersonic.chat.query.plugin.ParamOption; import com.tencent.supersonic.chat.query.plugin.PluginSemanticQuery; import com.tencent.supersonic.chat.query.plugin.WebBase; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.*; +import java.net.URI; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.parser.SqlParseException; +import org.springframework.http.*; import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; import java.util.HashMap; import java.util.Map; @@ -27,7 +33,7 @@ public class WebServiceQuery extends PluginSemanticQuery { public static String QUERY_MODE = "WEB_SERVICE"; - private S2ThreadContext s2ThreadContext; + private RestTemplate restTemplate; public WebServiceQuery() { QueryManager.register(this); @@ -43,13 +49,9 @@ public class WebServiceQuery extends PluginSemanticQuery { QueryResult queryResult = new QueryResult(); queryResult.setQueryMode(QUERY_MODE); Map properties = parseInfo.getProperties(); - PluginParseResult pluginParseResult = (PluginParseResult) properties.get(Constants.CONTEXT); + PluginParseResult pluginParseResult =JsonUtil.toObject(JsonUtil.toString(properties.get(Constants.CONTEXT)),PluginParseResult.class); WebServiceResponse webServiceResponse = buildResponse(pluginParseResult); - Object object = webServiceResponse.getResult(); - Map data=JsonUtil.toMap(JsonUtil.toString(object),String.class,Object.class); - queryResult.setQueryResults((List>) data.get("resultList")); - queryResult.setQueryColumns((List) data.get("columns")); - //queryResult.setResponse(webServiceResponse); + queryResult.setResponse(webServiceResponse); queryResult.setQueryState(QueryState.SUCCESS); parseInfo.setProperties(null); return queryResult; @@ -60,22 +62,22 @@ public class WebServiceQuery extends PluginSemanticQuery { Plugin plugin = pluginParseResult.getPlugin(); WebBase webBase = JsonUtil.toObject(plugin.getConfig(), WebBase.class); webServiceResponse.setWebBase(webBase); - //http todo - s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class); - String authHeader = s2ThreadContext.get().getToken(); - log.info("authHeader:{}", authHeader); + List paramOptions = webBase.getParamOptions(); + Map params = new HashMap<>(); + paramOptions.forEach(o -> params.put(o.getKey(), o.getValue())); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity entity = new HttpEntity<>(JSON.toJSONString(params), headers); + URI requestUrl = UriComponentsBuilder.fromHttpUrl(webBase.getUrl()).build().encode().toUri(); + ResponseEntity responseEntity = null; + Object objectResponse = null; + restTemplate = ContextUtils.getBean(RestTemplate.class); try { - Map headers = new HashMap<>(); - headers.put("Authorization", authHeader); - Map params = new HashMap<>(); - params.put("queryText", pluginParseResult.getRequest().getQueryText()); - HttpClientResult httpClientResult = HttpClientUtils.doGet(webBase.getUrl(), headers, params); - log.info(" response body:{}", httpClientResult.getContent()); - Map result = JsonUtil.toMap(JsonUtil.toString(httpClientResult.getContent()), String.class, Object.class); - log.info(" result:{}", result); - Map data = JsonUtil.toMap(JsonUtil.toString(result.get("data")), String.class, Object.class); - log.info(" data:{}", data); - webServiceResponse.setResult(data); + responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity, Object.class); + objectResponse = responseEntity.getBody(); + log.info("objectResponse:{}", objectResponse); + Map response = JsonUtil.objectToMap(objectResponse); + webServiceResponse.setResult(response); } catch (Exception e) { log.info("Exception:{}", e.getMessage()); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/QueryMatcher.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/QueryMatcher.java index ce74dd3a7..f10a524a7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/QueryMatcher.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/QueryMatcher.java @@ -26,8 +26,6 @@ public class QueryMatcher { private boolean supportOrderBy; private List orderByTypes = Arrays.asList(AggregateTypeEnum.MAX, AggregateTypeEnum.MIN, AggregateTypeEnum.TOPN); - private Long FREQUENCY = 9999999L; - private double SIMILARITY = 1.0; public QueryMatcher() { for (SchemaElementType type : SchemaElementType.values()) { @@ -52,11 +50,10 @@ public class QueryMatcher { * @return a list of all matched schema elements, * empty list if no matches can be found */ - public List match(List candidateElementMatches, QueryFilters queryFilters) { + public List match(List candidateElementMatches) { List elementMatches = new ArrayList<>(); - List schemaElementMatchWithQueryFilter = addSchemaElementMatch(candidateElementMatches, queryFilters); HashMap schemaElementTypeCount = new HashMap<>(); - for (SchemaElementMatch schemaElementMatch : schemaElementMatchWithQueryFilter) { + for (SchemaElementMatch schemaElementMatch : candidateElementMatches) { SchemaElementType schemaElementType = schemaElementMatch.getElement().getType(); if (schemaElementTypeCount.containsKey(schemaElementType)) { schemaElementTypeCount.put(schemaElementType, schemaElementTypeCount.get(schemaElementType) + 1); @@ -75,7 +72,7 @@ public class QueryMatcher { } // add element match if its element type is not declared as unused - for (SchemaElementMatch elementMatch : schemaElementMatchWithQueryFilter) { + for (SchemaElementMatch elementMatch : candidateElementMatches) { QueryMatchOption elementOption = elementOptionMap.get(elementMatch.getElement().getType()); if (Objects.nonNull(elementOption) && !elementOption.getSchemaElementOption() .equals(QueryMatchOption.OptionType.UNUSED)) { @@ -86,32 +83,6 @@ public class QueryMatcher { return elementMatches; } - private List addSchemaElementMatch(List candidateElementMatches, QueryFilters queryFilter) { - List schemaElementMatchWithQueryFilter = new ArrayList<>(candidateElementMatches); - if (queryFilter == null || CollectionUtils.isEmpty(queryFilter.getFilters())) { - return schemaElementMatchWithQueryFilter; - } - QueryMatchOption queryMatchOption = elementOptionMap.get(SchemaElementType.VALUE); - if (queryMatchOption != null && QueryMatchOption.OptionType.REQUIRED.equals(queryMatchOption.getSchemaElementOption())) { - for (QueryFilter filter : queryFilter.getFilters()) { - SchemaElement element = SchemaElement.builder() - .id(filter.getElementID()) - .name(String.valueOf(filter.getValue())) - .type(SchemaElementType.VALUE) - .build(); - SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder() - .element(element) - .frequency(FREQUENCY) - .word(String.valueOf(filter.getValue())) - .similarity(SIMILARITY) - .detectWord(Constants.EMPTY) - .build(); - schemaElementMatchWithQueryFilter.add(schemaElementMatch); - } - } - return schemaElementMatchWithQueryFilter; - } - private int getCount(HashMap schemaElementTypeCount, SchemaElementType schemaElementType) { if (schemaElementTypeCount.containsKey(schemaElementType)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java index 173eb2b6e..5a105b436 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java @@ -9,9 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.config.ChatConfigRich; import com.tencent.supersonic.chat.query.QueryManager; -import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.QueryReqBuilder; @@ -24,6 +22,7 @@ import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; import java.io.Serializable; import java.util.*; @@ -42,33 +41,51 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { public List match(List candidateElementMatches, QueryContext queryCtx) { - return queryMatcher.match(candidateElementMatches, queryCtx.getRequest().getQueryFilters()); + return queryMatcher.match(candidateElementMatches); } - public void fillParseInfo(Long domainId, ChatContext chatContext){ + public void fillParseInfo(Long domainId, ChatContext chatContext) { parseInfo.setQueryMode(getQueryMode()); - ConfigService configService = ContextUtils.getBean(ConfigService.class); - ChatConfigRich chatConfig = configService.getConfigRichInfo(domainId); - SemanticService schemaService = ContextUtils.getBean(SemanticService.class); DomainSchema domainSchema = schemaService.getDomainSchema(domainId); - fillSchemaElement(parseInfo, domainSchema, chatConfig); + fillSchemaElement(parseInfo, domainSchema); // inherit date info from context - if (parseInfo.getDateInfo() == null && chatContext.getParseInfo().getDateInfo() != null) { + if (parseInfo.getDateInfo() == null && chatContext.getParseInfo().getDateInfo() != null + && isSameQueryMode(getQueryMode(), chatContext.getParseInfo().getQueryMode())) { + log.info("inherit date info from context"); parseInfo.setDateInfo(chatContext.getParseInfo().getDateInfo()); } } - private void fillSchemaElement(SemanticParseInfo parseInfo, DomainSchema domainSchema, ChatConfigRich chaConfigRich) { + public boolean isSameQueryMode(String queryModeQuery, String queryModeChat) { + if (Strings.isNotEmpty(queryModeQuery) && Strings.isNotEmpty(queryModeChat)) { + return QueryManager.isEntityQuery(queryModeQuery) && QueryManager.isEntityQuery(queryModeChat) + || QueryManager.isMetricQuery(queryModeQuery) && QueryManager.isMetricQuery(queryModeChat); + } + return true; + } + + private void fillSchemaElement(SemanticParseInfo parseInfo, DomainSchema domainSchema) { parseInfo.setDomain(domainSchema.getDomain()); Map> dim2Values = new HashMap<>(); + Map> id2Values = new HashMap<>(); + for (SchemaElementMatch schemaMatch : parseInfo.getElementMatches()) { SchemaElement element = schemaMatch.getElement(); switch (element.getType()) { case ID: + SchemaElement entityElement = domainSchema.getElement(SchemaElementType.ENTITY, element.getId()); + if (entityElement != null) { + if (id2Values.containsKey(element.getId())) { + id2Values.get(element.getId()).add(schemaMatch); + } else { + id2Values.put(element.getId(), new ArrayList<>(Arrays.asList(schemaMatch))); + } + } + break; case VALUE: SchemaElement dimElement = domainSchema.getElement(SchemaElementType.DIMENSION, element.getId()); if (dimElement != null) { @@ -85,10 +102,41 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { case METRIC: parseInfo.getMetrics().add(element); break; + case ENTITY: + parseInfo.setEntity(element); + break; default: } } + if (!id2Values.isEmpty()) { + for (Map.Entry> entry : id2Values.entrySet()) { + SchemaElement entity = domainSchema.getElement(SchemaElementType.ENTITY, entry.getKey()); + + if (entry.getValue().size() == 1) { + SchemaElementMatch schemaMatch = entry.getValue().get(0); + QueryFilter dimensionFilter = new QueryFilter(); + dimensionFilter.setValue(schemaMatch.getWord()); + dimensionFilter.setBizName(entity.getBizName()); + dimensionFilter.setName(entity.getName()); + dimensionFilter.setOperator(FilterOperatorEnum.EQUALS); + dimensionFilter.setElementID(schemaMatch.getElement().getId()); + parseInfo.getDimensionFilters().add(dimensionFilter); + parseInfo.setEntity(domainSchema.getEntity()); + } else { + QueryFilter dimensionFilter = new QueryFilter(); + List vals = new ArrayList<>(); + entry.getValue().stream().forEach(i -> vals.add(i.getWord())); + dimensionFilter.setValue(vals); + dimensionFilter.setBizName(entity.getBizName()); + dimensionFilter.setName(entity.getName()); + dimensionFilter.setOperator(FilterOperatorEnum.IN); + dimensionFilter.setElementID(entry.getKey()); + parseInfo.getDimensionFilters().add(dimensionFilter); + } + } + } + if (!dim2Values.isEmpty()) { for (Map.Entry> entry : dim2Values.entrySet()) { SchemaElement dimension = domainSchema.getElement(SchemaElementType.DIMENSION, entry.getKey()); @@ -102,7 +150,7 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { dimensionFilter.setOperator(FilterOperatorEnum.EQUALS); dimensionFilter.setElementID(schemaMatch.getElement().getId()); parseInfo.getDimensionFilters().add(dimensionFilter); - setEntityId(schemaMatch.getWord(), chaConfigRich, parseInfo); + parseInfo.setEntity(domainSchema.getEntity()); } else { QueryFilter dimensionFilter = new QueryFilter(); List vals = new ArrayList<>(); @@ -118,16 +166,6 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { } } - public void setEntityId(String value, ChatConfigRich chaConfigRichDesc, - SemanticParseInfo semanticParseInfo) { - if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null - && chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) { - SchemaElement dimSchemaResp = chaConfigRichDesc.getChatDetailRichConfig().getEntity().getDimItem(); - if (Objects.nonNull(dimSchemaResp) && StringUtils.isNumeric(value)) { - semanticParseInfo.setEntity(Long.valueOf(value)); - } - } - } @Override public QueryResult execute(User user) { @@ -202,12 +240,13 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { return parseInfo; } + @Override public void setParseInfo(SemanticParseInfo parseInfo) { this.parseInfo = parseInfo; } public static List resolve(List candidateElementMatches, - QueryContext queryContext) { + QueryContext queryContext) { List matchedQueries = new ArrayList<>(); for (RuleSemanticQuery semanticQuery : QueryManager.getRuleQueries()) { List matches = semanticQuery.match(candidateElementMatches, queryContext); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityDetailQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityDetailQuery.java index 6ca497420..e9c4b699d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityDetailQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityDetailQuery.java @@ -14,7 +14,7 @@ public class EntityDetailQuery extends EntitySemanticQuery { public EntityDetailQuery() { super(); queryMatcher.addOption(DIMENSION, REQUIRED, AT_LEAST, 1) - .addOption(VALUE, REQUIRED, AT_LEAST, 1); + .addOption(ID, REQUIRED, AT_LEAST, 1); } @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityFilterQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityFilterQuery.java index 5899d9ccb..a7866eac6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityFilterQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityFilterQuery.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.chat.query.rule.entity; import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.*; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -16,7 +16,8 @@ public class EntityFilterQuery extends EntityListQuery { public EntityFilterQuery() { super(); - queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1); + queryMatcher.addOption(VALUE, OPTIONAL, AT_LEAST, 0); + queryMatcher.addOption(ID, OPTIONAL, AT_LEAST, 0); } @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityListQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityListQuery.java index 45398e1aa..c5ddcee52 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityListQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityListQuery.java @@ -1,37 +1,43 @@ package com.tencent.supersonic.chat.query.rule.entity; import com.tencent.supersonic.chat.api.pojo.ChatContext; +import com.tencent.supersonic.chat.api.pojo.DomainSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.config.ChatConfigRich; -import com.tencent.supersonic.chat.config.ChatDefaultRichConfig; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; import com.tencent.supersonic.chat.service.ConfigService; +import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.util.ContextUtils; import java.util.LinkedHashSet; +import java.util.Objects; import java.util.Set; -public abstract class EntityListQuery extends EntitySemanticQuery{ +public abstract class EntityListQuery extends EntitySemanticQuery { @Override - public void fillParseInfo(Long domainId, ChatContext chatContext){ + public void fillParseInfo(Long domainId, ChatContext chatContext) { super.fillParseInfo(domainId, chatContext); this.addEntityDetailAndOrderByMetric(parseInfo); } private void addEntityDetailAndOrderByMetric(SemanticParseInfo parseInfo) { - if (parseInfo.getDomainId() > 0L) { + Long domainId = parseInfo.getDomainId(); + if (Objects.nonNull(domainId) && domainId > 0L) { ConfigService configService = ContextUtils.getBean(ConfigService.class); - ChatConfigRich chaConfigRichDesc = configService.getConfigRichInfo( - parseInfo.getDomainId()); + ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(parseInfo.getDomainId()); + SemanticService schemaService = ContextUtils.getBean(SemanticService.class); + DomainSchema domainSchema = schemaService.getDomainSchema(domainId); + if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null - && chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) { + && Objects.nonNull(domainSchema) && Objects.nonNull(domainSchema.getEntity())) { Set dimensions = new LinkedHashSet(); Set metrics = new LinkedHashSet(); Set orders = new LinkedHashSet(); - ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig(); + ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig(); if (chatDefaultConfig != null) { chatDefaultConfig.getMetrics().stream() .forEach(metric -> { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java index 50981348e..d5813aa6b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java @@ -4,9 +4,9 @@ import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.config.ChatConfigResp; -import com.tencent.supersonic.chat.config.ChatConfigRich; -import com.tencent.supersonic.chat.config.ChatDefaultRichConfig; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.common.pojo.DateConf; @@ -85,8 +85,8 @@ public abstract class EntitySemanticQuery extends RuleSemanticQuery { parseInfo.setLimit(ENTITY_MAX_RESULTS); if (parseInfo.getDateInfo() == null) { ConfigService configService = ContextUtils.getBean(ConfigService.class); - ChatConfigRich chatConfig = configService.getConfigRichInfo(parseInfo.getDomainId()); - ChatDefaultRichConfig defaultConfig = chatConfig.getChatDetailRichConfig().getChatDefaultConfig(); + ChatConfigRichResp chatConfig = configService.getConfigRichInfo(parseInfo.getDomainId()); + ChatDefaultRichConfigResp defaultConfig = chatConfig.getChatDetailRichConfig().getChatDefaultConfig(); int unit = 1; if (Objects.nonNull(defaultConfig) && Objects.nonNull(defaultConfig.getUnit())) { @@ -94,7 +94,7 @@ public abstract class EntitySemanticQuery extends RuleSemanticQuery { } String date = LocalDate.now().plusDays(-unit).toString(); DateConf dateInfo = new DateConf(); - dateInfo.setDateMode(DateConf.DateMode.BETWEEN_CONTINUOUS); + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); dateInfo.setStartDate(date); dateInfo.setEndDate(date); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityTopNQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityTopNQuery.java deleted file mode 100644 index ae0d34e61..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntityTopNQuery.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.tencent.supersonic.chat.query.rule.entity; - -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; - -import org.springframework.stereotype.Component; - -@Component -public class EntityTopNQuery extends EntityListQuery { - - public static final String QUERY_MODE = "ENTITY_LIST_TOPN"; - - public EntityTopNQuery() { - super(); - queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1) - .setSupportOrderBy(true); - } - - @Override - public String getQueryMode() { - return QUERY_MODE; - } - -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricDomainQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricDomainQuery.java index badc4aa2b..4cf92ba8f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricDomainQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricDomainQuery.java @@ -1,17 +1,12 @@ package com.tencent.supersonic.chat.query.rule.metric; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.service.SemanticService; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import java.util.Objects; -import org.springframework.stereotype.Component; - -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DOMAIN; import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.*; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import org.springframework.stereotype.Component; @Component public class MetricDomainQuery extends MetricSemanticQuery { @@ -31,14 +26,7 @@ public class MetricDomainQuery extends MetricSemanticQuery { @Override public QueryResult execute(User user) { QueryResult queryResult = super.execute(user); - if (!Objects.isNull(queryResult)) { - QueryResultWithSchemaResp queryResp = new QueryResultWithSchemaResp(); - queryResp.setColumns(queryResult.getQueryColumns()); - queryResp.setResultList(queryResult.getQueryResults()); - AggregateInfo aggregateInfo = ContextUtils.getBean(SemanticService.class) - .getAggregateInfo(user, parseInfo, queryResp); - queryResult.setAggregateInfo(aggregateInfo); - } + fillAggregateInfo(user, queryResult); return queryResult; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java new file mode 100644 index 000000000..3cd998ef4 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java @@ -0,0 +1,92 @@ +package com.tencent.supersonic.chat.query.rule.metric; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; +import com.tencent.supersonic.semantic.api.query.pojo.Filter; +import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; +import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; + +@Slf4j +@Component +public class MetricEntityQuery extends MetricSemanticQuery { + + public static final String QUERY_MODE = "METRIC_ENTITY"; + + public MetricEntityQuery() { + super(); + queryMatcher.addOption(ID, REQUIRED, AT_LEAST, 1) + .addOption(ENTITY, REQUIRED, AT_LEAST, 1); + } + + @Override + public String getQueryMode() { + return QUERY_MODE; + } + + @Override + public QueryResult execute(User user) { + if (!isMultiStructQuery()) { + QueryResult queryResult = super.execute(user); + fillAggregateInfo(user, queryResult); + return queryResult; + } + return super.multiStructExecute(user); + } + + protected boolean isMultiStructQuery() { + Set filterBizName = new HashSet<>(); + parseInfo.getDimensionFilters().stream() + .filter(filter -> filter.getElementID() != null) + .forEach(filter -> filterBizName.add(filter.getBizName())); + return filterBizName.size() > 1; + } + + @Override + protected QueryStructReq convertQueryStruct() { + QueryStructReq queryStructReq = super.convertQueryStruct(); + addDimension(queryStructReq, true); + return queryStructReq; + } + + @Override + protected QueryMultiStructReq convertQueryMultiStruct() { + QueryMultiStructReq queryMultiStructReq = super.convertQueryMultiStruct(); + for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { + addDimension(queryStructReq, false); + } + return queryMultiStructReq; + } + + private void addDimension(QueryStructReq queryStructReq, boolean onlyOperateInFilter) { + if (!queryStructReq.getDimensionFilters().isEmpty()) { + List dimensions = queryStructReq.getGroups(); + log.info("addDimension before [{}]", queryStructReq.getGroups()); + List filters = new ArrayList<>(queryStructReq.getDimensionFilters()); + if (onlyOperateInFilter) { + filters = filters.stream().filter(filter + -> filter.getOperator().equals(FilterOperatorEnum.IN)).collect(Collectors.toList()); + } + filters.forEach(d -> { + if (!dimensions.contains(d.getBizName())) { + dimensions.add(d.getBizName()); + }}); + queryStructReq.setGroups(dimensions); + log.info("addDimension after [{}]", queryStructReq.getGroups()); + } + } + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java index bb6125f47..213397c3e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java @@ -1,23 +1,22 @@ package com.tencent.supersonic.chat.query.rule.metric; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; + import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.service.SemanticService; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import java.util.*; -import java.util.stream.Collectors; -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.*; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.OPTIONAL; @Slf4j @Component @@ -27,8 +26,7 @@ public class MetricFilterQuery extends MetricSemanticQuery { public MetricFilterQuery() { super(); - queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1) - .addOption(ENTITY, OPTIONAL, AT_MOST, 1); + queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1); } @Override @@ -40,14 +38,7 @@ public class MetricFilterQuery extends MetricSemanticQuery { public QueryResult execute(User user) { if (!isMultiStructQuery()) { QueryResult queryResult = super.execute(user); - if (Objects.nonNull(queryResult)) { - QueryResultWithSchemaResp queryResp = new QueryResultWithSchemaResp(); - queryResp.setColumns(queryResult.getQueryColumns()); - queryResp.setResultList(queryResult.getQueryResults()); - AggregateInfo aggregateInfo = ContextUtils.getBean(SemanticService.class) - .getAggregateInfo(user,parseInfo,queryResp); - queryResult.setAggregateInfo(aggregateInfo); - } + fillAggregateInfo(user, queryResult); return queryResult; } return super.multiStructExecute(user); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java index 177123be8..dfcd8898d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricSemanticQuery.java @@ -1,27 +1,32 @@ package com.tencent.supersonic.chat.query.rule.metric; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; -import com.tencent.supersonic.chat.config.ChatConfigResp; -import com.tencent.supersonic.chat.config.ChatConfigRich; -import com.tencent.supersonic.chat.config.ChatDefaultRichConfig; +import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; +import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.chat.service.ConfigService; +import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.util.ContextUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - +import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import java.time.LocalDate; import java.util.ArrayList; import java.util.List; import java.util.Objects; - -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; @Slf4j public abstract class MetricSemanticQuery extends RuleSemanticQuery { @@ -78,23 +83,30 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { } @Override - public void fillParseInfo(Long domainId, ChatContext chatContext){ + public void fillParseInfo(Long domainId, ChatContext chatContext) { super.fillParseInfo(domainId, chatContext); parseInfo.setLimit(METRIC_MAX_RESULTS); if (parseInfo.getDateInfo() == null) { ConfigService configService = ContextUtils.getBean(ConfigService.class); - ChatConfigRich chatConfig = configService.getConfigRichInfo(parseInfo.getDomainId()); - ChatDefaultRichConfig defaultConfig = chatConfig.getChatAggRichConfig().getChatDefaultConfig(); - + ChatConfigRichResp chatConfig = configService.getConfigRichInfo(parseInfo.getDomainId()); + ChatDefaultRichConfigResp defaultConfig = chatConfig.getChatAggRichConfig().getChatDefaultConfig(); + DateConf dateInfo = new DateConf(); int unit = 1; if (Objects.nonNull(defaultConfig) && Objects.nonNull(defaultConfig.getUnit())) { unit = defaultConfig.getUnit(); } String startDate = LocalDate.now().plusDays(-unit).toString(); - String endDate = LocalDate.now().plusDays(-1).toString(); - DateConf dateInfo = new DateConf(); - dateInfo.setDateMode(DateConf.DateMode.BETWEEN_CONTINUOUS); + String endDate = startDate; + + if (ChatDefaultConfigReq.TimeMode.LAST.equals(defaultConfig.getTimeMode())) { + dateInfo.setDateMode(DateConf.DateMode.BETWEEN); + } else if (ChatDefaultConfigReq.TimeMode.RECENT.equals(defaultConfig.getTimeMode())) { + dateInfo.setDateMode(DateConf.DateMode.RECENT); + endDate = LocalDate.now().plusDays(-1).toString(); + } + dateInfo.setUnit(unit); + dateInfo.setPeriod(defaultConfig.getPeriod()); dateInfo.setStartDate(startDate); dateInfo.setEndDate(endDate); @@ -102,4 +114,15 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { } } + public void fillAggregateInfo(User user, QueryResult queryResult) { + if (Objects.nonNull(queryResult)) { + QueryResultWithSchemaResp queryResp = new QueryResultWithSchemaResp(); + queryResp.setColumns(queryResult.getQueryColumns()); + queryResp.setResultList(queryResult.getQueryResults()); + AggregateInfo aggregateInfo = ContextUtils.getBean(SemanticService.class) + .getAggregateInfo(user, parseInfo, queryResp); + queryResult.setAggregateInfo(aggregateInfo); + } + } + } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java index 196e8e70a..01861694b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricTopNQuery.java @@ -54,7 +54,7 @@ public class MetricTopNQuery extends MetricSemanticQuery { super.fillParseInfo(domainId, chatContext); parseInfo.setLimit(ORDERBY_MAX_RESULTS); - parseInfo.setBonus(2.0); + parseInfo.setScore(2.0); parseInfo.setAggType(AggregateTypeEnum.SUM); SchemaElement metric = parseInfo.getMetrics().iterator().next(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java index d1ece0e2f..029faeb26 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatConfigController.java @@ -4,7 +4,11 @@ import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.api.component.SemanticLayer; -import com.tencent.supersonic.chat.config.*; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; @@ -65,12 +69,12 @@ public class ChatConfigController { @GetMapping("/richDesc/{domainId}") - public ChatConfigRich getDomainExtendRichInfo(@PathVariable("domainId") Long domainId) { + public ChatConfigRichResp getDomainExtendRichInfo(@PathVariable("domainId") Long domainId) { return configService.getConfigRichInfo(domainId); } @GetMapping("/richDesc/all") - public List getAllChatRichConfig() { + public List getAllChatRichConfig() { return configService.getAllChatRichConfig(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java index 85b0ea412..18f5875f5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java @@ -4,7 +4,7 @@ package com.tencent.supersonic.chat.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; -import com.tencent.supersonic.chat.api.pojo.response.QueryResponse; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.service.ChatService; import java.util.List; @@ -68,10 +68,10 @@ public class ChatController { } @PostMapping("/pageQueryInfo") - public PageInfo pageQueryInfo(@RequestBody PageQueryInfoReq pageQueryInfoCommand, - @RequestParam(value = "chatId") long chatId, - HttpServletRequest request, - HttpServletResponse response) { + public PageInfo pageQueryInfo(@RequestBody PageQueryInfoReq pageQueryInfoCommand, + @RequestParam(value = "chatId") long chatId, + HttpServletRequest request, + HttpServletResponse response) { pageQueryInfoCommand.setUserName(UserHolder.findUser(request, response).getName()); return chatService.queryInfo(pageQueryInfoCommand, chatId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java index 0356e27ef..976f60f1d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatQueryController.java @@ -2,8 +2,9 @@ package com.tencent.supersonic.chat.rest; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; -import com.tencent.supersonic.chat.api.pojo.request.QueryDataRequest; +import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; import com.tencent.supersonic.chat.service.QueryService; import com.tencent.supersonic.chat.service.SearchService; import javax.servlet.http.HttpServletRequest; @@ -31,28 +32,42 @@ public class ChatQueryController { @PostMapping("search") - public Object search(@RequestBody QueryRequest queryCtx, HttpServletRequest request, + public Object search(@RequestBody QueryReq queryCtx, HttpServletRequest request, HttpServletResponse response) { queryCtx.setUser(UserHolder.findUser(request, response)); return searchService.search(queryCtx); } @PostMapping("query") - public Object query(@RequestBody QueryRequest queryCtx, HttpServletRequest request, HttpServletResponse response) + public Object query(@RequestBody QueryReq queryCtx, HttpServletRequest request, HttpServletResponse response) throws Exception { queryCtx.setUser(UserHolder.findUser(request, response)); return queryService.executeQuery(queryCtx); } + @PostMapping("parse") + public Object parse(@RequestBody QueryReq queryCtx, HttpServletRequest request, HttpServletResponse response) + throws Exception { + queryCtx.setUser(UserHolder.findUser(request, response)); + return queryService.performParsing(queryCtx); + } + + @PostMapping("execute") + public Object execute(@RequestBody ExecuteQueryReq queryCtx, HttpServletRequest request, HttpServletResponse response) + throws Exception { + queryCtx.setUser(UserHolder.findUser(request, response)); + return queryService.performExecution(queryCtx); + } + @PostMapping("queryContext") - public Object queryContext(@RequestBody QueryRequest queryCtx, HttpServletRequest request, + public Object queryContext(@RequestBody QueryReq queryCtx, HttpServletRequest request, HttpServletResponse response) throws Exception { queryCtx.setUser(UserHolder.findUser(request, response)); return queryService.queryContext(queryCtx); } @PostMapping("queryData") - public Object queryData(@RequestBody QueryDataRequest queryData, HttpServletRequest request, HttpServletResponse response) + public Object queryData(@RequestBody QueryDataReq queryData, HttpServletRequest request, HttpServletResponse response) throws Exception { return queryService.executeDirectQuery(queryData, UserHolder.findUser(request, response)); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/PluginController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/PluginController.java index c47444f6e..94ee18ade 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/PluginController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/PluginController.java @@ -50,7 +50,7 @@ public class PluginController { } @PostMapping("/query") - List query(PluginQueryReq pluginQueryReq) { + List query(@RequestBody PluginQueryReq pluginQueryReq) { return pluginService.queryWithAuthCheck(pluginQueryReq); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java index 7118c20b6..ea58046fb 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/RecommendController.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.rest; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; -import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestion; -import com.tencent.supersonic.chat.api.pojo.response.RecommendResponse; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp; +import com.tencent.supersonic.chat.api.pojo.response.RecommendResp; import com.tencent.supersonic.chat.service.RecommendService; import javax.servlet.http.HttpServletRequest; @@ -25,31 +25,31 @@ public class RecommendController { private RecommendService recommendService; @GetMapping("recommend/{domainId}") - public RecommendResponse recommend(@PathVariable("domainId") Long domainId, - @RequestParam(value = "limit", required = false) Long limit, - HttpServletRequest request, - HttpServletResponse response) { - QueryRequest queryCtx = new QueryRequest(); + public RecommendResp recommend(@PathVariable("domainId") Long domainId, + @RequestParam(value = "limit", required = false) Long limit, + HttpServletRequest request, + HttpServletResponse response) { + QueryReq queryCtx = new QueryReq(); queryCtx.setUser(UserHolder.findUser(request, response)); queryCtx.setDomainId(domainId); return recommendService.recommend(queryCtx, limit); } @GetMapping("recommend/metric/{domainId}") - public RecommendResponse recommendMetricMode(@PathVariable("domainId") Long domainId, - @RequestParam(value = "limit", required = false) Long limit, - HttpServletRequest request, - HttpServletResponse response) { - QueryRequest queryCtx = new QueryRequest(); + public RecommendResp recommendMetricMode(@PathVariable("domainId") Long domainId, + @RequestParam(value = "limit", required = false) Long limit, + HttpServletRequest request, + HttpServletResponse response) { + QueryReq queryCtx = new QueryReq(); queryCtx.setUser(UserHolder.findUser(request, response)); queryCtx.setDomainId(domainId); return recommendService.recommendMetricMode(queryCtx, limit); } @GetMapping("recommend/question") - public List recommendQuestion(@RequestParam(value = "domainId", required = false) Long domainId, - HttpServletRequest request, - HttpServletResponse response) { + public List recommendQuestion(@RequestParam(value = "domainId", required = false) Long domainId, + HttpServletRequest request, + HttpServletResponse response) { return recommendService.recommendQuestion(domainId); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java index 590eec797..d7dc78917 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatService.java @@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; -import com.tencent.supersonic.chat.api.pojo.response.QueryResponse; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import java.util.List; @@ -25,8 +25,6 @@ public interface ChatService { public void updateContext(ChatContext chatCtx); - public void updateContext(ChatContext chatCtx, QueryContext queryCtx, SemanticParseInfo semanticParseInfo); - public void switchContext(ChatContext chatCtx); public Boolean addChat(User user, String chatName); @@ -41,9 +39,9 @@ public interface ChatService { Boolean deleteChat(Long chatId, String userName); - PageInfo queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId); + PageInfo queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId); - public void addQuery(QueryResult queryResult, QueryContext queryContext, ChatContext chatCtx); + public void addQuery(QueryResult queryResult, ChatContext chatCtx); public ChatQueryDO getLastQuery(long chatId); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java index d211d2580..0b1aa8b8f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ConfigService.java @@ -2,7 +2,11 @@ package com.tencent.supersonic.chat.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.config.*; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; import java.util.List; @@ -14,9 +18,9 @@ public interface ConfigService { List search(ChatConfigFilter filter, User user); - ChatConfigRich getConfigRichInfo(Long domainId); + ChatConfigRichResp getConfigRichInfo(Long domainId); ChatConfigResp fetchConfigByDomainId(Long domainId); - List getAllChatRichConfig(); + List getAllChatRichConfig(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/QueryService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/QueryService.java index b13d5d893..be45d6816 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/QueryService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/QueryService.java @@ -2,9 +2,11 @@ package com.tencent.supersonic.chat.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.request.QueryDataRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; import org.apache.calcite.sql.parser.SqlParseException; /*** @@ -12,9 +14,14 @@ import org.apache.calcite.sql.parser.SqlParseException; */ public interface QueryService { - QueryResult executeQuery(QueryRequest queryCtx) throws Exception; + ParseResp performParsing(QueryReq queryReq); - SemanticParseInfo queryContext(QueryRequest queryCtx); + QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception; + + QueryResult executeQuery(QueryReq queryReq) throws Exception; + + SemanticParseInfo queryContext(QueryReq queryReq); + + QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException; - QueryResult executeDirectQuery(QueryDataRequest queryData, User user) throws SqlParseException; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/RecommendService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/RecommendService.java index 8e8aff45c..ce85f7da2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/RecommendService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/RecommendService.java @@ -1,9 +1,9 @@ package com.tencent.supersonic.chat.service; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; -import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestion; -import com.tencent.supersonic.chat.api.pojo.response.RecommendResponse; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp; +import com.tencent.supersonic.chat.api.pojo.response.RecommendResp; import java.util.List; @@ -12,9 +12,9 @@ import java.util.List; */ public interface RecommendService { - RecommendResponse recommend(QueryRequest queryCtx, Long limit); + RecommendResp recommend(QueryReq queryCtx, Long limit); - RecommendResponse recommendMetricMode(QueryRequest queryCtx, Long limit); + RecommendResp recommendMetricMode(QueryReq queryCtx, Long limit); - List recommendQuestion(Long domainId); + List recommendQuestion(Long domainId); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SearchService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SearchService.java index 2460ef1e6..8f36c0996 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SearchService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SearchService.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.chat.service; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.SearchResult; import java.util.List; @@ -9,6 +9,6 @@ import java.util.List; */ public interface SearchService { - List search(QueryRequest queryCtx); + List search(QueryReq queryCtx); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java index fdac19b67..d9a43b71f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java @@ -16,19 +16,20 @@ import com.tencent.supersonic.chat.api.component.SemanticLayer; import com.tencent.supersonic.chat.api.pojo.DomainSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq; +import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; import com.tencent.supersonic.chat.api.pojo.response.DataInfo; import com.tencent.supersonic.chat.api.pojo.response.DomainInfo; import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.MetricInfo; -import com.tencent.supersonic.chat.config.ChatAggConfig; -import com.tencent.supersonic.chat.config.ChatConfigResp; -import com.tencent.supersonic.chat.config.ChatConfigRich; -import com.tencent.supersonic.chat.config.ChatDefaultRichConfig; -import com.tencent.supersonic.chat.config.ChatDetailConfig; -import com.tencent.supersonic.chat.config.EntityRichInfo; -import com.tencent.supersonic.chat.config.ItemVisibility; +import com.tencent.supersonic.chat.config.AggregatorConfig; import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.DateConf; @@ -36,11 +37,13 @@ import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.RatioOverType; +import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.knowledge.service.SchemaService; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import java.text.DecimalFormat; import java.time.DayOfWeek; import java.time.LocalDate; import java.time.LocalDateTime; @@ -56,6 +59,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -70,6 +74,8 @@ public class SemanticService { private SchemaService schemaService; @Autowired private ConfigService configService; + @Autowired + private AggregatorConfig aggregatorConfig; private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer(); @@ -122,25 +128,30 @@ public class SemanticService { } public EntityInfo getEntityInfo(Long domain) { - ChatConfigRich chaConfigRichDesc = configService.getConfigRichInfo(domain); + ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(domain); if (Objects.isNull(chaConfigRichDesc) || Objects.isNull(chaConfigRichDesc.getChatDetailRichConfig())) { return new EntityInfo(); } return getEntityInfo(chaConfigRichDesc); } - private EntityInfo getEntityInfo(ChatConfigRich chaConfigRichDesc) { + private EntityInfo getEntityInfo(ChatConfigRichResp chaConfigRichDesc) { EntityInfo entityInfo = new EntityInfo(); - EntityRichInfo entityDesc = chaConfigRichDesc.getChatDetailRichConfig().getEntity(); - if (entityDesc != null && Objects.nonNull(chaConfigRichDesc.getDomainId())) { + Long domainId = chaConfigRichDesc.getDomainId(); + if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(domainId)) { + SemanticService schemaService = ContextUtils.getBean(SemanticService.class); + DomainSchema domainSchema = schemaService.getDomainSchema(domainId); + if (Objects.isNull(domainSchema) || Objects.isNull(domainSchema.getEntity())) { + return entityInfo; + } DomainInfo domainInfo = new DomainInfo(); - domainInfo.setItemId(chaConfigRichDesc.getDomainId().intValue()); - domainInfo.setName(chaConfigRichDesc.getDomainName()); - domainInfo.setWords(entityDesc.getNames()); - domainInfo.setBizName(chaConfigRichDesc.getBizName()); - if (Objects.nonNull(entityDesc.getDimItem())) { - domainInfo.setPrimaryEntityBizName(entityDesc.getDimItem().getBizName()); + domainInfo.setItemId(domainId.intValue()); + domainInfo.setName(domainSchema.getDomain().getName()); + domainInfo.setWords(domainSchema.getDomain().getAlias()); + domainInfo.setBizName(domainSchema.getDomain().getBizName()); + if (Objects.nonNull(domainSchema.getEntity())) { + domainInfo.setPrimaryEntityBizName(domainSchema.getEntity().getBizName()); } entityInfo.setDomainInfo(domainInfo); @@ -149,7 +160,7 @@ public class SemanticService { if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig()) && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig())) { - ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig() + ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig() .getChatDefaultConfig(); if (!CollectionUtils.isEmpty(chatDefaultConfig.getDimensions())) { for (SchemaElement dimensionDesc : chatDefaultConfig.getDimensions()) { @@ -187,8 +198,21 @@ public class SemanticService { semanticParseInfo.setMetrics(getMetrics(domainInfo)); semanticParseInfo.setDimensions(getDimensions(domainInfo)); DateConf dateInfo = new DateConf(); - dateInfo.setUnit(1); - dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS); + int unit = 1; + ChatConfigResp chatConfigInfo = + configService.fetchConfigByDomainId(domainSchema.getDomain().getId()); + if (Objects.nonNull(chatConfigInfo) && Objects.nonNull(chatConfigInfo.getChatDetailConfig()) + && Objects.nonNull(chatConfigInfo.getChatDetailConfig().getChatDefaultConfig())) { + ChatDefaultConfigReq chatDefaultConfig = chatConfigInfo.getChatDetailConfig().getChatDefaultConfig(); + unit = chatDefaultConfig.getUnit(); + String date = LocalDate.now().plusDays(-unit).toString(); + dateInfo.setDateMode(DateMode.BETWEEN); + dateInfo.setStartDate(date); + dateInfo.setEndDate(date); + } else { + dateInfo.setUnit(unit); + dateInfo.setDateMode(DateMode.RECENT); + } semanticParseInfo.setDateInfo(dateInfo); // add filter @@ -286,8 +310,8 @@ public class SemanticService { private ItemVisibility generateFinalVisibility(ChatConfigResp chatConfigInfo) { ItemVisibility visibility = new ItemVisibility(); - ChatAggConfig chatAggConfig = chatConfigInfo.getChatAggConfig(); - ChatDetailConfig chatDetailConfig = chatConfigInfo.getChatDetailConfig(); + ChatAggConfigReq chatAggConfig = chatConfigInfo.getChatAggConfig(); + ChatDetailConfigReq chatDetailConfig = chatConfigInfo.getChatDetailConfig(); // both black is exist if (Objects.nonNull(chatAggConfig) && Objects.nonNull(chatAggConfig.getVisibility()) @@ -307,8 +331,8 @@ public class SemanticService { } public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, - QueryResultWithSchemaResp result) { - if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics())) { + QueryResultWithSchemaResp result) { + if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) || !aggregatorConfig.getEnableRatio()) { return new AggregateInfo(); } List resultMetricNames = result.getColumns().stream().map(c -> c.getNameEn()) @@ -319,24 +343,36 @@ public class SemanticService { AggregateInfo aggregateInfo = new AggregateInfo(); MetricInfo metricInfo = new MetricInfo(); metricInfo.setStatistics(new HashMap<>()); - String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); - - Optional lastDayOp = result.getResultList().stream() - .map(r -> r.get(dateField).toString()) - .sorted(Comparator.reverseOrder()).findFirst(); - if (lastDayOp.isPresent()) { - Optional> lastValue = result.getResultList().stream() - .filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst(); - if (lastValue.isPresent()) { - metricInfo.setValue(lastValue.get().get(ratioMetric.get().getBizName()).toString()); - } - metricInfo.setDate(lastValue.get().get(dateField).toString()); - } try { - queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL, - result, metricInfo); - queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER, - result, metricInfo); + String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); + + Optional lastDayOp = result.getResultList().stream().filter(r -> r.containsKey(dateField)) + .map(r -> r.get(dateField).toString()) + .sorted(Comparator.reverseOrder()).findFirst(); + if (lastDayOp.isPresent()) { + Optional> lastValue = result.getResultList().stream() + .filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst(); + if (lastValue.isPresent() && lastValue.get().containsKey(ratioMetric.get().getBizName())) { + DecimalFormat df = new DecimalFormat("#.####"); + metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName()))); + } + metricInfo.setDate(lastValue.get().get(dateField).toString()); + } + CompletableFuture metricInfoRoll = CompletableFuture + .supplyAsync(() -> { + return queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL, + result); + }); + CompletableFuture metricInfoOver = CompletableFuture + .supplyAsync(() -> { + return queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER, + result); + }); + CompletableFuture.allOf(metricInfoRoll, metricInfoOver); + metricInfo.setName(metricInfoRoll.get().getName()); + metricInfo.setValue(metricInfoRoll.get().getValue()); + metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics()); + metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics()); aggregateInfo.getMetricInfos().add(metricInfo); } catch (Exception e) { log.error("queryRatio error {}", e); @@ -346,8 +382,10 @@ public class SemanticService { return new AggregateInfo(); } - private void queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric, - AggOperatorEnum aggOperatorEnum, QueryResultWithSchemaResp results, MetricInfo metricInfo) { + private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric, + AggOperatorEnum aggOperatorEnum, QueryResultWithSchemaResp results) { + MetricInfo metricInfo = new MetricInfo(); + metricInfo.setStatistics(new HashMap<>()); QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum); DateConf dateInfo = semanticParseInfo.getDateInfo(); String dateField = QueryReqBuilder.getDateField(dateInfo); @@ -360,12 +398,18 @@ public class SemanticService { Map result = queryResp.getResultList().get(0); Optional valueColumn = queryResp.getColumns().stream() .filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst(); - if (valueColumn.isPresent()) { + + String valueField = String.format("%s_%s", valueColumn.get().getNameEn(), + aggOperatorEnum.getOperator()); + if (result.containsKey(valueColumn.get().getNameEn())) { + DecimalFormat df = new DecimalFormat("#.####"); + metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn()))); + } String ratio = ""; - if (Objects.nonNull(result.get(valueColumn.get().getNameEn()))) { + if (Objects.nonNull(result.get(valueField))) { ratio = String.format("%.2f", - (Double.valueOf(result.get(valueColumn.get().getNameEn()).toString()) * 100)) + "%"; + (Double.valueOf(result.get(valueField).toString()) * 100)) + "%"; } String statisticsRollName = RatioOverType.DAY_ON_DAY.getShowName(); String statisticsOverName = RatioOverType.WEEK_ON_DAY.getShowName(); @@ -383,10 +427,11 @@ public class SemanticService { } metricInfo.setName(metric.getName()); } + return metricInfo; } private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo, - QueryResultWithSchemaResp results) { + QueryResultWithSchemaResp results) { String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); Optional lastDayOp = results.getResultList().stream() .map(r -> r.get(dateField).toString()) @@ -395,7 +440,7 @@ public class SemanticService { String lastDay = lastDayOp.get(); DateConf dateConf = new DateConf(); dateConf.setPeriod(semanticParseInfo.getDateInfo().getPeriod()); - dateConf.setDateMode(DateMode.LIST_DISCRETE); + dateConf.setDateMode(DateMode.LIST); List dayList = new ArrayList<>(); dayList.add(lastDay); String start = ""; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java index 9a61e618c..21077ab10 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatServiceImpl.java @@ -9,7 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.QueryDO; -import com.tencent.supersonic.chat.api.pojo.response.QueryResponse; +import com.tencent.supersonic.chat.api.pojo.response.QueryResp; import com.tencent.supersonic.chat.api.pojo.request.PageQueryInfoReq; import com.tencent.supersonic.chat.persistence.repository.ChatContextRepository; import com.tencent.supersonic.chat.persistence.repository.ChatQueryRepository; @@ -68,14 +68,6 @@ public class ChatServiceImpl implements ChatService { chatContextRepository.updateContext(chatCtx); } - @Override - public void updateContext(ChatContext chatCtx, QueryContext queryCtx, - SemanticParseInfo semanticParseInfo) { - chatCtx.setParseInfo(semanticParseInfo); - chatCtx.setQueryText(queryCtx.getRequest().getQueryText()); - updateContext(chatCtx); - } - @Override public void switchContext(ChatContext chatCtx) { log.debug("switchContext ChatContext {}", chatCtx); @@ -126,15 +118,15 @@ public class ChatServiceImpl implements ChatService { } @Override - public PageInfo queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId) { + public PageInfo queryInfo(PageQueryInfoReq pageQueryInfoCommend, long chatId) { return chatQueryRepository.getChatQuery(pageQueryInfoCommend, chatId); } @Override - public void addQuery(QueryResult queryResult, QueryContext queryContext, ChatContext chatCtx) { - chatQueryRepository.createChatQuery(queryResult, queryContext.getRequest(), chatCtx); + public void addQuery(QueryResult queryResult, ChatContext chatCtx) { + chatQueryRepository.createChatQuery(queryResult, chatCtx); chatRepository.updateLastQuestion(chatCtx.getChatId().longValue(), - queryContext.getRequest().getQueryText(), getCurrentTime()); + chatCtx.getQueryText(), getCurrentTime()); } @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java index 3c1a762f3..3e652b64c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ConfigServiceImpl.java @@ -5,6 +5,8 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticLayer; import com.tencent.supersonic.chat.api.pojo.DomainSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.request.*; +import com.tencent.supersonic.chat.api.pojo.response.*; import com.tencent.supersonic.chat.config.*; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.SemanticService; @@ -135,8 +137,8 @@ public class ConfigServiceImpl implements ConfigService { } @Override - public ChatConfigRich getConfigRichInfo(Long domainId) { - ChatConfigRich chatConfigRich = new ChatConfigRich(); + public ChatConfigRichResp getConfigRichInfo(Long domainId) { + ChatConfigRichResp chatConfigRich = new ChatConfigRichResp(); ChatConfigResp chatConfigResp = chatConfigRepository.getConfigByDomainId(domainId); if (Objects.isNull(chatConfigResp)) { log.info("there is no chatConfigDesc for domainId:{}", domainId); @@ -154,24 +156,23 @@ public class ConfigServiceImpl implements ConfigService { return chatConfigRich; } - private ChatDetailRichConfig fillChatDetailRichConfig(DomainSchema domainSchema, ChatConfigRich chatConfigRich, ChatConfigResp chatConfigResp) { + private ChatDetailRichConfigResp fillChatDetailRichConfig(DomainSchema domainSchema, ChatConfigRichResp chatConfigRich, ChatConfigResp chatConfigResp) { if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatDetailConfig())) { return null; } - ChatDetailRichConfig detailRichConfig = new ChatDetailRichConfig(); - ChatDetailConfig chatDetailConfig = chatConfigResp.getChatDetailConfig(); + ChatDetailRichConfigResp detailRichConfig = new ChatDetailRichConfigResp(); + ChatDetailConfigReq chatDetailConfig = chatConfigResp.getChatDetailConfig(); ItemVisibilityInfo itemVisibilityInfo = fetchVisibilityDescByConfig(chatDetailConfig.getVisibility(), domainSchema); detailRichConfig.setVisibility(itemVisibilityInfo); detailRichConfig.setKnowledgeInfos(fillKnowledgeBizName(chatDetailConfig.getKnowledgeInfos(), domainSchema)); detailRichConfig.setGlobalKnowledgeConfig(chatDetailConfig.getGlobalKnowledgeConfig()); detailRichConfig.setChatDefaultConfig(fetchDefaultConfig(chatDetailConfig.getChatDefaultConfig(), domainSchema, itemVisibilityInfo)); - detailRichConfig.setEntity(generateRichEntity(chatDetailConfig.getEntity(), domainSchema)); return detailRichConfig; } - private EntityRichInfo generateRichEntity(Entity entity, DomainSchema domainSchema) { - EntityRichInfo entityRichInfo = new EntityRichInfo(); + private EntityRichInfoResp generateRichEntity(Entity entity, DomainSchema domainSchema) { + EntityRichInfoResp entityRichInfo = new EntityRichInfoResp(); if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) { return entityRichInfo; } @@ -183,12 +184,12 @@ public class ConfigServiceImpl implements ConfigService { return entityRichInfo; } - private ChatAggRichConfig fillChatAggRichConfig(DomainSchema domainSchema, ChatConfigResp chatConfigResp) { + private ChatAggRichConfigResp fillChatAggRichConfig(DomainSchema domainSchema, ChatConfigResp chatConfigResp) { if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) { return null; } - ChatAggConfig chatAggConfig = chatConfigResp.getChatAggConfig(); - ChatAggRichConfig chatAggRichConfig = new ChatAggRichConfig(); + ChatAggConfigReq chatAggConfig = chatConfigResp.getChatAggConfig(); + ChatAggRichConfigResp chatAggRichConfig = new ChatAggRichConfigResp(); ItemVisibilityInfo itemVisibilityInfo = fetchVisibilityDescByConfig(chatAggConfig.getVisibility(), domainSchema); chatAggRichConfig.setVisibility(itemVisibilityInfo); chatAggRichConfig.setKnowledgeInfos(fillKnowledgeBizName(chatAggConfig.getKnowledgeInfos(), domainSchema)); @@ -198,8 +199,8 @@ public class ConfigServiceImpl implements ConfigService { return chatAggRichConfig; } - private ChatDefaultRichConfig fetchDefaultConfig(ChatDefaultConfig chatDefaultConfig, DomainSchema domainSchema, ItemVisibilityInfo itemVisibilityInfo) { - ChatDefaultRichConfig defaultRichConfig = new ChatDefaultRichConfig(); + private ChatDefaultRichConfigResp fetchDefaultConfig(ChatDefaultConfigReq chatDefaultConfig, DomainSchema domainSchema, ItemVisibilityInfo itemVisibilityInfo) { + ChatDefaultRichConfigResp defaultRichConfig = new ChatDefaultRichConfigResp(); if (Objects.isNull(chatDefaultConfig)) { return defaultRichConfig; } @@ -245,8 +246,8 @@ public class ConfigServiceImpl implements ConfigService { } - private List fillKnowledgeBizName(List knowledgeInfos, - DomainSchema domainSchema) { + private List fillKnowledgeBizName(List knowledgeInfos, + DomainSchema domainSchema) { if (CollectionUtils.isEmpty(knowledgeInfos)) { return new ArrayList<>(); } @@ -264,11 +265,11 @@ public class ConfigServiceImpl implements ConfigService { } @Override - public List getAllChatRichConfig() { - List chatConfigRichInfoList = new ArrayList<>(); + public List getAllChatRichConfig() { + List chatConfigRichInfoList = new ArrayList<>(); List domainRespList = semanticLayer.getDomainListForAdmin(); domainRespList.stream().forEach(domainResp -> { - ChatConfigRich chatConfigRichInfo = getConfigRichInfo(domainResp.getId()); + ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(domainResp.getId()); if (Objects.nonNull(chatConfigRichInfo)) { chatConfigRichInfoList.add(chatConfigRichInfo); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java index d8f87a51e..ae2223165 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/PluginServiceImpl.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.service.impl; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticLayer; +import com.tencent.supersonic.chat.plugin.PluginParseConfig; import com.tencent.supersonic.chat.plugin.Plugin; import com.tencent.supersonic.chat.api.pojo.request.PluginQueryReq; import com.tencent.supersonic.chat.persistence.dataobject.PluginDO; @@ -14,19 +15,20 @@ import com.tencent.supersonic.chat.plugin.event.PluginDelEvent; import com.tencent.supersonic.chat.plugin.event.PluginUpdateEvent; import com.tencent.supersonic.chat.service.PluginService; import com.tencent.supersonic.chat.utils.ComponentFactory; +import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.response.DomainResp; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Optional; + +import java.util.*; import java.util.stream.Collectors; @Service +@Slf4j public class PluginServiceImpl implements PluginService { private PluginRepository pluginRepository; @@ -40,10 +42,12 @@ public class PluginServiceImpl implements PluginService { } @Override - public void createPlugin(Plugin plugin, User user){ + public synchronized void createPlugin(Plugin plugin, User user){ PluginDO pluginDO = convert(plugin, user); pluginRepository.createPlugin(pluginDO); - publisher.publishEvent(new PluginAddEvent(this, plugin)); + //compatible with H2 db + List plugins = getPluginList(); + publisher.publishEvent(new PluginAddEvent(this, plugins.get(plugins.size()-1))); } @Override @@ -93,6 +97,18 @@ public class PluginServiceImpl implements PluginService { if (StringUtils.isNotBlank(pluginQueryReq.getDomain())) { pluginDOExample.getOredCriteria().get(0).andDomainLike('%' + pluginQueryReq.getDomain() + '%'); } + if (StringUtils.isNotBlank(pluginQueryReq.getParseMode())) { + pluginDOExample.getOredCriteria().get(0).andParseModeEqualTo(pluginQueryReq.getParseMode()); + } + if (StringUtils.isNotBlank(pluginQueryReq.getName())) { + pluginDOExample.getOredCriteria().get(0).andNameLike('%' + pluginQueryReq.getName() + '%'); + } + if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) { + pluginDOExample.getOredCriteria().get(0).andPatternLike('%' + pluginQueryReq.getPattern() + '%'); + } + if (StringUtils.isNotBlank(pluginQueryReq.getCreatedBy())) { + pluginDOExample.getOredCriteria().get(0).andCreatedByEqualTo(pluginQueryReq.getCreatedBy()); + } List pluginDOS = pluginRepository.query(pluginDOExample); if (StringUtils.isNotBlank(pluginQueryReq.getPattern())) { pluginDOS = pluginDOS.stream().filter(pluginDO -> @@ -105,8 +121,18 @@ public class PluginServiceImpl implements PluginService { @Override public Optional getPluginByName(String name) { + log.info("name:{}", name); return getPluginList().stream() - .filter(plugin -> plugin.getName().equalsIgnoreCase(name)) + .filter(plugin -> { + if (StringUtils.isBlank(plugin.getParseModeConfig())) { + return false; + } + PluginParseConfig functionCallConfig = JsonUtil.toObject(plugin.getParseModeConfig(), PluginParseConfig.class); + if (Objects.isNull(functionCallConfig)) { + return false; + } + return functionCallConfig.getName().equalsIgnoreCase(name); + }) .findFirst(); } @@ -120,8 +146,8 @@ public class PluginServiceImpl implements PluginService { List domainIdAuthorized = semanticLayer.getDomainListForAdmin().stream() .map(DomainResp::getId).collect(Collectors.toList()); plugins = plugins.stream().filter(plugin -> { - if (CollectionUtils.isEmpty(plugin.getDomainList())) { - return false; + if (CollectionUtils.isEmpty(plugin.getDomainList()) || plugin.isContainsAllDomain()) { + return true; } for (Long domainId : plugin.getDomainList()) { if (domainIdAuthorized.contains(domainId)) { @@ -136,8 +162,7 @@ public class PluginServiceImpl implements PluginService { public Plugin convert(PluginDO pluginDO){ Plugin plugin = new Plugin(); BeanUtils.copyProperties(pluginDO,plugin); - plugin.setParseMode(ParseMode.valueOf(pluginDO.getParseMode())); - if (pluginDO.getDomain() != null) { + if (StringUtils.isNotBlank(pluginDO.getDomain())) { plugin.setDomainList(Arrays.stream(pluginDO.getDomain().split(",")) .map(Long::parseLong).collect(Collectors.toList())); } @@ -152,7 +177,6 @@ public class PluginServiceImpl implements PluginService { pluginDO.setUpdatedAt(new Date()); pluginDO.setUpdatedBy(user.getName()); pluginDO.setDomain(StringUtils.join(plugin.getDomainList(), ",")); - pluginDO.setParseMode(plugin.getParseMode().name()); return pluginDO; } @@ -161,7 +185,6 @@ public class PluginServiceImpl implements PluginService { pluginDO.setUpdatedAt(new Date()); pluginDO.setUpdatedBy(user.getName()); pluginDO.setDomain(StringUtils.join(plugin.getDomainList(), ",")); - pluginDO.setParseMode(plugin.getParseMode().name()); return pluginDO; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index ae4e51ae0..83df0ff3c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -6,15 +6,19 @@ import com.tencent.supersonic.chat.api.component.*; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.query.QuerySelector; -import com.tencent.supersonic.chat.api.pojo.request.QueryDataRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.service.QueryService; import com.tencent.supersonic.chat.utils.ComponentFactory; + +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -41,7 +45,82 @@ public class QueryServiceImpl implements QueryService { private QuerySelector querySelector = ComponentFactory.getQuerySelector(); @Override - public QueryResult executeQuery(QueryRequest queryReq) throws Exception { + public ParseResp performParsing(QueryReq queryReq) { + QueryContext queryCtx = new QueryContext(queryReq); + // in order to support multi-turn conversation, chat context is needed + ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId()); + + schemaMappers.stream().forEach(mapper -> { + mapper.map(queryCtx); + log.info("{} result:{}", mapper.getClass().getSimpleName(), JsonUtil.toString(queryCtx)); + }); + + semanticParsers.stream().forEach(parser -> { + parser.parse(queryCtx, chatCtx); + log.info("{} result:{}", parser.getClass().getSimpleName(), JsonUtil.toString(queryCtx)); + }); + + ParseResp parseResult; + if (queryCtx.getCandidateQueries().size() > 0) { + log.debug("pick before [{}]", queryCtx.getCandidateQueries().stream().collect( + Collectors.toList())); + List selectedQueries = querySelector.select(queryCtx.getCandidateQueries()); + log.debug("pick after [{}]", selectedQueries.stream().collect( + Collectors.toList())); + + List selectedParses = selectedQueries.stream() + .map(q -> q.getParseInfo()).collect(Collectors.toList()); + List candidateParses = queryCtx.getCandidateQueries().stream() + .map(q -> q.getParseInfo()).collect(Collectors.toList()); + + parseResult = ParseResp.builder() + .chatId(queryReq.getChatId()) + .queryText(queryReq.getQueryText()) + .state(selectedParses.size() > 1 ? ParseResp.ParseState.PENDING : ParseResp.ParseState.COMPLETED) + .selectedParses(selectedParses) + .candidateParses(candidateParses) + .build(); + } else { + parseResult = ParseResp.builder() + .chatId(queryReq.getChatId()) + .queryText(queryReq.getQueryText()) + .state(ParseResp.ParseState.FAILED) + .build(); + } + + return parseResult; + } + + @Override + public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception { + SemanticParseInfo parseInfo = queryReq.getParseInfo(); + SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); + if (semanticQuery == null) { + return null; + } + semanticQuery.setParseInfo(parseInfo); + + // in order to support multi-turn conversation, chat context is needed + ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId()); + + QueryResult queryResult = semanticQuery.execute(queryReq.getUser()); + if (queryResult != null) { + queryResult.setChatContext(parseInfo); + // update chat context after a successful semantic query + if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) { + chatCtx.setParseInfo(parseInfo); + chatService.updateContext(chatCtx); + } + chatCtx.setQueryText(queryReq.getQueryText()); + chatCtx.setUser(queryReq.getUser().getName()); + chatService.addQuery(queryResult, chatCtx); + } + + return queryResult; + } + + @Override + public QueryResult executeQuery(QueryReq queryReq) throws Exception { QueryContext queryCtx = new QueryContext(queryReq); // in order to support multi-turn conversation, chat context is needed ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId()); @@ -60,17 +139,21 @@ public class QueryServiceImpl implements QueryService { if (queryCtx.getCandidateQueries().size() > 0) { log.info("pick before [{}]", queryCtx.getCandidateQueries().stream().collect( Collectors.toList())); - SemanticQuery semanticQuery = querySelector.select(queryCtx.getCandidateQueries()); - log.info("pick after [{}]", semanticQuery); + List selectedQueries = querySelector.select(queryCtx.getCandidateQueries()); + log.info("pick after [{}]", selectedQueries.stream().collect( + Collectors.toList())); + SemanticQuery semanticQuery = selectedQueries.get(0); queryResult = semanticQuery.execute(queryReq.getUser()); if (queryResult != null) { + chatCtx.setQueryText(queryReq.getQueryText()); // update chat context after a successful semantic query if (queryReq.isSaveAnswer() && QueryState.SUCCESS.equals(queryResult.getQueryState())) { - chatService.updateContext(chatCtx, queryCtx, semanticQuery.getParseInfo()); + chatCtx.setParseInfo(semanticQuery.getParseInfo()); + chatService.updateContext(chatCtx); } queryResult.setChatContext(chatCtx.getParseInfo()); - chatService.addQuery(queryResult, queryCtx, chatCtx); + chatService.addQuery(queryResult, chatCtx); } } @@ -78,13 +161,13 @@ public class QueryServiceImpl implements QueryService { } @Override - public SemanticParseInfo queryContext(QueryRequest queryCtx) { + public SemanticParseInfo queryContext(QueryReq queryCtx) { ChatContext context = chatService.getOrCreateContext(queryCtx.getChatId()); return context.getParseInfo(); } @Override - public QueryResult executeDirectQuery(QueryDataRequest queryData, User user) throws SqlParseException { + public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException { SemanticQuery semanticQuery = QueryManager.createRuleQuery(queryData.getQueryMode()); BeanUtils.copyProperties(queryData, semanticQuery.getParseInfo()); return semanticQuery.execute(user); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java index ab314f84a..7ba8cc18d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/RecommendServiceImpl.java @@ -3,12 +3,12 @@ package com.tencent.supersonic.chat.service.impl; import com.tencent.supersonic.chat.api.pojo.DomainSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; -import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestion; -import com.tencent.supersonic.chat.config.ChatConfigFilter; -import com.tencent.supersonic.chat.config.ChatConfigResp; -import com.tencent.supersonic.chat.config.ChatConfigRich; -import com.tencent.supersonic.chat.api.pojo.response.RecommendResponse; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.response.RecommendQuestionResp; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.api.pojo.response.RecommendResp; import java.util.ArrayList; import java.util.Comparator; @@ -37,14 +37,14 @@ public class RecommendServiceImpl implements RecommendService { private SemanticService semanticService; @Override - public RecommendResponse recommend(QueryRequest queryCtx, Long limit) { + public RecommendResp recommend(QueryReq queryCtx, Long limit) { if (Objects.isNull(limit) || limit <= 0) { limit = Long.MAX_VALUE; } log.debug("limit:{}", limit); Long domainId = queryCtx.getDomainId(); if (Objects.isNull(domainId)) { - return new RecommendResponse(); + return new RecommendResp(); } DomainSchema domainSchema = semanticService.getDomainSchema(domainId); @@ -75,21 +75,21 @@ public class RecommendServiceImpl implements RecommendService { return item; }).collect(Collectors.toList()); - RecommendResponse response = new RecommendResponse(); + RecommendResp response = new RecommendResp(); response.setDimensions(dimensions); response.setMetrics(metrics); return response; } @Override - public RecommendResponse recommendMetricMode(QueryRequest queryCtx, Long limit) { - RecommendResponse recommendResponse = recommend(queryCtx, limit); + public RecommendResp recommendMetricMode(QueryReq queryCtx, Long limit) { + RecommendResp recommendResponse = recommend(queryCtx, limit); // filter black Item if (Objects.isNull(recommendResponse)) { return recommendResponse; } - ChatConfigRich chatConfigRich = configService.getConfigRichInfo(Long.valueOf(queryCtx.getDomainId())); + ChatConfigRichResp chatConfigRich = configService.getConfigRichInfo(Long.valueOf(queryCtx.getDomainId())); if (Objects.nonNull(chatConfigRich) && Objects.nonNull(chatConfigRich.getChatAggRichConfig()) && Objects.nonNull(chatConfigRich.getChatAggRichConfig().getVisibility())) { List blackMetricIdList = chatConfigRich.getChatAggRichConfig().getVisibility().getBlackMetricIdList(); @@ -105,15 +105,15 @@ public class RecommendServiceImpl implements RecommendService { } @Override - public List recommendQuestion(Long domainId) { - List recommendQuestions = new ArrayList<>(); + public List recommendQuestion(Long domainId) { + List recommendQuestions = new ArrayList<>(); ChatConfigFilter chatConfigFilter = new ChatConfigFilter(); chatConfigFilter.setDomainId(domainId); List chatConfigRespList = configService.search(chatConfigFilter, null); if (!CollectionUtils.isEmpty(chatConfigRespList)) { chatConfigRespList.stream().forEach(chatConfigResp -> { if (Objects.nonNull(chatConfigResp) && !CollectionUtils.isEmpty(chatConfigResp.getRecommendedQuestions())) { - recommendQuestions.add(new RecommendQuestion(chatConfigResp.getDomainId(), chatConfigResp.getRecommendedQuestions())); + recommendQuestions.add(new RecommendQuestionResp(chatConfigResp.getDomainId(), chatConfigResp.getRecommendedQuestions())); } }); return recommendQuestions; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java index 745c1c337..1fac983c3 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SearchServiceImpl.java @@ -7,7 +7,7 @@ import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.SearchResult; import com.tencent.supersonic.chat.mapper.DomainInfoStat; import com.tencent.supersonic.chat.mapper.DomainWithSemanticType; @@ -55,7 +55,7 @@ public class SearchServiceImpl implements SearchService { private SearchMatchStrategy searchMatchStrategy; @Override - public List search(QueryRequest queryCtx) { + public List search(QueryReq queryCtx) { String queryText = queryCtx.getQueryText(); // 1.get meta info SemanticSchema semanticSchemaDb = schemaService.getSemanticSchema(); @@ -109,8 +109,8 @@ public class SearchServiceImpl implements SearchService { return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList()); } - private List getPossibleDomains(QueryRequest queryCtx, List originals, - DomainInfoStat domainStat, Long webDomainId) { + private List getPossibleDomains(QueryReq queryCtx, List originals, + DomainInfoStat domainStat, Long webDomainId) { if (Objects.nonNull(webDomainId) && webDomainId > 0) { List result = new ArrayList<>(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/CacheUtils.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/CacheUtils.java index 1062f27eb..c6bec3994 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/CacheUtils.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/CacheUtils.java @@ -14,11 +14,11 @@ public class CacheUtils { .maximumSize(1000) .build(); public static void put(QueryContext queryContext, ChatContext chatCtx, Object v){ - String key=chatCtx.getUser()+"_"+chatCtx.getChatId()+"_"+queryContext.getRequest().getQueryText(); + String key=chatCtx.getUser()+"_"+chatCtx.getChatId()+"_"+ queryContext.getRequest().getQueryText(); cache.put(key,v); } - public static Object get(QueryContext queryContext,ChatContext chatCtx){ - String key=chatCtx.getUser()+"_"+chatCtx.getChatId()+"_"+queryContext.getRequest().getQueryText(); + public static Object get(QueryContext queryContext, ChatContext chatCtx){ + String key=chatCtx.getUser()+"_"+chatCtx.getChatId()+"_"+ queryContext.getRequest().getQueryText(); return cache.getIfPresent(key); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatConfigHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatConfigHelper.java index 28a2f59bb..e2a823f04 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatConfigHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatConfigHelper.java @@ -5,7 +5,8 @@ import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.DomainSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestion; +import com.tencent.supersonic.chat.api.pojo.request.*; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.config.*; import com.tencent.supersonic.chat.persistence.dataobject.ChatConfigDO; import com.tencent.supersonic.common.pojo.enums.StatusEnum; @@ -103,9 +104,9 @@ public class ChatConfigHelper { BeanUtils.copyProperties(chatConfigDO, chatConfigDescriptor); - chatConfigDescriptor.setChatDetailConfig(JsonUtil.toObject(chatConfigDO.getChatDetailConfig(), ChatDetailConfig.class)); - chatConfigDescriptor.setChatAggConfig(JsonUtil.toObject(chatConfigDO.getChatAggConfig(), ChatAggConfig.class)); - chatConfigDescriptor.setRecommendedQuestions(JsonUtil.toList(chatConfigDO.getRecommendedQuestions(), RecommendedQuestion.class)); + chatConfigDescriptor.setChatDetailConfig(JsonUtil.toObject(chatConfigDO.getChatDetailConfig(), ChatDetailConfigReq.class)); + chatConfigDescriptor.setChatAggConfig(JsonUtil.toObject(chatConfigDO.getChatAggConfig(), ChatAggConfigReq.class)); + chatConfigDescriptor.setRecommendedQuestions(JsonUtil.toList(chatConfigDO.getRecommendedQuestions(), RecommendedQuestionReq.class)); chatConfigDescriptor.setStatusEnum(StatusEnum.of(chatConfigDO.getStatus())); chatConfigDescriptor.setCreatedBy(chatConfigDO.getCreatedBy()); @@ -132,15 +133,15 @@ public class ChatConfigHelper { return chatConfigResp; } - private ChatDetailConfig generateEmptyChatDetailConfigResp() { - ChatDetailConfig chatDetailConfig = new ChatDetailConfig(); + private ChatDetailConfigReq generateEmptyChatDetailConfigResp() { + ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); ItemVisibility visibility = new ItemVisibility(); chatDetailConfig.setVisibility(visibility); return chatDetailConfig; } - private ChatAggConfig generateEmptyChatAggConfigResp() { - ChatAggConfig chatAggConfig = new ChatAggConfig(); + private ChatAggConfigReq generateEmptyChatAggConfigResp() { + ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); ItemVisibility visibility = new ItemVisibility(); chatAggConfig.setVisibility(visibility); return chatAggConfig; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatGptHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatGptHelper.java new file mode 100644 index 000000000..b7672c7b6 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/ChatGptHelper.java @@ -0,0 +1,78 @@ +package com.tencent.supersonic.chat.utils; + + +import com.plexpt.chatgpt.ChatGPT; +import com.plexpt.chatgpt.entity.chat.ChatCompletion; +import com.plexpt.chatgpt.entity.chat.ChatCompletionResponse; +import com.plexpt.chatgpt.entity.chat.Message; +import com.plexpt.chatgpt.util.Proxys; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.net.Proxy; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; + + +@Component +public class ChatGptHelper { + + @Value("${llm.chatgpt.apikey:sk-kdgPxxx}") + private String apiKey; + + @Value("${llm.chatgpt.apiHost:https://api.openai.com/}") + private String apiHost; + + @Value("${llm.chatgpt.proxyIp:default}") + private String proxyIp; + + @Value("${llm.chatgpt.proxyPort:8080}") + private Integer proxyPort; + + + public ChatGPT getChatGPT(){ + Proxy proxy = null; + if (!"default".equals(proxyIp)){ + proxy = Proxys.http(proxyIp, proxyPort); + } + return ChatGPT.builder() + .apiKey(apiKey) + .proxy(proxy) + .timeout(900) + .apiHost(apiHost) //反向代理地址 + .build() + .init(); + } + + public String inferredTime(String queryText){ + long nowTime = System.currentTimeMillis(); + Date date = new Date(nowTime); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + String formattedDate = sdf.format(date); + Message system = Message.ofSystem("现在时间 "+formattedDate+"，你是一个专业的数据分析师，你的任务是基于数据，专业的解答用户的问题。" + + "你需要遵守以下规则：\n" + + "1.返回规范的数据格式，json，如： 输入：近 10 天的日活跃数，输出：{\"start\":\"2023-07-21\",\"end\":\"2023-07-31\"}" + + "2.你对时间数据要求规范，能从近 10 天，国庆节，端午节，获取到相应的时间，填写到 json 中。\n"+ + "3.你的数据时间，只有当前及之前时间即可,超过则回复去年\n" + + "4.只需要解析出时间，时间可以是时间月和年或日、日历采用公历\n"+ + "5.时间给出要是绝对正确，不能瞎编\n" + ); + Message message = Message.of("输入："+queryText+"，输出："); + ChatCompletion chatCompletion = ChatCompletion.builder() + .model(ChatCompletion.Model.GPT_3_5_TURBO_16K.getName()) + .messages(Arrays.asList(system, message)) + .maxTokens(10000) + .temperature(0.9) + .build(); + ChatCompletionResponse response = getChatGPT().chatCompletion(chatCompletion); + Message res = response.getChoices().get(0).getMessage(); + return res.getContent(); + } + + public static void main(String[] args) { + + } + + +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java index ae8abf276..5e62d1754 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java @@ -6,6 +6,10 @@ import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; import com.tencent.supersonic.chat.api.component.SemanticLayer; import com.tencent.supersonic.chat.api.pojo.DomainSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig; +import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp; import com.tencent.supersonic.chat.config.*; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO; @@ -124,13 +128,13 @@ public class DictMetaHelper { private void fillDimValueDOList(List dimValueDOList, Long domainId, Map dimIdAndDescPair) { - ChatConfigRich chaConfigRichDesc = configService.getConfigRichInfo(domainId); + ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(domainId); if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatAggRichConfig())) { - ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig(); - List knowledgeAggInfo = chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos(); + ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig(); + List knowledgeAggInfo = chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos(); - List knowledgeDetailInfo = chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos(); + List knowledgeDetailInfo = chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos(); fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, domainId); fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, domainId); @@ -139,7 +143,7 @@ public class DictMetaHelper { } } - private void fillKnowledgeDimValue(List knowledgeInfos, ChatDefaultRichConfig chatDefaultConfig, + private void fillKnowledgeDimValue(List knowledgeInfos, ChatDefaultRichConfigResp chatDefaultConfig, List dimValueDOList, Map dimIdAndDescPair, Long domainId) { if (!CollectionUtils.isEmpty(knowledgeInfos)) { List dimensions = new ArrayList<>(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java index fe7bae341..66c1e471f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java @@ -91,7 +91,7 @@ public class DictQueryHelper { } private List generateFileData(List> resultList, String nature, String dimName, - String metricName) { + String metricName) { List data = new ArrayList<>(); if (CollectionUtils.isEmpty(resultList)) { return data; @@ -160,7 +160,7 @@ public class DictQueryHelper { queryStructCmd.setOrders(orders); DateConf dateInfo = new DateConf(); - dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS); + dateInfo.setDateMode(DateConf.DateMode.RECENT); dateInfo.setUnit(defaultMetricDesc.getUnit()); queryStructCmd.setDateInfo(dateInfo); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java index 44a1875e8..82213445b 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java @@ -3,9 +3,7 @@ package com.tencent.supersonic.chat.utils; import com.google.common.collect.Lists; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery; -import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery; -import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery; +import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; @@ -17,11 +15,8 @@ import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; +import java.time.LocalDate; +import java.util.*; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.util.Strings; @@ -35,7 +30,7 @@ public class QueryReqBuilder { QueryStructReq queryStructCmd = new QueryStructReq(); queryStructCmd.setDomainId(parseInfo.getDomainId()); queryStructCmd.setNativeQuery(parseInfo.getNativeQuery()); - queryStructCmd.setDateInfo(parseInfo.getDateInfo()); + queryStructCmd.setDateInfo(rewrite2Between(parseInfo.getDateInfo())); List dimensionFilters = parseInfo.getDimensionFilters().stream() .filter(chatFilter -> Strings.isNotEmpty(chatFilter.getBizName())) @@ -49,16 +44,47 @@ public class QueryReqBuilder { queryStructCmd.setMetricFilters(metricFilters); addDateDimension(parseInfo); - List dimensions = parseInfo.getDimensions().stream().map(entry -> entry.getBizName()) + List dimensions = parseInfo.getDimensions().stream().map(SchemaElement::getBizName) .collect(Collectors.toList()); queryStructCmd.setGroups(dimensions); queryStructCmd.setLimit(parseInfo.getLimit()); - Set order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), parseInfo.getMetrics()); - queryStructCmd.setOrders(new ArrayList<>(order)); - queryStructCmd.setAggregators(getAggregatorByMetric(parseInfo.getMetrics(), parseInfo.getAggType())); + // only one metric is queried at once + Set metrics = parseInfo.getMetrics(); + if (!CollectionUtils.isEmpty(metrics)) { + SchemaElement metricElement = parseInfo.getMetrics().iterator().next(); + Set order = getOrder(parseInfo.getOrders(), parseInfo.getAggType(), metricElement); + queryStructCmd.setAggregators(getAggregatorByMetric(parseInfo.getAggType(), metricElement)); + queryStructCmd.setOrders(new ArrayList<>(order)); + } + + deletionDuplicated(queryStructCmd); + return queryStructCmd; } + private static void deletionDuplicated(QueryStructReq queryStructReq) { + if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) && queryStructReq.getGroups().size() > 1) { + Set groups = new HashSet<>(); + groups.addAll(queryStructReq.getGroups()); + queryStructReq.getGroups().clear(); + queryStructReq.getGroups().addAll(groups); + } + } + + private static DateConf rewrite2Between(DateConf dateInfo) { + DateConf dateInfoNew = new DateConf(); + BeanUtils.copyProperties(dateInfo, dateInfoNew); + if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT.equals(dateInfo.getDateMode())) { + int unit = dateInfo.getUnit(); + String startDate = LocalDate.now().plusDays(-unit).toString(); + String endDate = LocalDate.now().plusDays(-1).toString(); + dateInfoNew.setDateMode(DateConf.DateMode.BETWEEN); + dateInfoNew.setStartDate(startDate); + dateInfoNew.setEndDate(endDate); + } + return dateInfoNew; + } + public static QueryMultiStructReq buildMultiStructReq(SemanticParseInfo parseInfo) { QueryStructReq queryStructReq = buildStructReq(parseInfo); QueryMultiStructReq queryMultiStructReq = new QueryMultiStructReq(); @@ -75,11 +101,12 @@ public class QueryReqBuilder { /** * convert to QueryDslReq + * * @param querySql * @param domainId * @return */ - public static QueryDslReq buildDslReq(String querySql,Long domainId) { + public static QueryDslReq buildDslReq(String querySql, Long domainId) { QueryDslReq queryDslReq = new QueryDslReq(); if (Objects.nonNull(querySql)) { queryDslReq.setSql(querySql); @@ -89,11 +116,11 @@ public class QueryReqBuilder { } - private static List getAggregatorByMetric(Set metrics, AggregateTypeEnum aggregateType) { + private static List getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) { List aggregators = new ArrayList<>(); - String agg = (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)) ? "" - : aggregateType.name(); - for (SchemaElement metric : metrics) { + if(metric != null) { + String agg = (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)) ? "" + : aggregateType.name(); aggregators.add(new Aggregator(metric.getBizName(), AggOperatorEnum.of(agg))); } return aggregators; @@ -118,41 +145,42 @@ public class QueryReqBuilder { return; } } + + if (Objects.nonNull(parseInfo.getAggType()) && !parseInfo.getAggType().equals(AggregateTypeEnum.NONE)) { + return; + } + SchemaElement dimension = new SchemaElement(); dimension.setBizName(dateField); - if (MetricDomainQuery.QUERY_MODE.equals(queryMode) - || MetricGroupByQuery.QUERY_MODE.equals(queryMode) - || MetricFilterQuery.QUERY_MODE.equals(queryMode) - ) { + if (QueryManager.isMetricQuery(queryMode)) { parseInfo.getDimensions().add(dimension); } } } - public static Set getOrder(Set parseOrder, AggregateTypeEnum aggregator, Set metrics) { + public static Set getOrder(Set parseOrder, AggregateTypeEnum aggregator, SchemaElement metric) { if (!CollectionUtils.isEmpty(parseOrder)) { return parseOrder; } Set orders = new LinkedHashSet(); - if (CollectionUtils.isEmpty(metrics)) { + if (metric == null) { return orders; } + if ((AggregateTypeEnum.TOPN.equals(aggregator) || AggregateTypeEnum.MAX.equals(aggregator) || AggregateTypeEnum.MIN.equals( aggregator))) { - for (SchemaElement metric : metrics) { - Order order = new Order(); - order.setColumn(metric.getBizName()); - order.setDirection("desc"); - orders.add(order); - } + Order order = new Order(); + order.setColumn(metric.getBizName()); + order.setDirection("desc"); + orders.add(order); } return orders; } public static String getDateField(DateConf dateConf) { - if(Objects.isNull(dateConf)) { + if (Objects.isNull(dateConf)) { return ""; } String dateField = TimeDimensionEnum.DAY.getName(); @@ -165,7 +193,7 @@ public class QueryReqBuilder { return dateField; } - public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo,SchemaElement metric,AggOperatorEnum aggOperatorEnum) { + public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, SchemaElement metric, AggOperatorEnum aggOperatorEnum) { QueryStructReq queryStructCmd = buildStructReq(parseInfo); queryStructCmd.setNativeQuery(false); queryStructCmd.setOrders(new ArrayList<>()); diff --git a/chat/core/src/main/resources/mapper/PluginDOMapper.xml b/chat/core/src/main/resources/mapper/PluginDOMapper.xml index 0fe6e3f5e..b9b4842e7 100644 --- a/chat/core/src/main/resources/mapper/PluginDOMapper.xml +++ b/chat/core/src/main/resources/mapper/PluginDOMapper.xml @@ -14,7 +14,9 @@ + + @@ -50,7 +52,7 @@ updated_by - config + parse_mode_config, config, comment @@ -216,9 +232,15 @@ updated_by = #{updatedBy,jdbcType=VARCHAR}, + + parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR}, + config = #{config,jdbcType=LONGVARCHAR}, + + comment = #{comment,jdbcType=LONGVARCHAR}, + where id = #{id,jdbcType=BIGINT} @@ -233,7 +255,9 @@ created_by = #{createdBy,jdbcType=VARCHAR}, updated_at = #{updatedAt,jdbcType=TIMESTAMP}, updated_by = #{updatedBy,jdbcType=VARCHAR}, - config = #{config,jdbcType=LONGVARCHAR} + parse_mode_config = #{parseModeConfig,jdbcType=LONGVARCHAR}, + config = #{config,jdbcType=LONGVARCHAR}, + comment = #{comment,jdbcType=LONGVARCHAR} where id = #{id,jdbcType=BIGINT} diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeRangeParserTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeRangeParserTest.java index 23d916e33..52d0ba795 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeRangeParserTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/application/parser/TimeRangeParserTest.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.application.parser; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.parser.rule.TimeRangeParser; import org.junit.jupiter.api.Test; @@ -27,7 +27,7 @@ class TimeRangeParserTest { void parse() { TimeRangeParser timeRangeParser = new TimeRangeParser(); - QueryRequest queryRequest = new QueryRequest(); + QueryReq queryRequest = new QueryReq(); ChatContext chatCtx = new ChatContext(); SchemaMapInfo schemaMap = new SchemaMapInfo(); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/mapper/HanlpDictMapperTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/mapper/HanlpDictMapperTest.java index 470215470..f96174c50 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/mapper/HanlpDictMapperTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/mapper/HanlpDictMapperTest.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.chat.mapper; import com.tencent.supersonic.chat.api.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.test.context.ContextTest; import org.junit.jupiter.api.Test; @@ -12,7 +12,7 @@ class HanlpDictMapperTest extends ContextTest { @Test void map() { - QueryRequest queryRequest = new QueryRequest(); + QueryReq queryRequest = new QueryReq(); queryRequest.setChatId(1); queryRequest.setDomainId(2L); queryRequest.setQueryText("supersonic按部门访问次数"); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java index e79eaf500..c00439b44 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/MockBeansConfiguration.java @@ -6,6 +6,9 @@ import static org.mockito.Mockito.when; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.component.SemanticLayer; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp; +import com.tencent.supersonic.chat.api.pojo.response.EntityRichInfoResp; import com.tencent.supersonic.chat.config.*; import com.tencent.supersonic.chat.persistence.repository.impl.ChatContextRepositoryImpl; import com.tencent.supersonic.chat.service.QueryService; @@ -40,12 +43,12 @@ public class MockBeansConfiguration { public static void buildHttpSemanticServiceImpl(SemanticLayer httpSemanticLayer, List dimensionDescs, List metricDescs) { - ChatConfigRich chaConfigRichDesc = new ChatConfigRich(); + ChatConfigRichResp chaConfigRichDesc = new ChatConfigRichResp(); DefaultMetric defaultMetricDesc = new DefaultMetric(); defaultMetricDesc.setUnit(3); defaultMetricDesc.setPeriod(Constants.DAY); // chaConfigRichDesc.setDefaultMetrics(new ArrayList<>(Arrays.asList(defaultMetricDesc))); - EntityRichInfo entityDesc = new EntityRichInfo(); + EntityRichInfoResp entityDesc = new EntityRichInfoResp(); List dimensionDescs1 = new ArrayList<>(); DimSchemaResp dimensionDesc = new DimSchemaResp(); dimensionDesc.setId(162L); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java index 16fad98df..e60fde11c 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/test/context/SemanticParseObjectHelper.java @@ -66,7 +66,7 @@ public class SemanticParseObjectHelper { if (dayAgo > 0) { DateConf dateInfo = new DateConf(); dateInfo.setUnit(dayAgo); - dateInfo.setDateMode(DateConf.DateMode.RECENT_UNITS); + dateInfo.setDateMode(DateConf.DateMode.RECENT); return dateInfo; } return null; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/EntityWordBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/EntityWordBuilder.java index 98a2ae368..cd5f60c68 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/EntityWordBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/builder/EntityWordBuilder.java @@ -3,12 +3,14 @@ package com.tencent.supersonic.knowledge.dictionary.builder; import com.google.common.collect.Lists; import java.util.List; +import java.util.Objects; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.knowledge.dictionary.DictWord; import com.tencent.supersonic.knowledge.dictionary.DictWordType; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; /** * dimension value wordNature @@ -20,13 +22,23 @@ public class EntityWordBuilder extends BaseWordBuilder { @Override public List doGet(String word, SchemaElement schemaElement) { List result = Lists.newArrayList(); - DictWord dictWord = new DictWord(); - dictWord.setWord(word); + + if (Objects.isNull(schemaElement)) { + return result; + } + Long domain = schemaElement.getDomain(); String nature = DictWordType.NATURE_SPILT + domain + DictWordType.NATURE_SPILT + schemaElement.getId() + DictWordType.ENTITY.getType(); - dictWord.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY * 2, nature)); - result.add(dictWord); + + if (!CollectionUtils.isEmpty(schemaElement.getAlias())) { + schemaElement.getAlias().stream().forEach(alias -> { + DictWord dictWordAlias = new DictWord(); + dictWordAlias.setWord(alias); + dictWordAlias.setNatureWithFrequency(String.format("%s " + DEFAULT_FREQUENCY * 2, nature)); + result.add(dictWordAlias); + }); + } return result; } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticLayer.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticLayer.java index 56ad0ecbe..2ef4b38b4 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticLayer.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/BaseSemanticLayer.java @@ -4,19 +4,13 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.tencent.supersonic.chat.api.component.SemanticLayer; import com.tencent.supersonic.chat.api.pojo.DomainSchema; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.ResultData; import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.core.ParameterizedTypeReference; @@ -76,31 +70,5 @@ public abstract class BaseSemanticLayer implements SemanticLayer { return domainSchemaList; } - protected void deletionDuplicated(QueryStructReq queryStructReq) { - if (!CollectionUtils.isEmpty(queryStructReq.getGroups()) && queryStructReq.getGroups().size() > 1) { - Set groups = new HashSet<>(); - groups.addAll(queryStructReq.getGroups()); - queryStructReq.getGroups().clear(); - queryStructReq.getGroups().addAll(groups); - } - } - - protected void onlyQueryFirstMetric(QueryStructReq queryStructReq) { - if (!CollectionUtils.isEmpty(queryStructReq.getAggregators()) && queryStructReq.getAggregators().size() > 1) { - log.info("multi metric in aggregators:{} , only query first one", queryStructReq.getAggregators()); - List aggregators = queryStructReq.getAggregators().subList(0, 1); - List excludeAggregators = queryStructReq.getAggregators().stream().map(a -> a.getColumn()) - .filter(a -> !a.equals(aggregators.get(0).getColumn())).collect( - Collectors.toList()); - queryStructReq.setAggregators(aggregators); - List orders = queryStructReq.getOrders().stream() - .filter(o -> !excludeAggregators.contains(o.getColumn())).collect( - Collectors.toList()); - log.info("multi metric in orders:{} ", queryStructReq.getOrders()); - queryStructReq.setOrders(orders); - - } - } - protected abstract List doFetchDomainSchema(List ids); } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DomainSchemaBuilder.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DomainSchemaBuilder.java index f6a4c2e16..0417830a8 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DomainSchemaBuilder.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DomainSchemaBuilder.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.chat.api.pojo.DomainSchema; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap; +import com.tencent.supersonic.semantic.api.model.pojo.Entity; import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp; import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp; import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp; @@ -13,6 +14,7 @@ import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; import java.util.*; +import java.util.stream.Collectors; public class DomainSchemaBuilder { @@ -100,18 +102,21 @@ public class DomainSchemaBuilder { domainSchema.getDimensions().addAll(dimensions); domainSchema.getDimensionValues().addAll(dimensionValues); - if (!CollectionUtils.isEmpty(resp.getEntityNames())) { - Set entities = new HashSet<>(); - for (String entity : resp.getEntityNames()) { - entities.add(SchemaElement.builder() - .domain(resp.getId()) - .id(resp.getId()) - .name(entity) - .bizName(entity) - .type(SchemaElementType.ENTITY) - .build()); + + Entity entity = resp.getEntity(); + if (Objects.nonNull(entity)) { + SchemaElement entityElement = new SchemaElement(); + + if (!CollectionUtils.isEmpty(entity.getNames()) && Objects.nonNull(entity.getEntityId())) { + Map idAndDimPair = dimensions.stream() + .collect(Collectors.toMap(SchemaElement::getId, schemaElement -> schemaElement,(k1,k2)->k2)); + if (idAndDimPair.containsKey(entity.getEntityId())) { + entityElement = idAndDimPair.get(entity.getEntityId()); + entityElement.setType(SchemaElementType.ENTITY); + } + entityElement.setAlias(entity.getNames()); + domainSchema.setEntity(entityElement); } - domainSchema.getEntities().addAll(entities); } return domainSchema; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java index 9cc896198..168d448e6 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java @@ -36,8 +36,6 @@ public class LocalSemanticLayer extends BaseSemanticLayer { @Override public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) { - deletionDuplicated(queryStructReq); - onlyQueryFirstMetric(queryStructReq); try { QueryService queryService = ContextUtils.getBean(QueryService.class); QueryResultWithSchemaResp queryResultWithSchemaResp = queryService.queryByStruct(queryStructReq, user); @@ -50,10 +48,6 @@ public class LocalSemanticLayer extends BaseSemanticLayer { @Override public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { - for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { - deletionDuplicated(queryStructReq); - onlyQueryFirstMetric(queryStructReq); - } try { QueryService queryService = ContextUtils.getBean(QueryService.class); return queryService.queryByMultiStruct(queryMultiStructReq, user); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java index 7a229b1ba..10be3de0a 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java @@ -58,8 +58,6 @@ public class RemoteSemanticLayer extends BaseSemanticLayer { @Override public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) { - deletionDuplicated(queryStructReq); - onlyQueryFirstMetric(queryStructReq); DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); return searchByRestTemplate( defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByStructPath(), @@ -68,10 +66,6 @@ public class RemoteSemanticLayer extends BaseSemanticLayer { @Override public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { - for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { - deletionDuplicated(queryStructReq); - onlyQueryFirstMetric(queryStructReq); - } DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); return searchByRestTemplate( defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByMultiStructPath(), diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java index a3f0d292c..39b3c1d82 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java @@ -4,6 +4,7 @@ import static java.time.LocalDate.now; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import lombok.Data; @@ -13,7 +14,7 @@ public class DateConf { private static final long serialVersionUID = 3074129990945004340L; - private DateMode dateMode = DateMode.RECENT_UNITS; + private DateMode dateMode = DateMode.RECENT; /** * like 2021-10-22, dateMode=1 @@ -42,15 +43,32 @@ public class DateConf { */ private String text; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DateConf dateConf = (DateConf) o; + return dateMode == dateConf.dateMode && + Objects.equals(startDate, dateConf.startDate) && + Objects.equals(endDate, dateConf.endDate) && + Objects.equals(unit, dateConf.unit) && + Objects.equals(period, dateConf.period); + } + + @Override + public int hashCode() { + return Objects.hash(dateMode, startDate, endDate, unit, period); + } + public enum DateMode { /** * date mode - * 1 - BETWEEN_CONTINUOUS, continuous static value, [startDate, endDate] - * 2 - LIST_DISCRETE, discrete static value, [dateList] - * 3 - RECENT_UNITS, dynamic time related to the actual available time of the element, [unit, period] - * 4 - AVAILABLE_TIME, dynamic time which guaranteed to query some data, [startDate, endDate] + * 1 - BETWEEN, continuous static value, [startDate, endDate] + * 2 - LIST, discrete static value, [dateList] + * 3 - RECENT, dynamic time related to the actual available time of the element, [unit, period] + * 4 - AVAILABLE, dynamic time which guaranteed to query some data, [startDate, endDate] */ - BETWEEN_CONTINUOUS, LIST_DISCRETE, RECENT_UNITS, AVAILABLE_TIME + BETWEEN, LIST, RECENT, AVAILABLE } @Override diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/RatioOverType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/RatioOverType.java index 2b58b7062..5083aaebe 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/RatioOverType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/RatioOverType.java @@ -2,9 +2,9 @@ package com.tencent.supersonic.common.pojo.enums; public enum RatioOverType { DAY_ON_DAY("日环比"), - WEEK_ON_DAY("周同比"), + WEEK_ON_DAY("周环比"), WEEK_ON_WEEK("周环比"), - MONTH_ON_WEEK("月同比"), + MONTH_ON_WEEK("月环比"), MONTH_ON_MONTH("月环比"), YEAR_ON_MONTH("年同比"), YEAR_ON_YEAR("年环比"); diff --git a/docs/images/supersonic_components.png b/docs/images/supersonic_components.png index c5cfad5aa..ef528d99b 100644 Binary files a/docs/images/supersonic_components.png and b/docs/images/supersonic_components.png differ diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index b61047efd..2821fcf14 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -1,7 +1,8 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\ com.tencent.supersonic.chat.mapper.HanlpDictMapper, \ com.tencent.supersonic.chat.mapper.FuzzyNameMapper, \ - com.tencent.supersonic.chat.mapper.QueryFilterMapper + com.tencent.supersonic.chat.mapper.QueryFilterMapper, \ + com.tencent.supersonic.chat.mapper.EntityMapper com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.rule.QueryModeParser, \ @@ -20,5 +21,8 @@ com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.parser.function.DomainResolver=\ com.tencent.supersonic.chat.parser.function.HeuristicDomainResolver -com.tencent.supersonic.auth.authentication.domain.interceptor.AuthenticationInterceptor=\ - com.tencent.supersonic.auth.authentication.domain.interceptor.DefaultAuthenticationInterceptor +com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ + com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor + +com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ + com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor \ No newline at end of file diff --git a/launchers/chat/src/main/resources/db/chat-data-h2.sql b/launchers/chat/src/main/resources/db/chat-data-h2.sql index 65155e3e4..8c65577f1 100644 --- a/launchers/chat/src/main/resources/db/chat-data-h2.sql +++ b/launchers/chat/src/main/resources/db/chat-data-h2.sql @@ -27,6 +27,3 @@ insert into s2_chat_query (`question_id`,`create_time`,`query_text`,`user_name`, insert into s2_chat_query (`question_id`,`create_time`,`query_text`,`user_name`,`query_state`,`chat_id`,`query_response`,`score`,`feedback`) VALUES(4, '2023-06-10 10:41:18.589','alice 访问次数','admin',0,2,'{"queryMode":"METRIC_FILTER","querySql":"SELECT `sys_imp_date` , `pv` FROM ( SELECT `sys_imp_date` , `s2_pv_uv_statis_pv` AS `pv` FROM ( SELECT SUM ( `s2_pv_uv_statis_pv` ) AS `s2_pv_uv_statis_pv` , `sys_imp_date` FROM ( SELECT `user_name` , `pv` AS `s2_pv_uv_statis_pv` , `imp_date` AS `sys_imp_date` FROM ( SELECT `imp_date` , `user_name` , `page` , 1 AS `pv` , `user_name` AS `uv` FROM `s2_pv_uv_statis` ) AS `s2_pv_uv_statis` ) AS `src00_s2_pv_uv_statis_b825` WHERE ( `sys_imp_date` >= ''2023-06-03'' AND `sys_imp_date` <= ''2023-06-09'' AND `user_name` = ''alice'' ) GROUP BY `sys_imp_date` ) AS `s2_pv_uv_statis_0` ) AS `s2_pv_uv_statis_1` LIMIT 10","queryState":0,"queryColumns":[{"name":"date","type":"VARCHAR","nameEn":"sys_imp_date","showType":"DATE","authorized":true},{"name":"访问次数","type":"BIGINT","nameEn":"pv","showType":"NUMBER","authorized":true}],"entityInfo":{"domainInfo":{"itemId":1,"name":"超音数","bizName":"supersonic","words":["用户","用户姓名"],"primaryEntityBizName":"user_name"},"dimensions":[{"itemId":1,"name":"部门","bizName":"department","value":"sales"},{"itemId":2,"name":"用户名","bizName":"user_name","value":"alice"}],"metrics":[{"itemId":2,"name":"访问次数","bizName":"pv","value":"2"}],"entityId":"alice"},"chatContext":{"queryMode":"METRIC_FILTER","domainId":1,"domainName":"超音数","entity":0,"metrics":[{"id":2,"name":"访问次数","bizName":"pv","status":1,"sensitiveLevel":0}],"dimensions":[{"bizName":"sys_imp_date","status":1,"sensitiveLevel":0}],"dimensionFilters":[{"bizName":"user_name","name":"用户名","operator":"=","value":"alice","elementID":2}],"metricFilters":[],"orders":[],"dateInfo":{"dateMode":"RECENT_UNITS","startDate":"2023-06-03","endDate":"2023-06-09","dateList":[],"unit":7,"period":"DAY"},"limit":10,"nativeQuery":false},"queryResults":[{"sys_imp_date":"2023-06-03","pv":2},{"sys_imp_date":"2023-06-04","pv":2},{"sys_imp_date":"2023-06-06","pv":2},{"sys_imp_date":"2023-06-07","pv":2},{"sys_imp_date":"2023-06-08","pv":5},{"sys_imp_date":"2023-06-09","pv":2}]}',0,''); insert into s2_chat_query (`question_id`,`create_time`,`query_text`,`user_name`,`query_state`,`chat_id`,`query_response`,`score`,`feedback`) VALUES(5, '2023-06-10 10:41:48.211','停留时长','admin',0,2,'{"queryMode":"METRIC_FILTER","querySql":"SELECT `sys_imp_date` , `stay_hours` FROM ( SELECT `sys_imp_date` , `s2_stay_time_statis_stay_hours` AS `stay_hours` FROM ( SELECT SUM ( `s2_stay_time_statis_stay_hours` ) AS `s2_stay_time_statis_stay_hours` , `sys_imp_date` FROM ( SELECT `user_name` , `stay_hours` AS `s2_stay_time_statis_stay_hours` , `imp_date` AS `sys_imp_date` FROM ( SELECT `imp_date` , `page` , `user_name` , `stay_hours` FROM `s2_stay_time_statis` ) AS `s2_stay_time_statis` ) AS `src00_s2_stay_time_statis_df18` WHERE ( `sys_imp_date` >= ''2023-06-03'' AND `sys_imp_date` <= ''2023-06-09'' AND `user_name` = ''alice'' ) GROUP BY `sys_imp_date` ) AS `s2_stay_time_statis_0` ) AS `s2_stay_time_statis_1` LIMIT 10","queryState":0,"queryColumns":[{"name":"date","type":"VARCHAR","nameEn":"sys_imp_date","showType":"DATE","authorized":true},{"name":"停留时长","type":"DOUBLE","nameEn":"stay_hours","showType":"NUMBER","authorized":true}],"entityInfo":{"domainInfo":{"itemId":1,"name":"超音数","bizName":"supersonic","words":["用户","用户姓名"],"primaryEntityBizName":"user_name"},"dimensions":[{"itemId":1,"name":"部门","bizName":"department","value":"sales"},{"itemId":2,"name":"用户名","bizName":"user_name","value":"alice"}],"metrics":[{"itemId":2,"name":"访问次数","bizName":"pv","value":"2"}],"entityId":"alice"},"chatContext":{"queryMode":"METRIC_FILTER","domainId":1,"domainName":"超音数","entity":0,"metrics":[{"id":1,"name":"停留时长","bizName":"stay_hours","status":1,"sensitiveLevel":0}],"dimensions":[{"bizName":"sys_imp_date","status":1,"sensitiveLevel":0}],"dimensionFilters":[{"bizName":"user_name","name":"用户名","operator":"=","value":"alice","elementID":2}],"metricFilters":[],"orders":[],"dateInfo":{"dateMode":"RECENT_UNITS","startDate":"2023-06-03","endDate":"2023-06-09","dateList":[],"unit":7,"period":"DAY"},"limit":10,"nativeQuery":false},"queryResults":[{"sys_imp_date":"2023-06-03","stay_hours":0.5963801306980994},{"sys_imp_date":"2023-06-04","stay_hours":1.5120376931855422},{"sys_imp_date":"2023-06-06","stay_hours":3.7790223355266317},{"sys_imp_date":"2023-06-07","stay_hours":0.8654528466186735},{"sys_imp_date":"2023-06-08","stay_hours":0.9796159603778489},{"sys_imp_date":"2023-06-09","stay_hours":0.6705580511822682}]}',0,''); insert into s2_chat_query (`question_id`,`create_time`,`query_text`,`user_name`,`query_state`,`chat_id`,`query_response`,`score`,`feedback`) VALUES(6, '2023-06-10 10:42:02.184','访问','admin',0,2,'{"queryMode":"METRIC_FILTER","querySql":"SELECT `sys_imp_date` , `stay_hours` FROM ( SELECT `sys_imp_date` , `s2_stay_time_statis_stay_hours` AS `stay_hours` FROM ( SELECT SUM ( `s2_stay_time_statis_stay_hours` ) AS `s2_stay_time_statis_stay_hours` , `sys_imp_date` FROM ( SELECT `user_name` , `stay_hours` AS `s2_stay_time_statis_stay_hours` , `imp_date` AS `sys_imp_date` FROM ( SELECT `imp_date` , `page` , `user_name` , `stay_hours` FROM `s2_stay_time_statis` ) AS `s2_stay_time_statis` ) AS `src00_s2_stay_time_statis_df18` WHERE ( `sys_imp_date` >= ''2023-06-03'' AND `sys_imp_date` <= ''2023-06-09'' AND `user_name` = ''alice'' ) GROUP BY `sys_imp_date` ) AS `s2_stay_time_statis_0` ) AS `s2_stay_time_statis_1` LIMIT 10","queryState":0,"queryColumns":[{"name":"date","type":"VARCHAR","nameEn":"sys_imp_date","showType":"DATE","authorized":true},{"name":"停留时长","type":"DOUBLE","nameEn":"stay_hours","showType":"NUMBER","authorized":true}],"entityInfo":{"domainInfo":{"itemId":1,"name":"超音数","bizName":"supersonic","words":["用户","用户姓名"],"primaryEntityBizName":"user_name"},"dimensions":[{"itemId":1,"name":"部门","bizName":"department","value":"sales"},{"itemId":2,"name":"用户名","bizName":"user_name","value":"alice"}],"metrics":[{"itemId":2,"name":"访问次数","bizName":"pv","value":"2"}],"entityId":"alice"},"chatContext":{"queryMode":"METRIC_FILTER","domainId":1,"domainName":"超音数","entity":0,"metrics":[{"id":1,"name":"停留时长","bizName":"stay_hours","status":1,"sensitiveLevel":0}],"dimensions":[{"bizName":"sys_imp_date","status":1,"sensitiveLevel":0}],"dimensionFilters":[{"bizName":"user_name","name":"用户名","operator":"=","value":"alice","elementID":2}],"metricFilters":[],"orders":[],"dateInfo":{"dateMode":"RECENT_UNITS","startDate":"2023-06-03","endDate":"2023-06-09","dateList":[],"unit":7,"period":"DAY"},"limit":10,"nativeQuery":false},"queryResults":[{"sys_imp_date":"2023-06-03","stay_hours":0.5963801306980994},{"sys_imp_date":"2023-06-04","stay_hours":1.5120376931855422},{"sys_imp_date":"2023-06-06","stay_hours":3.7790223355266317},{"sys_imp_date":"2023-06-07","stay_hours":0.8654528466186735},{"sys_imp_date":"2023-06-08","stay_hours":0.9796159603778489},{"sys_imp_date":"2023-06-09","stay_hours":0.6705580511822682}]}',0,''); - -insert into s2_plugin (id, type, domain, pattern, parse_mode, name, created_at, created_by, updated_at, updated_by, config) VALUES (1, 'WEB_PAGE', 1, '访问情况', 'EMBEDDING_RECALL', '访问情况', '2023-06-11 19:36:47', 'admin', '2023-06-21 15:26:46', 'admin', '{"params":{"487C128A":"2"}, "url":"www.test.com"}'); -insert into s2_plugin (id, type, domain, pattern, parse_mode, name, created_at, created_by, updated_at, updated_by, config) VALUES (2, 'DSL', null, '', 'FUNCTION_CALL', '访问情况', '2023-06-11 19:36:47', 'admin', '2023-06-21 15:26:46', 'admin', ''); diff --git a/launchers/chat/src/main/resources/db/chat-schema-h2.sql b/launchers/chat/src/main/resources/db/chat-schema-h2.sql index 2842efa02..b0bb7d909 100644 --- a/launchers/chat/src/main/resources/db/chat-schema-h2.sql +++ b/launchers/chat/src/main/resources/db/chat-schema-h2.sql @@ -121,12 +121,14 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` `domain` varchar(100) NULL, `pattern` varchar(500) NULL, `parse_mode` varchar(100) NULL, + `parse_mode_config` LONGVARCHAR NULL, `name` varchar(100) NULL, `created_at` TIMESTAMP NULL, `created_by` varchar(100) null, `updated_at` TIMESTAMP NULL, `updated_by` varchar(100) NULL, `config` LONGVARCHAR NULL, + `comment` LONGVARCHAR NULL, PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_plugin IS 'plugin information table'; diff --git a/launchers/semantic/src/main/resources/META-INF/spring.factories b/launchers/semantic/src/main/resources/META-INF/spring.factories index 91b5a4e62..1d40fdda2 100644 --- a/launchers/semantic/src/main/resources/META-INF/spring.factories +++ b/launchers/semantic/src/main/resources/META-INF/spring.factories @@ -1,2 +1,6 @@ -com.tencent.supersonic.auth.authentication.domain.interceptor.AuthenticationInterceptor=\ - com.tencent.supersonic.auth.authentication.domain.interceptor.DefaultAuthenticationInterceptor \ No newline at end of file +com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ + com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor + +com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ + com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor + diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java index 43ad1e99a..e93264adc 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java @@ -1,63 +1,198 @@ package com.tencent.supersonic; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.*; +import com.tencent.supersonic.chat.api.pojo.response.ParseResp; +import com.tencent.supersonic.chat.parser.ParseMode; +import com.tencent.supersonic.chat.plugin.Plugin; +import com.tencent.supersonic.chat.query.plugin.ParamOption; +import com.tencent.supersonic.chat.query.plugin.WebBase; import com.tencent.supersonic.chat.service.ChatService; +import com.tencent.supersonic.chat.service.ConfigService; +import com.tencent.supersonic.chat.service.PluginService; import com.tencent.supersonic.chat.service.QueryService; +import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; +import java.util.*; + @Component @Slf4j -public class ConfigureDemo implements ApplicationListener { +public class ConfigureDemo implements ApplicationListener { @Autowired private QueryService queryService; @Autowired private ChatService chatService; + @Autowired + protected ConfigService configService; + @Autowired + private PluginService pluginService; + private User user = User.getFakeUser(); - public void addSampleChats()throws Exception { + private void parseAndExecute(int chatId, String queryText) throws Exception { + QueryReq queryRequest = new QueryReq(); + queryRequest.setQueryText(queryText); + queryRequest.setChatId(chatId); + queryRequest.setUser(User.getFakeUser()); + ParseResp parseResp = queryService.performParsing(queryRequest); + + ExecuteQueryReq executeReq = new ExecuteQueryReq(); + executeReq.setQueryText(queryRequest.getQueryText()); + executeReq.setParseInfo(parseResp.getSelectedParses().get(0)); + executeReq.setChatId(parseResp.getChatId()); + executeReq.setUser(queryRequest.getUser()); + queryService.performExecution(executeReq); + } + + public void addSampleChats() throws Exception { chatService.addChat(user, "样例对话1"); - QueryRequest queryRequest = new QueryRequest(); - queryRequest.setQueryText("超音数 访问次数"); - queryRequest.setChatId(1); - queryRequest.setUser(User.getFakeUser()); - queryService.executeQuery(queryRequest); - - queryRequest.setQueryText("按部门统计"); - queryService.executeQuery(queryRequest); - - queryRequest.setQueryText("查询近30天"); - queryService.executeQuery(queryRequest); + parseAndExecute(1, "超音数 访问次数"); + parseAndExecute(1, "按部门统计"); + parseAndExecute(1, "查询近30天"); } public void addSampleChats2() throws Exception { chatService.addChat(user, "样例对话2"); - QueryRequest queryRequest = new QueryRequest(); - queryRequest.setChatId(2); - queryRequest.setUser(User.getFakeUser()); - queryRequest.setQueryText("alice 停留时长"); - queryService.executeQuery(queryRequest); + parseAndExecute(2, "alice 停留时长"); + parseAndExecute(2, "对比alice和lucy的访问次数"); + parseAndExecute(2, "访问次数最高的部门"); + } - queryRequest.setQueryText("对比alice和lucy的访问次数"); - queryService.executeQuery(queryRequest); + public void addDemoChatConfig_1() { + ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); + chatConfigBaseReq.setDomainId(1L); - queryRequest.setQueryText("访问次数最高的部门"); - queryService.executeQuery(queryRequest); + ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); + ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); + List dimensionIds_0 = Arrays.asList(1L, 2L); + List metricIds_0 = Arrays.asList(1L); + chatDefaultConfigDetail.setDimensionIds(dimensionIds_0); + chatDefaultConfigDetail.setMetricIds(metricIds_0); + chatDefaultConfigDetail.setUnit(7); + chatDefaultConfigDetail.setPeriod("DAY"); + chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); + ItemVisibility visibility_0 = new ItemVisibility(); + chatDetailConfig.setVisibility(visibility_0); + chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); + + + ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); + ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); + List dimensionIds_1 = Arrays.asList(1L, 2L); + List metricIds_1 = Arrays.asList(1L); + chatDefaultConfigAgg.setDimensionIds(dimensionIds_1); + chatDefaultConfigAgg.setMetricIds(metricIds_1); + chatDefaultConfigAgg.setUnit(7); + chatDefaultConfigAgg.setPeriod("DAY"); + chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT); + chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg); + ItemVisibility visibility_1 = new ItemVisibility(); + chatAggConfig.setVisibility(visibility_1); + chatConfigBaseReq.setChatAggConfig(chatAggConfig); + + List recommendedQuestions = new ArrayList<>(); + RecommendedQuestionReq recommendedQuestionReq_0 = new RecommendedQuestionReq("超音数访问次数"); + RecommendedQuestionReq recommendedQuestionReq_1 = new RecommendedQuestionReq("超音数访问人数"); + RecommendedQuestionReq recommendedQuestionReq_2 = new RecommendedQuestionReq("超音数按部门访问次数"); + recommendedQuestions.add(recommendedQuestionReq_0); + recommendedQuestions.add(recommendedQuestionReq_1); + recommendedQuestions.add(recommendedQuestionReq_2); + chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); + + configService.addConfig(chatConfigBaseReq, user); + } + + public void addDemoChatConfig_2() { + ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); + chatConfigBaseReq.setDomainId(2L); + + ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); + ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); + List dimensionIds_0 = Arrays.asList(4L, 5L, 6L, 7L); + List metricIds_0 = Arrays.asList(4L); + chatDefaultConfigDetail.setDimensionIds(dimensionIds_0); + chatDefaultConfigDetail.setMetricIds(metricIds_0); + chatDefaultConfigDetail.setUnit(7); + chatDefaultConfigDetail.setPeriod("DAY"); + chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); + ItemVisibility visibility_0 = new ItemVisibility(); + chatDetailConfig.setVisibility(visibility_0); + chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); + + + ChatAggConfigReq chatAggConfig = new ChatAggConfigReq(); + ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq(); + List dimensionIds_1 = Arrays.asList(4L, 5L, 6L, 7L); + List metricIds_1 = Arrays.asList(4L); + chatDefaultConfigAgg.setDimensionIds(dimensionIds_1); + chatDefaultConfigAgg.setMetricIds(metricIds_1); + chatDefaultConfigAgg.setUnit(7); + chatDefaultConfigAgg.setPeriod("DAY"); + chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT); + chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg); + ItemVisibility visibility_1 = new ItemVisibility(); + chatAggConfig.setVisibility(visibility_1); + chatConfigBaseReq.setChatAggConfig(chatAggConfig); + + List recommendedQuestions = new ArrayList<>(); + chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); + + configService.addConfig(chatConfigBaseReq, user); + } + + private void addPlugin_1() { + Plugin plugin_1 = new Plugin(); + plugin_1.setType("WEB_PAGE"); + plugin_1.setDomainList(Arrays.asList(1L)); + plugin_1.setPattern("访问情况"); + plugin_1.setParseModeConfig(null); + plugin_1.setName("访问情况"); + WebBase webBase = new WebBase(); + webBase.setUrl("www.test.com"); + ParamOption paramOption = new ParamOption(); + paramOption.setKey("name"); + paramOption.setParamType(ParamOption.ParamType.SEMANTIC); + paramOption.setElementId(2L); + paramOption.setDomainId(1L); + List paramOptions = Arrays.asList(paramOption); + webBase.setParamOptions(paramOptions); + plugin_1.setConfig(JsonUtil.toString(webBase)); + + pluginService.createPlugin(plugin_1, user); + } + + private void addPlugin_2() { + Plugin plugin_1 = new Plugin(); + plugin_1.setType("DSL"); + plugin_1.setDomainList(new ArrayList<>()); + plugin_1.setPattern(""); + plugin_1.setParseMode(ParseMode.FUNCTION_CALL); + plugin_1.setParseModeConfig(null); + plugin_1.setName("访问情况"); + plugin_1.setConfig(""); + pluginService.createPlugin(plugin_1, user); } @Override public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { try { + addDemoChatConfig_1(); + addDemoChatConfig_2(); + addPlugin_1(); + addPlugin_2(); addSampleChats(); addSampleChats2(); } catch (Exception e) { - log.error("Failed to add sample chats"); + log.error("Failed to add sample chats", e); } } + + } diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 4130a3f2c..c433857e6 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -1,7 +1,8 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\ com.tencent.supersonic.chat.mapper.HanlpDictMapper, \ com.tencent.supersonic.chat.mapper.FuzzyNameMapper, \ - com.tencent.supersonic.chat.mapper.QueryFilterMapper + com.tencent.supersonic.chat.mapper.QueryFilterMapper, \ + com.tencent.supersonic.chat.mapper.EntityMapper com.tencent.supersonic.chat.api.component.SemanticParser=\ com.tencent.supersonic.chat.parser.rule.QueryModeParser, \ @@ -21,5 +22,8 @@ com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.parser.function.DomainResolver=\ com.tencent.supersonic.chat.parser.function.HeuristicDomainResolver -com.tencent.supersonic.auth.authentication.domain.interceptor.AuthenticationInterceptor=\ - com.tencent.supersonic.auth.authentication.domain.interceptor.DefaultAuthenticationInterceptor +com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ + com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor + +com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ + com.tencent.supersonic.auth.authentication.adaptor.DefaultUserAdaptor diff --git a/launchers/standalone/src/main/resources/db/data-h2.sql b/launchers/standalone/src/main/resources/db/data-h2.sql index b09845661..27b296b13 100644 --- a/launchers/standalone/src/main/resources/db/data-h2.sql +++ b/launchers/standalone/src/main/resources/db/data-h2.sql @@ -4,21 +4,6 @@ insert into s2_user (id, `name`, password, display_name, email) values (2, 'jack insert into s2_user (id, `name`, password, display_name, email) values (3, 'tom','123456','tom','tom@xx.com'); insert into s2_user (id, `name`, password, display_name, email) values (4, 'lucy','123456','lucy','lucy@xx.com'); ---insert into s2_chat_config (`id` ,`domain_id` ,`default_metrics`,`visibility`,`entity_info` ,`dictionary_info`,`created_at`,`updated_at`,`created_by`,`updated_by`,`status` ) values (1,1,'[{"metricId":1,"unit":7,"period":"DAY"}]','{"blackDimIdList":[],"blackMetricIdList":[]}','{"entityIds":[2],"names":["用户","用户姓名"],"detailData":{"dimensionIds":[1,2],"metricIds":[2]}}','[{"itemId":1,"type":"DIMENSION","blackList":[],"isDictInfo":true},{"itemId":2,"type":"DIMENSION","blackList":[],"isDictInfo":true},{"itemId":3,"type":"DIMENSION","blackList":[],"isDictInfo":true}]','2023-05-24 18:00:00','2023-05-25 11:00:00','admin','admin',1); -insert into s2_chat_config (`id` ,`domain_id` ,`chat_detail_config`,`chat_agg_config`, `recommended_questions`, `created_at`,`updated_at`,`created_by`,`updated_by`,`status` ) -values (1,1, - '{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":2,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[1,2],"metricIds":[1],"unit":7,"period":"DAY"},"entity":null}', - '{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":2,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[1,2],"metricIds":[1],"unit":7,"period":"DAY"}}', - '[{"question":"超音数访问次数"},{"question":"超音数访问人数"},{"question":"超音数按部门访问次数"}]', - '2023-05-24 18:00:00','2023-05-25 11:00:00','admin','admin',1); -insert into s2_chat_config (`id` ,`domain_id` ,`chat_detail_config`,`chat_agg_config`,`created_at`,`updated_at`,`created_by`,`updated_by`,`status` ) -values (2,2, - '{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":7,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[4,5,6,7],"metricIds":[4],"unit":7,"period":"DAY"},"entity":{"entityId":7,"names":["歌手","艺人"]}}', - '{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":7,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[4,5,6,7],"metricIds":[4],"unit":7,"period":"DAY"}}', - '2023-05-24 18:00:00','2023-05-25 11:00:00','admin','admin',1); - -insert into s2_plugin (id, `type`, `domain`, pattern, parse_mode, `name`, created_at, created_by, updated_at, updated_by, config) VALUES (1, 'WEB_PAGE', 1, '访问情况', 'EMBEDDING_RECALL', '访问情况', '2023-06-11 19:36:47', 'admin', '2023-06-21 15:26:46', 'admin', '{"params":{"487C128A":"2"}, "url":"www.test.com"}'); - -- sample models insert into s2_database (id, domain_id , `name`, description, `type` ,config ,created_at ,created_by ,updated_at ,updated_by) VALUES(1, 1, 'H2数据实例', '', 'h2', '{"password":"semantic","url":"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false","userName":"root"}', '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin'); insert into s2_database (id, domain_id , `name`, description, `type` ,config ,created_at ,created_by ,updated_at ,updated_by) VALUES(2, 2, 'H2数据实例', '', 'h2', '{"password":"semantic","url":"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false","userName":"root"}', '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin'); @@ -37,7 +22,7 @@ insert into s2_dimension (id , domain_id, datasource_id, `name`, biz_name, descr insert into s2_dimension (id , domain_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(6, 2, 4, '风格', 'genre', '风格', 1, 2, 'categorical', NULL, 'genre', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY'); insert into s2_dimension (id , domain_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(7, 2, 4, '歌手名', 'singer_name', '歌手名', 1, 2, 'categorical', NULL, 'singer_name', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY'); insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org) VALUES(1, '超音数', 'supersonic', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin' ); -insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org) VALUES(2, '艺人库', 'singer', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin' ); +insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org, entity) VALUES(2, '艺人库', 'singer', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin','{"entityId": 7, "names": ["歌手", "艺人"]}' ); insert into s2_metric (id, domain_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(1, 1, '停留时长', 'stay_hours', '停留时长', 1, 2, 'ATOMIC', '{"expr":"s2_stay_time_statis_stay_hours","measures":[{"agg":"sum","expr":"stay_hours","isCreateMetric":1,"datasourceId":1,"bizName":"s2_stay_time_statis_stay_hours","name":"s2_stay_time_statis_stay_hours"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL ); insert into s2_metric (id, domain_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(2, 1, '访问次数', 'pv', '访问次数', 1, 0, 'ATOMIC', ' {"expr":"s2_pv_uv_statis_pv","measures":[{"agg":"sum","bizName":"s2_pv_uv_statis_pv","datasourceId":2,"expr":"pv","isCreateMetric":1,"name":"s2_pv_uv_statis_pv"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL ); insert into s2_metric (id, domain_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(3, 1, '访问人数', 'uv', '访问人数', 1, 0, 'ATOMIC', ' {"expr":"s2_pv_uv_statis_uv","measures":[{"agg":"count_distinct","bizName":"s2_pv_uv_statis_uv","datasourceId":2,"expr":"uv","isCreateMetric":1,"name":"s2_pv_uv_statis_uv"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL ); diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index a5ffde8e6..a625909f8 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -55,40 +55,6 @@ CREATE TABLE IF NOT EXISTS `s2_chat_config` ( COMMENT ON TABLE s2_chat_config IS 'chat config information table '; - - -CREATE TABLE IF NOT EXISTS `s2_dictionary` ( - `id` INT NOT NULL AUTO_INCREMENT, - `domain_id` INT NOT NULL , - `dim_value_infos` LONGVARCHAR , -- dimension value setting information - `created_at` TIMESTAMP NOT NULL , - `updated_at` TIMESTAMP NOT NULL , - `created_by` varchar(100) NOT NULL , - `updated_by` varchar(100) DEFAULT NULL , - `status` INT NOT NULL DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted - PRIMARY KEY (`id`), - UNIQUE (domain_id) - ); -COMMENT ON TABLE s2_dictionary IS 'dictionary configuration information table'; - - -CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( - `id` INT NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL , -- task name - `description` varchar(255) , - `command`LONGVARCHAR NOT NULL , -- task Request Parameters - `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 - `status` INT NOT NULL , -- the final status of the task - `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , - `created_by` varchar(100) NOT NULL , - `progress` DOUBLE default 0.00 , -- task real-time progress - `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds - `message` LONGVARCHAR , -- remark related information - PRIMARY KEY (`id`) - ); -COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; - - create table s2_user ( id INT AUTO_INCREMENT, @@ -321,12 +287,14 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` `domain` varchar(100) NULL, `pattern` varchar(500) NULL, `parse_mode` varchar(100) NULL, + `parse_mode_config` LONGVARCHAR NULL, `name` varchar(100) NULL, `created_at` TIMESTAMP NULL, `created_by` varchar(100) null, `updated_at` TIMESTAMP NULL, `updated_by` varchar(100) NULL, `config` LONGVARCHAR NULL, + `comment` LONGVARCHAR NULL, PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_plugin IS 'plugin information table'; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/BaseQueryTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/BaseQueryTest.java index 0d03808da..3fe268d76 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/BaseQueryTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/BaseQueryTest.java @@ -1,19 +1,18 @@ package com.tencent.supersonic.integration; import com.tencent.supersonic.StandaloneLauncher; -import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; +import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.QueryService; -import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.util.DataUtils; -import org.apache.commons.lang3.RandomStringUtils; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -22,13 +21,10 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import java.time.LocalDate; -import java.util.Comparator; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; @RunWith(SpringRunner.class) @SpringBootTest(classes = StandaloneLauncher.class) @@ -38,6 +34,7 @@ public class BaseQueryTest { protected final int unit = 7; protected final String startDay = LocalDate.now().plusDays(-unit).toString(); protected final String endDay = LocalDate.now().plusDays(-1).toString(); + protected final String period = "DAY"; @Autowired @Qualifier("chatQueryService") @@ -47,29 +44,39 @@ public class BaseQueryTest { @Autowired protected ConfigService configService; - protected Integer getNewChat(String chatName) { - chatService.addChat(User.getFakeUser(), chatName); - Optional chatId = chatService.getAll(User.getFakeUser().getName()).stream().map(c -> c.getChatId()).sorted(Comparator.reverseOrder()).findFirst(); - if (chatId.isPresent()) { - return chatId.get().intValue(); - } - return 1; - } - protected QueryResult submitMultiTurnChat(String queryText) throws Exception { - QueryRequest queryContextReq = DataUtils.getQueryContextReq(20, queryText); - return queryService.executeQuery(queryContextReq); + ParseResp parseResp = submitParse(queryText); + + ExecuteQueryReq request = new ExecuteQueryReq(); + request.setChatId(parseResp.getChatId()); + request.setQueryText(parseResp.getQueryText()); + request.setUser(DataUtils.getUser()); + request.setParseInfo(parseResp.getSelectedParses().get(0)); + + return queryService.performExecution(request); } protected QueryResult submitNewChat(String queryText) throws Exception { - chatService.addChat(User.getFakeUser(), RandomStringUtils.random(5)); + ParseResp parseResp = submitParse(queryText); - ChatContext chatContext = chatService.getOrCreateContext(10); + ExecuteQueryReq request = new ExecuteQueryReq(); + request.setChatId(parseResp.getChatId()); + request.setQueryText(parseResp.getQueryText()); + request.setUser(DataUtils.getUser()); + request.setParseInfo(parseResp.getSelectedParses().get(0)); + + QueryResult result = queryService.performExecution(request); + + ChatContext chatContext = chatService.getOrCreateContext(parseResp.getChatId()); chatContext.setParseInfo(new SemanticParseInfo()); chatService.updateContext(chatContext); - QueryRequest queryContextReq = DataUtils.getQueryContextReq(10, queryText); - return queryService.executeQuery(queryContextReq); + return result; + } + + protected ParseResp submitParse(String queryText) { + QueryReq queryContextReq = DataUtils.getQueryContextReq(10, queryText); + return queryService.performParsing(queryContextReq); } protected void assertSchemaElements(Set expected, Set actual) { @@ -81,17 +88,6 @@ public class BaseQueryTest { assertEquals(expectedNames, actualNames); } - protected void assertDateConf(DateConf expected, DateConf actual) { - Boolean timeFilterExist = expected.getStartDate().equals(actual.getStartDate()) - && expected.getEndDate().equals(actual.getEndDate()) - && expected.getDateMode().equals(actual.getDateMode()) - || expected.getUnit().equals(actual.getUnit()) && - expected.getDateMode().equals(actual.getDateMode()) && - expected.getPeriod().equals(actual.getPeriod()); - - assertTrue(timeFilterExist); - } - protected void assertQueryResult(QueryResult expected, QueryResult actual) { SemanticParseInfo expectedParseInfo = expected.getChatContext(); SemanticParseInfo actualParseInfo = actual.getChatContext(); @@ -106,7 +102,7 @@ public class BaseQueryTest { assertEquals(expectedParseInfo.getDimensionFilters(), actualParseInfo.getDimensionFilters()); assertEquals(expectedParseInfo.getMetricFilters(), actualParseInfo.getMetricFilters()); - assertDateConf(expectedParseInfo.getDateInfo(), actualParseInfo.getDateInfo()); + assertEquals(expectedParseInfo.getDateInfo(), actualParseInfo.getDateInfo()); } } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java index 8d0737a32..b522827f7 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java @@ -1,23 +1,46 @@ package com.tencent.supersonic.integration; +import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.query.rule.entity.EntityFilterQuery; +import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.util.DataUtils; import org.junit.Test; - import java.util.ArrayList; import java.util.List; - import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; public class EntityQueryTest extends BaseQueryTest { @Test - public void queryTest_ENTITY_LIST_FILTER()throws Exception { + public void queryTest_METRIC_ENTITY_QUERY() throws Exception { + QueryResult actualResult = submitNewChat("艺人周杰伦的播放量"); + + QueryResult expectedResult = new QueryResult(); + SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); + expectedResult.setChatContext(expectedParseInfo); + + expectedResult.setQueryMode(MetricEntityQuery.QUERY_MODE); + expectedParseInfo.setAggType(NONE); + + QueryFilter dimensionFilter = DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 7L); + expectedParseInfo.getDimensionFilters().add(dimensionFilter); + + SchemaElement metric = SchemaElement.builder().name("播放量").build(); + expectedParseInfo.getMetrics().add(metric); + + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, 7, period, startDay, endDay)); + expectedParseInfo.setNativeQuery(false); + + assertQueryResult(expectedResult, actualResult); + } + + @Test + public void queryTest_ENTITY_LIST_FILTER() throws Exception { QueryResult actualResult = submitNewChat("爱情、流行类型的艺人"); QueryResult expectedResult = new QueryResult(); @@ -33,7 +56,19 @@ public class EntityQueryTest extends BaseQueryTest { QueryFilter dimensionFilter = DataUtils.getFilter("genre", FilterOperatorEnum.IN, list, "风格", 6L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(1, DateConf.DateMode.RECENT_UNITS, "DAY")); + SchemaElement metric = SchemaElement.builder().name("播放量").build(); + expectedParseInfo.getMetrics().add(metric); + + SchemaElement dim1 = SchemaElement.builder().name("歌手名").build(); + SchemaElement dim2 = SchemaElement.builder().name("活跃区域").build(); + SchemaElement dim3 = SchemaElement.builder().name("风格").build(); + SchemaElement dim4 = SchemaElement.builder().name("代表作").build(); + expectedParseInfo.getDimensions().add(dim1); + expectedParseInfo.getDimensions().add(dim2); + expectedParseInfo.getDimensions().add(dim3); + expectedParseInfo.getDimensions().add(dim4); + + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, startDay, startDay)); expectedParseInfo.setNativeQuery(true); assertQueryResult(expectedResult, actualResult); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java index 611e035fc..86af90a9d 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java @@ -4,9 +4,9 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.config.ChatConfigEditReqReq; -import com.tencent.supersonic.chat.config.ChatConfigResp; -import com.tencent.supersonic.chat.config.ItemVisibility; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; +import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; import com.tencent.supersonic.chat.query.rule.metric.MetricDomainQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery; @@ -42,7 +42,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -61,7 +61,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -81,7 +81,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -106,7 +106,7 @@ public class MetricQueryTest extends BaseQueryTest { QueryFilter dimensionFilter = DataUtils.getFilter("user_name", FilterOperatorEnum.IN, list, "用户名", 2L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -126,7 +126,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户名")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(3, DateConf.DateMode.RECENT_UNITS, "DAY")); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(3, DateConf.DateMode.RECENT, "DAY")); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -145,7 +145,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -157,7 +157,7 @@ public class MetricQueryTest extends BaseQueryTest { DateFormat textFormat = new SimpleDateFormat("yyyy年mm月dd日"); String dateStr = textFormat.format(format.parse(startDay)); - QueryResult actualResult = submitNewChat(String.format("想知道{}alice的访问次数", dateStr)); + QueryResult actualResult = submitNewChat(String.format("想知道%salice的访问次数", dateStr)); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -171,7 +171,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, startDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -198,7 +198,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java index 7902a015d..9488da4a7 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java @@ -34,7 +34,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -57,7 +57,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -80,7 +80,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getDimensionFilters().add(DataUtils.getFilter("user_name", FilterOperatorEnum.EQUALS, "lucy", "用户名", 2L)); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -101,7 +101,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("停留时长")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, endDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -124,7 +124,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("停留时长")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN_CONTINUOUS, startDay, startDay)); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay)); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); @@ -145,7 +145,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("停留时长")); expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); - expectedParseInfo.setDateInfo(DataUtils.getDateConf(30, DateConf.DateMode.RECENT_UNITS, "DAY")); + expectedParseInfo.setDateInfo(DataUtils.getDateConf(30, DateConf.DateMode.RECENT, "DAY")); expectedParseInfo.setNativeQuery(false); assertQueryResult(expectedResult, actualResult); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/BasePluginTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/BasePluginTest.java index bb0e00d19..7c0f02508 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/BasePluginTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/BasePluginTest.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.integration.plugin; import com.tencent.supersonic.StandaloneLauncher; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.api.pojo.response.QueryState; -import com.tencent.supersonic.chat.query.plugin.WebBase; +import com.tencent.supersonic.chat.query.plugin.WebBaseResult; import com.tencent.supersonic.chat.query.plugin.webpage.WebPageQuery; import com.tencent.supersonic.chat.query.plugin.webpage.WebPageResponse; import lombok.extern.slf4j.Slf4j; @@ -25,10 +25,10 @@ public class BasePluginTest { Assert.assertEquals(queryResult.getQueryState(), QueryState.SUCCESS); Assert.assertEquals(queryResult.getQueryMode(), WebPageQuery.QUERY_MODE); WebPageResponse webPageResponse = (WebPageResponse) queryResult.getResponse(); - WebBase webPage = webPageResponse.getWebPage(); + WebBaseResult webPage = webPageResponse.getWebPage(); Assert.assertEquals(webPage.getUrl(), "www.test.com"); - Map valueParams = webPage.getValueParams(); - Assert.assertEquals(valueParams.get("name"), "alice"); + Assert.assertEquals(1, webPage.getParams().size()); + Assert.assertEquals("alice", webPage.getParams().get(0).getValue()); } } \ No newline at end of file diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java index bb6b8394c..8e231a187 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/plugin/PluginRecognizeTest.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.integration.plugin; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryFilters; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.parser.embedding.EmbeddingConfig; import com.tencent.supersonic.chat.plugin.PluginManager; @@ -29,7 +29,7 @@ public class PluginRecognizeTest extends BasePluginTest{ public void webPageRecognize() throws Exception { PluginMockConfiguration.mockEmbeddingRecognize(pluginManager, "最近的访问情况怎么样","1"); PluginMockConfiguration.mockEmbeddingUrl(embeddingConfig); - QueryRequest queryContextReq = DataUtils.getQueryContextReq(1000, "alice最近的访问情况怎么样"); + QueryReq queryContextReq = DataUtils.getQueryContextReq(1000, "alice最近的访问情况怎么样"); QueryResult queryResult = queryService.executeQuery(queryContextReq); assertPluginRecognizeResult(queryResult); } @@ -38,11 +38,12 @@ public class PluginRecognizeTest extends BasePluginTest{ public void webPageRecognizeWithQueryFilter() throws Exception { PluginMockConfiguration.mockEmbeddingRecognize(pluginManager, "在超音数最近的情况怎么样","1"); PluginMockConfiguration.mockEmbeddingUrl(embeddingConfig); - QueryRequest queryRequest = DataUtils.getQueryContextReq(1000, "在超音数最近的情况怎么样"); + QueryReq queryRequest = DataUtils.getQueryContextReq(1000, "在超音数最近的情况怎么样"); QueryFilters queryFilters = new QueryFilters(); QueryFilter queryFilter = new QueryFilter(); queryFilter.setElementID(2L); queryFilter.setValue("alice"); + queryRequest.setDomainId(1L); queryFilters.getFilters().add(queryFilter); queryRequest.setQueryFilters(queryFilters); QueryResult queryResult = queryService.executeQuery(queryRequest); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index 7269133f0..22f8a96cc 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -4,19 +4,27 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; -import com.tencent.supersonic.chat.api.pojo.request.QueryRequest; +import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import java.util.Set; +import static java.time.LocalDate.now; + public class DataUtils { - public static QueryRequest getQueryContextReq(Integer id, String query) { - QueryRequest queryContextReq = new QueryRequest(); + private static final User user_test = new User(1L, "admin", "admin", "admin@email"); + + public static User getUser() { + return user_test; + } + + public static QueryReq getQueryContextReq(Integer id, String query) { + QueryReq queryContextReq = new QueryReq(); queryContextReq.setQueryText(query);//"alice的访问次数" queryContextReq.setChatId(id); - queryContextReq.setUser(new User(1L, "admin", "admin", "admin@email")); + queryContextReq.setUser(user_test); return queryContextReq; } @@ -64,6 +72,18 @@ public class DataUtils { dateInfo.setUnit(unit); dateInfo.setDateMode(dateMode); dateInfo.setPeriod(period); + dateInfo.setStartDate(now().plusDays(-unit).toString()); + dateInfo.setEndDate(now().toString()); + return dateInfo; + } + + public static DateConf getDateConf(DateConf.DateMode dateMode, Integer unit, String period, String startDate, String endDate) { + DateConf dateInfo = new DateConf(); + dateInfo.setUnit(unit); + dateInfo.setDateMode(dateMode); + dateInfo.setPeriod(period); + dateInfo.setStartDate(startDate); + dateInfo.setEndDate(endDate); return dateInfo; } diff --git a/launchers/standalone/src/test/resources/META-INF/spring.factories b/launchers/standalone/src/test/resources/META-INF/spring.factories index c9a823edb..41d5b4236 100644 --- a/launchers/standalone/src/test/resources/META-INF/spring.factories +++ b/launchers/standalone/src/test/resources/META-INF/spring.factories @@ -20,5 +20,5 @@ com.tencent.supersonic.chat.query.QuerySelector=\ com.tencent.supersonic.chat.application.query.DomainResolver=\ com.tencent.supersonic.chat.application.query.HeuristicDomainResolver -com.tencent.supersonic.auth.authentication.domain.interceptor.AuthenticationInterceptor=\ - com.tencent.supersonic.auth.authentication.domain.interceptor.DefaultAuthenticationInterceptor +com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\ + com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor diff --git a/launchers/standalone/src/test/resources/db/data-h2.sql b/launchers/standalone/src/test/resources/db/data-h2.sql index 2c9f9b80f..502793606 100644 --- a/launchers/standalone/src/test/resources/db/data-h2.sql +++ b/launchers/standalone/src/test/resources/db/data-h2.sql @@ -5,14 +5,14 @@ insert into s2_user (id, `name`, password, display_name, email) values (3, 'tom' insert into s2_user (id, `name`, password, display_name, email) values (4, 'lucy','123456','lucy','lucy@xx.com'); --insert into s2_chat_config (`id` ,`domain_id` ,`default_metrics`,`visibility`,`entity_info` ,`dictionary_info`,`created_at`,`updated_at`,`created_by`,`updated_by`,`status` ) values (1,1,'[{"metricId":1,"unit":7,"period":"DAY"}]','{"blackDimIdList":[],"blackMetricIdList":[]}','{"entityIds":[2],"names":["用户","用户姓名"],"detailData":{"dimensionIds":[1,2],"metricIds":[2]}}','[{"itemId":1,"type":"DIMENSION","blackList":[],"isDictInfo":true},{"itemId":2,"type":"DIMENSION","blackList":[],"isDictInfo":true},{"itemId":3,"type":"DIMENSION","blackList":[],"isDictInfo":true}]','2023-05-24 18:00:00','2023-05-25 11:00:00','admin','admin',1); -insert into s2_chat_config (`id` ,`domain_id` ,`chat_detail_config`,`chat_agg_config`,`created_at`,`updated_at`,`created_by`,`updated_by`,`status` ) -values (1,1,'{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":2,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[1,2],"metricIds":[1],"unit":7,"period":"DAY"},"entity":null}', - '{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":2,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[1,2],"metricIds":[1],"ratioMetricIds":[2],"unit":7,"period":"DAY"}}', - '2023-05-24 18:00:00','2023-05-25 11:00:00','admin','admin',1); -insert into s2_chat_config (`id` ,`domain_id` ,`chat_detail_config`,`chat_agg_config`,`created_at`,`updated_at`,`created_by`,`updated_by`,`status` ) -values (2,2,'{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":7,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[4,5,6,7],"metricIds":[4],"unit":7,"period":"DAY"},"entity":{"entityId":1,"names":["歌手","艺人"]}}', - '{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":7,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[4,5,6,7],"metricIds":[4],"unit":7,"period":"DAY"}}', - '2023-05-24 18:00:00','2023-05-25 11:00:00','admin','admin',1); +-- insert into s2_chat_config (`id` ,`domain_id` ,`chat_detail_config`,`chat_agg_config`,`created_at`,`updated_at`,`created_by`,`updated_by`,`status` ) +-- values (1,1,'{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":2,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[1,2],"metricIds":[1],"unit":7,"period":"DAY"}}', +-- '{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":2,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[1,2],"metricIds":[1],"ratioMetricIds":[2],"unit":7,"period":"DAY","timeMode":"RECENT"}}', +-- '2023-05-24 18:00:00','2023-05-25 11:00:00','admin','admin',1); +-- insert into s2_chat_config (`id` ,`domain_id` ,`chat_detail_config`,`chat_agg_config`,`created_at`,`updated_at`,`created_by`,`updated_by`,`status` ) +-- values (2,2,'{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":7,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[4,5,6,7],"metricIds":[4],"unit":7,"period":"DAY"}}', +-- '{"visibility":{"blackDimIdList":[],"blackMetricIdList":[]},"knowledgeInfos":[{"itemId":7,"type":"DIMENSION","searchEnable":true}],"chatDefaultConfig":{"dimensionIds":[4,5,6,7],"metricIds":[4],"unit":7,"period":"DAY","timeMode":"RECENT"}}', +-- '2023-05-24 18:00:00','2023-05-25 11:00:00','admin','admin',1); insert into s2_chat (chat_id, `chat_name`, create_time, last_time, creator,last_question,is_delete,is_top) values (1, '超音数访问统计','2023-06-10 10:00:52.495','2023-06-10 10:00:52','admin','您好，欢迎使用内容智能小Q','0','0'); insert into s2_chat (chat_id, `chat_name`, create_time, last_time, creator,last_question,is_delete,is_top) values (2, '用户访问统计','2023-06-10 10:01:04.528','2023-06-10 10:01:04','admin','您好，欢迎使用内容智能小Q','0','0'); @@ -27,8 +27,8 @@ insert into s2_chat_query (`question_id`,`create_time`,`query_text`,`user_name`, insert into s2_chat_query (`question_id`,`create_time`,`query_text`,`user_name`,`query_state`,`chat_id`,`query_response`,`score`,`feedback`) VALUES(5, '2023-06-10 10:41:48.211','停留时长','admin',0,2,'{"queryMode":"METRIC_FILTER","querySql":"SELECT `sys_imp_date` , `stay_hours` FROM ( SELECT `sys_imp_date` , `s2_stay_time_statis_stay_hours` AS `stay_hours` FROM ( SELECT SUM ( `s2_stay_time_statis_stay_hours` ) AS `s2_stay_time_statis_stay_hours` , `sys_imp_date` FROM ( SELECT `user_name` , `stay_hours` AS `s2_stay_time_statis_stay_hours` , `imp_date` AS `sys_imp_date` FROM ( SELECT `imp_date` , `page` , `user_name` , `stay_hours` FROM `s2_stay_time_statis` ) AS `s2_stay_time_statis` ) AS `src00_s2_stay_time_statis_df18` WHERE ( `sys_imp_date` >= ''2023-06-03'' AND `sys_imp_date` <= ''2023-06-09'' AND `user_name` = ''alice'' ) GROUP BY `sys_imp_date` ) AS `s2_stay_time_statis_0` ) AS `s2_stay_time_statis_1` LIMIT 10","queryState":0,"queryColumns":[{"name":"date","type":"VARCHAR","nameEn":"sys_imp_date","showType":"DATE","authorized":true},{"name":"停留时长","type":"DOUBLE","nameEn":"stay_hours","showType":"NUMBER","authorized":true}],"entityInfo":{"domainInfo":{"itemId":1,"name":"超音数","bizName":"supersonic","words":["用户","用户姓名"],"primaryEntityBizName":"user_name"},"dimensions":[{"itemId":1,"name":"部门","bizName":"department","value":"sales"},{"itemId":2,"name":"用户名","bizName":"user_name","value":"alice"}],"metrics":[{"itemId":2,"name":"访问次数","bizName":"pv","value":"2"}],"entityId":"alice"},"chatContext":{"queryMode":"METRIC_FILTER","domainId":1,"domainName":"超音数","entity":0,"metrics":[{"id":1,"name":"停留时长","bizName":"stay_hours","status":1,"sensitiveLevel":0}],"dimensions":[{"bizName":"sys_imp_date","status":1,"sensitiveLevel":0}],"dimensionFilters":[{"bizName":"user_name","name":"用户名","operator":"=","value":"alice","elementID":2}],"metricFilters":[],"orders":[],"dateInfo":{"dateMode":"RECENT_UNITS","startDate":"2023-06-03","endDate":"2023-06-09","dateList":[],"unit":7,"period":"DAY"},"limit":10,"nativeQuery":false},"queryResults":[{"sys_imp_date":"2023-06-03","stay_hours":0.5963801306980994},{"sys_imp_date":"2023-06-04","stay_hours":1.5120376931855422},{"sys_imp_date":"2023-06-06","stay_hours":3.7790223355266317},{"sys_imp_date":"2023-06-07","stay_hours":0.8654528466186735},{"sys_imp_date":"2023-06-08","stay_hours":0.9796159603778489},{"sys_imp_date":"2023-06-09","stay_hours":0.6705580511822682}]}',0,''); insert into s2_chat_query (`question_id`,`create_time`,`query_text`,`user_name`,`query_state`,`chat_id`,`query_response`,`score`,`feedback`) VALUES(6, '2023-06-10 10:42:02.184','访问','admin',0,2,'{"queryMode":"METRIC_FILTER","querySql":"SELECT `sys_imp_date` , `stay_hours` FROM ( SELECT `sys_imp_date` , `s2_stay_time_statis_stay_hours` AS `stay_hours` FROM ( SELECT SUM ( `s2_stay_time_statis_stay_hours` ) AS `s2_stay_time_statis_stay_hours` , `sys_imp_date` FROM ( SELECT `user_name` , `stay_hours` AS `s2_stay_time_statis_stay_hours` , `imp_date` AS `sys_imp_date` FROM ( SELECT `imp_date` , `page` , `user_name` , `stay_hours` FROM `s2_stay_time_statis` ) AS `s2_stay_time_statis` ) AS `src00_s2_stay_time_statis_df18` WHERE ( `sys_imp_date` >= ''2023-06-03'' AND `sys_imp_date` <= ''2023-06-09'' AND `user_name` = ''alice'' ) GROUP BY `sys_imp_date` ) AS `s2_stay_time_statis_0` ) AS `s2_stay_time_statis_1` LIMIT 10","queryState":0,"queryColumns":[{"name":"date","type":"VARCHAR","nameEn":"sys_imp_date","showType":"DATE","authorized":true},{"name":"停留时长","type":"DOUBLE","nameEn":"stay_hours","showType":"NUMBER","authorized":true}],"entityInfo":{"domainInfo":{"itemId":1,"name":"超音数","bizName":"supersonic","words":["用户","用户姓名"],"primaryEntityBizName":"user_name"},"dimensions":[{"itemId":1,"name":"部门","bizName":"department","value":"sales"},{"itemId":2,"name":"用户名","bizName":"user_name","value":"alice"}],"metrics":[{"itemId":2,"name":"访问次数","bizName":"pv","value":"2"}],"entityId":"alice"},"chatContext":{"queryMode":"METRIC_FILTER","domainId":1,"domainName":"超音数","entity":0,"metrics":[{"id":1,"name":"停留时长","bizName":"stay_hours","status":1,"sensitiveLevel":0}],"dimensions":[{"bizName":"sys_imp_date","status":1,"sensitiveLevel":0}],"dimensionFilters":[{"bizName":"user_name","name":"用户名","operator":"=","value":"alice","elementID":2}],"metricFilters":[],"orders":[],"dateInfo":{"dateMode":"RECENT_UNITS","startDate":"2023-06-03","endDate":"2023-06-09","dateList":[],"unit":7,"period":"DAY"},"limit":10,"nativeQuery":false},"queryResults":[{"sys_imp_date":"2023-06-03","stay_hours":0.5963801306980994},{"sys_imp_date":"2023-06-04","stay_hours":1.5120376931855422},{"sys_imp_date":"2023-06-06","stay_hours":3.7790223355266317},{"sys_imp_date":"2023-06-07","stay_hours":0.8654528466186735},{"sys_imp_date":"2023-06-08","stay_hours":0.9796159603778489},{"sys_imp_date":"2023-06-09","stay_hours":0.6705580511822682}]}',0,''); -insert into s2_plugin (id, `type`, `domain`, pattern, parse_mode, `name`, created_at, created_by, updated_at, updated_by, config) VALUES (1, 'WEB_PAGE', 1, '访问情况', 'EMBEDDING_RECALL', '访问情况', '2023-06-11 19:36:47', 'admin', '2023-06-21 15:26:46', 'admin', '{"params":{"name":"2"}, "url":"www.test.com"}'); -insert into s2_plugin (id, `type`, `domain`, pattern, parse_mode, `name`, created_at, created_by, updated_at, updated_by, config) VALUES (2, 'WEB_PAGE', 2, '播放表现', 'EMBEDDING_RECALL', '播放表现', '2023-06-11 19:36:47', 'admin', '2023-06-21 15:26:46', 'admin', '{"params":{"name":"7"}, "url":"www.test.com"}'); +-- insert into s2_plugin (id, `type`, `domain`, pattern, parse_mode, parse_mode_config, `name`, created_at, created_by, updated_at, updated_by, config) VALUES (1, 'WEB_PAGE', 1, '访问情况', 'EMBEDDING_RECALL', null, '访问情况', '2023-06-11 19:36:47', 'admin', '2023-06-21 15:26:46', 'admin', '{"params":{"1":[{"key":"name","paramType":"SEMANTIC","elementId":2}]}, "url":"www.test.com"}'); +-- insert into s2_plugin (id, `type`, `domain`, pattern, parse_mode, parse_mode_config, `name`, created_at, created_by, updated_at, updated_by, config) VALUES (2, 'DSL', null, '', 'FUNCTION_CALL', null, '访问情况', '2023-06-11 19:36:47', 'admin', '2023-06-21 15:26:46', 'admin', ''); -- semantic data insert into s2_database (id, domain_id , `name`, description, `type` ,config ,created_at ,created_by ,updated_at ,updated_by) VALUES(1, 1, 'H2数据实例', '', 'h2', '{"password":"semantic","url":"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false","userName":"root"}', '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin'); @@ -48,7 +48,8 @@ insert into s2_dimension (id , domain_id, datasource_id, `name`, biz_name, descr insert into s2_dimension (id , domain_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(6, 2, 4, '风格', 'genre', '风格', 1, 2, 'categorical', NULL, 'genre', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY'); insert into s2_dimension (id , domain_id, datasource_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, expr, created_at, created_by, updated_at, updated_by, semantic_type) VALUES(7, 2, 4, '歌手名', 'singer_name', '歌手名', 1, 2, 'categorical', NULL, 'singer_name', '2023-05-24 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin', 'CATEGORY'); insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org) VALUES(1, '超音数', 'supersonic', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin' ); -insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org) VALUES(2, '艺人库', 'singer', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin' ); +-- insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org) VALUES(2, '艺人库', 'singer', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin' ); +insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org, entity) VALUES(2, '艺人库', 'singer', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin','{"entityId": 7, "names": ["歌手", "艺人"]}' ); insert into s2_metric (id, domain_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(1, 1, '停留时长', 'stay_hours', '停留时长', 1, 2, 'expr', '{"expr":"s2_stay_time_statis_stay_hours","measures":[{"agg":"sum","expr":"stay_hours","isCreateMetric":1,"datasourceId":1,"bizName":"s2_stay_time_statis_stay_hours","name":"s2_stay_time_statis_stay_hours"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL ); insert into s2_metric (id, domain_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(2, 1, '访问次数', 'pv', '访问次数', 1, 0, 'expr', ' {"expr":"s2_pv_uv_statis_pv","measures":[{"agg":"sum","bizName":"s2_pv_uv_statis_pv","datasourceId":2,"expr":"pv","isCreateMetric":1,"name":"s2_pv_uv_statis_pv"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL ); insert into s2_metric (id, domain_id, `name`, biz_name, description, status, sensitive_level, `type`, type_params, created_at, created_by, updated_at, updated_by, data_format_type, data_format) VALUES(3, 1, '访问人数', 'uv', '访问人数', 1, 0, 'expr', ' {"expr":"s2_pv_uv_statis_uv","measures":[{"agg":"count_distinct","bizName":"s2_pv_uv_statis_uv","datasourceId":2,"expr":"uv","isCreateMetric":1,"name":"s2_pv_uv_statis_uv"}]}' , '2023-05-24 17:00:00', 'admin', '2023-05-25 00:00:00', 'admin', NULL, NULL ); diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index ba2b35a7b..18d01dca8 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -320,12 +320,14 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` `domain` varchar(100) NULL, `pattern` varchar(500) NULL, `parse_mode` varchar(100) NULL, + `parse_mode_config` LONGVARCHAR NULL, `name` varchar(100) NULL, `created_at` TIMESTAMP NULL, `created_by` varchar(100) null, `updated_at` TIMESTAMP NULL, `updated_by` varchar(100) NULL, `config` LONGVARCHAR NULL, + `comment` LONGVARCHAR NULL, PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_plugin IS 'plugin information table'; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/PageSchemaItemReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/PageSchemaItemReq.java index de2ace366..d3ce23d20 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/PageSchemaItemReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/PageSchemaItemReq.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.api.model.request; import com.tencent.supersonic.common.pojo.PageBaseReq; import lombok.Data; +import java.util.List; @Data public class PageSchemaItemReq extends PageBaseReq { @@ -10,7 +11,7 @@ public class PageSchemaItemReq extends PageBaseReq { private String name; private String bizName; private String createdBy; - private Long domainId; + private List domainIds; private Integer sensitiveLevel; private Integer status; } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DomainSchemaResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DomainSchemaResp.java index 00ee2dcca..a57be2e5b 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DomainSchemaResp.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/DomainSchemaResp.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.semantic.api.model.response; import java.util.List; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -10,7 +11,6 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class DomainSchemaResp extends DomainResp { - private List entityNames; private List metrics; private List dimensions; diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/MetricReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/MetricReq.java index 17016ef22..5e493a1aa 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/MetricReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/MetricReq.java @@ -15,5 +15,6 @@ public class MetricReq { private String where; private Long limit; private List order; + private boolean nativeQuery = false; } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java index 5d06e641e..73c22a320 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DimensionServiceImpl.java @@ -109,6 +109,7 @@ public class DimensionServiceImpl implements DimensionService { public PageInfo queryDimension(PageDimensionReq pageDimensionReq) { DimensionFilter dimensionFilter = new DimensionFilter(); BeanUtils.copyProperties(pageDimensionReq, dimensionFilter); + dimensionFilter.setDomainIds(pageDimensionReq.getDomainIds()); PageInfo dimensionDOPageInfo = PageHelper.startPage(pageDimensionReq.getCurrent(), pageDimensionReq.getPageSize()) .doSelectPageInfo(() -> queryDimension(dimensionFilter)); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java index f8fa1d773..41353d46a 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/DomainServiceImpl.java @@ -2,7 +2,9 @@ package com.tencent.supersonic.semantic.model.application; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.request.DomainReq; @@ -24,12 +26,7 @@ import com.tencent.supersonic.semantic.model.domain.pojo.Domain; import com.tencent.supersonic.semantic.model.domain.repository.DomainRepository; import com.tencent.supersonic.semantic.model.domain.utils.DomainConvert; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -47,14 +44,17 @@ public class DomainServiceImpl implements DomainService { private final MetricService metricService; private final DimensionService dimensionService; private final DatasourceService datasourceService; + private final UserService userService; public DomainServiceImpl(DomainRepository domainRepository, @Lazy MetricService metricService, - @Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService) { + @Lazy DimensionService dimensionService, @Lazy DatasourceService datasourceService, + UserService userService) { this.domainRepository = domainRepository; this.metricService = metricService; this.dimensionService = dimensionService; this.datasourceService = datasourceService; + this.userService = userService; } @@ -130,7 +130,7 @@ public class DomainServiceImpl implements DomainService { @Override public List getDomainListForAdmin(String userName) { List domainDOS = domainRepository.getDomainList(); - List orgIds = Lists.newArrayList(); + Set orgIds = Sets.newHashSet(); log.info("orgIds:{},userName:{}", orgIds, userName); Map> metricDomainMap = metricService.getMetrics().stream() .collect(Collectors.groupingBy(MetricResp::getDomainId)); @@ -144,7 +144,7 @@ public class DomainServiceImpl implements DomainService { @Override public List getDomainListForViewer(String userName) { List domainDOS = domainRepository.getDomainList(); - List orgIds = Lists.newArrayList(); + Set orgIds = Sets.newHashSet(); log.info("orgIds:{},userName:{}", orgIds, userName); return convertList(domainDOS, new HashMap<>(), new HashMap<>()).stream() .filter(domainDesc -> checkViewerPermission(orgIds, userName, domainDesc)) @@ -165,7 +165,7 @@ public class DomainServiceImpl implements DomainService { return ""; } Map map = getDomainFullPathMap(); - return map.containsKey(domainId) ? map.get(domainId) : ""; + return map.getOrDefault(domainId, ""); } @Override @@ -201,6 +201,32 @@ public class DomainServiceImpl implements DomainService { return getDomainList().stream().collect(Collectors.toMap(DomainResp::getId, a -> a, (k1, k2) -> k1)); } + @Override + public Set getDomainChildren(List domainIds) { + Set childDomains = new HashSet<>(); + if (CollectionUtils.isEmpty(domainIds)) { + return childDomains; + } + Map allDomainMap = getDomainMap(); + for (Long domainId : domainIds) { + DomainResp domain = allDomainMap.get(domainId); + if (domain != null) { + childDomains.add(domain); + Queue queue = new LinkedList<>(); + queue.add(domain); + while (!queue.isEmpty()) { + DomainResp currentDomain = queue.poll(); + for (DomainResp child : allDomainMap.values()) { + if (Objects.equals(child.getParentId(), currentDomain.getId())) { + childDomains.add(child); + queue.add(child); + } + } + } + } + } + return childDomains; + } public Map getDomainFullPathMap() { Map domainFullPathMap = new HashMap<>(); @@ -286,13 +312,16 @@ public class DomainServiceImpl implements DomainService { } - private boolean checkAdminPermission(List orgIds, String userName, DomainResp domainDesc) { + private boolean checkAdminPermission(Set orgIds, String userName, DomainResp domainDesc) { List admins = domainDesc.getAdmins(); List adminOrgs = domainDesc.getAdminOrgs(); if (admins.contains(userName) || domainDesc.getCreatedBy().equals(userName)) { return true; } + if (CollectionUtils.isEmpty(adminOrgs)) { + return false; + } for (String orgId : orgIds) { if (adminOrgs.contains(orgId)) { return true; @@ -301,7 +330,7 @@ public class DomainServiceImpl implements DomainService { return false; } - private boolean checkViewerPermission(List orgIds, String userName, DomainResp domainDesc) { + private boolean checkViewerPermission(Set orgIds, String userName, DomainResp domainDesc) { if (domainDesc.getIsOpen() == 1) { return true; } @@ -312,6 +341,9 @@ public class DomainServiceImpl implements DomainService { if (admins.contains(userName) || viewers.contains(userName) || domainDesc.getCreatedBy().equals(userName)) { return true; } + if (CollectionUtils.isEmpty(adminOrgs) && CollectionUtils.isEmpty(viewOrgs)) { + return false; + } for (String orgId : orgIds) { if (adminOrgs.contains(orgId)) { return true; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java index 3d12e030a..56f1289fd 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/application/MetricServiceImpl.java @@ -102,6 +102,9 @@ public class MetricServiceImpl implements MetricService { public PageInfo queryMetric(PageMetricReq pageMetricReq) { MetricFilter metricFilter = new MetricFilter(); BeanUtils.copyProperties(pageMetricReq, metricFilter); + Set domainResps = domainService.getDomainChildren(pageMetricReq.getDomainIds()); + List domainIds = domainResps.stream().map(DomainResp::getId).collect(Collectors.toList()); + metricFilter.setDomainIds(domainIds); PageInfo metricDOPageInfo = PageHelper.startPage(pageMetricReq.getCurrent(), pageMetricReq.getPageSize()) .doSelectPageInfo(() -> queryMetric(metricFilter)); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DomainService.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DomainService.java index e466d0ff8..72415ed79 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DomainService.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/DomainService.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.DomainSchemaResp; import java.util.List; import java.util.Map; +import java.util.Set; public interface DomainService { @@ -36,6 +37,8 @@ public interface DomainService { List getDomainListForViewer(String userName); + Set getDomainChildren(List domainId); + List fetchDomainSchema(DomainSchemaFilterReq filter, User user); } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java index 1771eb67b..b33e18e18 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/pojo/MetaFilter.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.semantic.model.domain.pojo; import lombok.Data; +import java.util.List; @Data @@ -14,7 +15,7 @@ public class MetaFilter { private String createdBy; - private Long domainId; + private List domainIds; private Integer sensitiveLevel; diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java index 62845f58d..b51d3b404 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/mapper/MetricDOCustomMapper.java @@ -4,6 +4,7 @@ package com.tencent.supersonic.semantic.model.infrastructure.mapper; import com.tencent.supersonic.semantic.model.domain.dataobject.MetricDO; import java.util.List; +import com.tencent.supersonic.semantic.model.domain.pojo.MetricFilter; import org.apache.ibatis.annotations.Mapper; @@ -14,4 +15,6 @@ public interface MetricDOCustomMapper { void batchUpdate(List metricDOS); + List query(MetricFilter metricFilter); + } diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java index 1d92f0299..4fa484720 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/DimensionRepositoryImpl.java @@ -7,6 +7,7 @@ import com.tencent.supersonic.semantic.model.domain.pojo.DimensionFilter; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOCustomMapper; import com.tencent.supersonic.semantic.model.infrastructure.mapper.DimensionDOMapper; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; @@ -97,8 +98,8 @@ public class DimensionRepositoryImpl implements DimensionRepository { if (dimensionFilter.getCreatedBy() != null) { dimensionDOExample.getOredCriteria().get(0).andCreatedByEqualTo(dimensionFilter.getCreatedBy()); } - if (dimensionFilter.getDomainId() != null) { - dimensionDOExample.getOredCriteria().get(0).andDomainIdEqualTo(dimensionFilter.getDomainId()); + if (CollectionUtils.isNotEmpty(dimensionFilter.getDomainIds())) { + dimensionDOExample.getOredCriteria().get(0).andDomainIdIn(dimensionFilter.getDomainIds()); } if (dimensionFilter.getSensitiveLevel() != null) { dimensionDOExample.getOredCriteria().get(0).andSensitiveLevelEqualTo(dimensionFilter.getSensitiveLevel()); diff --git a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java index 0f0b80733..10acb3c5a 100644 --- a/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java +++ b/semantic/model/src/main/java/com/tencent/supersonic/semantic/model/infrastructure/repository/MetricRepositoryImpl.java @@ -74,33 +74,7 @@ public class MetricRepositoryImpl implements MetricRepository { @Override public List getMetric(MetricFilter metricFilter) { - MetricDOExample metricDOExample = new MetricDOExample(); - metricDOExample.createCriteria(); - if (metricFilter.getId() != null) { - metricDOExample.getOredCriteria().get(0).andIdEqualTo(metricFilter.getId()); - } - if (metricFilter.getName() != null) { - metricDOExample.getOredCriteria().get(0).andNameLike("%" + metricFilter.getName() + "%"); - } - if (metricFilter.getBizName() != null) { - metricDOExample.getOredCriteria().get(0).andBizNameLike("%" + metricFilter.getBizName() + "%"); - } - if (metricFilter.getCreatedBy() != null) { - metricDOExample.getOredCriteria().get(0).andCreatedByEqualTo(metricFilter.getCreatedBy()); - } - if (metricFilter.getDomainId() != null) { - metricDOExample.getOredCriteria().get(0).andDomainIdEqualTo(metricFilter.getDomainId()); - } - if (metricFilter.getSensitiveLevel() != null) { - metricDOExample.getOredCriteria().get(0).andSensitiveLevelEqualTo(metricFilter.getSensitiveLevel()); - } - if (metricFilter.getStatus() != null) { - metricDOExample.getOredCriteria().get(0).andStatusEqualTo(metricFilter.getStatus()); - } - if (StringUtils.isNotBlank(metricFilter.getType())) { - metricDOExample.getOredCriteria().get(0).andTypeEqualTo(metricFilter.getType()); - } - return metricDOMapper.selectByExampleWithBLOBs(metricDOExample); + return metricDOCustomMapper.query(metricFilter); } @Override diff --git a/semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml b/semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml index 8054bdcc3..0e2047e18 100644 --- a/semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml +++ b/semantic/model/src/main/resources/mapper/custom/MetricDOCustomMapper.xml @@ -97,4 +97,29 @@ + + diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java index c2155807f..2d4455aff 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java @@ -42,6 +42,7 @@ public class QueryParser { if (!parseSqlReq.getSql().isEmpty()) { return parser(parseSqlReq); } + metricReq.setNativeQuery(queryStructReq.getNativeQuery()); return parser(metricReq); } @@ -94,7 +95,7 @@ public class QueryParser { } public QueryStatement parser(MetricReq metricCommand) { - return parser(metricCommand, true); + return parser(metricCommand, !metricCommand.isNativeQuery()); } public QueryStatement parser(MetricReq metricCommand, boolean isAgg) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java index ae6d087d0..db5c399dd 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/calcite/sql/render/SourceRender.java @@ -2,12 +2,6 @@ package com.tencent.supersonic.semantic.query.parser.calcite.sql.render; import com.tencent.supersonic.semantic.api.query.request.MetricReq; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DimensionNode; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; -import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Constants; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.DataSource; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Dimension; @@ -15,7 +9,13 @@ import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Identify; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Measure; import com.tencent.supersonic.semantic.query.parser.calcite.dsl.Metric; import com.tencent.supersonic.semantic.query.parser.calcite.schema.SemanticSchema; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.Renderer; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.TableView; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DataSourceNode; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.DimensionNode; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.FilterNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.IdentifyNode; +import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.semantic.query.parser.calcite.sql.node.SemanticNode; import java.util.ArrayList; import java.util.HashSet; @@ -33,7 +33,7 @@ import org.springframework.util.CollectionUtils; @Slf4j public class SourceRender extends Renderer { - public static TableView renderOne(String alias, List fieldWhere, + public static TableView renderOne(String alias, List fieldWheres, List reqMetrics, List reqDimensions, String queryWhere, DataSource datasource, SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { @@ -41,11 +41,8 @@ public class SourceRender extends Renderer { TableView output = new TableView(); List queryMetrics = new ArrayList<>(reqMetrics); List queryDimensions = new ArrayList<>(reqDimensions); + List fieldWhere = new ArrayList<>(fieldWheres); if (!fieldWhere.isEmpty()) { -// SqlNode sqlNode = SemanticNode.parse(queryWhere, scope); -// if (addWhere) { -// output.getFilter().add(sqlNode); -// } Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); whereDimMetric(fieldWhere, queryMetrics, queryDimensions, datasource, schema, dimensions, metrics); @@ -69,7 +66,8 @@ public class SourceRender extends Renderer { } } for (String dimension : queryDimensions) { - if(dimension.contains(Constants.DIMENSION_IDENTIFY) && queryDimensions.contains(dimension.split(Constants.DIMENSION_IDENTIFY)[1])){ + if (dimension.contains(Constants.DIMENSION_IDENTIFY) && queryDimensions.contains( + dimension.split(Constants.DIMENSION_IDENTIFY)[1])) { continue; } buildDimension(dimension.contains(Constants.DIMENSION_IDENTIFY) ? dimension : "", @@ -213,14 +211,15 @@ public class SourceRender extends Renderer { continue; } String filterField = field; - if(field.contains(Constants.DIMENSION_IDENTIFY)) { + if (field.contains(Constants.DIMENSION_IDENTIFY)) { filterField = field.split(Constants.DIMENSION_IDENTIFY)[1]; } - addField(filterField,field,datasource,schema,dimensions,metrics); + addField(filterField, field, datasource, schema, dimensions, metrics); } } - private static void addField(String field,String oriField,DataSource datasource, SemanticSchema schema, Set dimensions, + private static void addField(String field, String oriField, DataSource datasource, SemanticSchema schema, + Set dimensions, Set metrics) { Optional dimension = datasource.getDimensions().stream() .filter(d -> d.getName().equalsIgnoreCase(field)).findFirst(); @@ -251,9 +250,11 @@ public class SourceRender extends Renderer { Optional datasourceMetric = schema.getMetrics() .stream().filter(m -> m.getName().equalsIgnoreCase(field)).findFirst(); if (datasourceMetric.isPresent()) { - Set measures = datasourceMetric.get().getMetricTypeParams().getMeasures().stream().map(m->m.getName()).collect( - Collectors.toSet()); - if(datasource.getMeasures().stream().map(m->m.getName()).collect(Collectors.toSet()).containsAll(measures)){ + Set measures = datasourceMetric.get().getMetricTypeParams().getMeasures().stream() + .map(m -> m.getName()).collect( + Collectors.toSet()); + if (datasource.getMeasures().stream().map(m -> m.getName()).collect(Collectors.toSet()) + .containsAll(measures)) { metrics.add(oriField); return; } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java index 758ff94b7..58fc6cd16 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java @@ -197,9 +197,9 @@ public class CalculateAggConverter implements SemanticConverter { public String getOverSelect(QueryStructReq queryStructCmd, boolean isOver) { String aggStr = queryStructCmd.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s", + return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s", f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(), - f.getColumn()); + f.getFunc().getOperator(),f.getColumn()); } else { return f.getColumn(); } @@ -335,9 +335,9 @@ public class CalculateAggConverter implements SemanticConverter { String aggStr = queryStructCmd.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { return String.format( - "if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s", + "if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s", f.getColumn(), f.getColumn(), f.getColumn(), f.getColumn(), - f.getColumn()); + f.getColumn(),f.getFunc().getOperator(),f.getColumn()); } else { return f.getColumn(); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DateUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DateUtils.java index 978143f3e..01573fddd 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DateUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DateUtils.java @@ -39,7 +39,7 @@ public class DateUtils { private String sysDateWeekCol; public Boolean recentMode(DateConf dateInfo) { - if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT_UNITS == dateInfo.getDateMode() + if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT == dateInfo.getDateMode() && DAY.equalsIgnoreCase(dateInfo.getPeriod()) && Objects.nonNull(dateInfo.getUnit())) { return true; } @@ -47,7 +47,7 @@ public class DateUtils { } public boolean hasAvailableDataMode(DateConf dateInfo) { - if (Objects.nonNull(dateInfo) && DateConf.DateMode.AVAILABLE_TIME == dateInfo.getDateMode()) { + if (Objects.nonNull(dateInfo) && DateConf.DateMode.AVAILABLE == dateInfo.getDateMode()) { return true; } return false; @@ -263,16 +263,16 @@ public class DateUtils { public String getDateWhereStr(DateConf dateInfo, ItemDateResp dateDate) { String dateStr = ""; switch (dateInfo.getDateMode()) { - case BETWEEN_CONTINUOUS: + case BETWEEN: dateStr = betweenDateStr(dateDate, dateInfo); break; - case LIST_DISCRETE: + case LIST: dateStr = listDateStr(dateDate, dateInfo); break; - case RECENT_UNITS: + case RECENT: dateStr = recentDateStr(dateDate, dateInfo); break; - case AVAILABLE_TIME: + case AVAILABLE: dateStr = hasDataModeStr(dateDate, dateInfo); break; default: diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java index 2b7f87ac8..23d5756a2 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java @@ -101,10 +101,10 @@ public class QueryStructUtils { if (Objects.isNull(dateDate) || Strings.isEmpty(dateDate.getStartDate()) && Strings.isEmpty(dateDate.getEndDate())) { - if (dateInfo.getDateMode().equals(DateMode.LIST_DISCRETE)) { + if (dateInfo.getDateMode().equals(DateMode.LIST)) { return dateUtils.listDateStr(dateDate, dateInfo); } - if (dateInfo.getDateMode().equals(DateMode.BETWEEN_CONTINUOUS)) { + if (dateInfo.getDateMode().equals(DateMode.BETWEEN)) { return dateUtils.betweenDateStr(dateDate, dateInfo); } if (dateUtils.hasAvailableDataMode(dateInfo)) {